diff --git a/.devcontainer/.MakeParticleStack.dff b/.devcontainer/.MakeParticleStack.dff new file mode 100644 index 00000000..f6be1f18 --- /dev/null +++ b/.devcontainer/.MakeParticleStack.dff @@ -0,0 +1,15 @@ +Read coordinates from file?:: yes +X-dimension of original MIP:: 5760 +Y-dimension of original MIP:: 4092 +Input x,y,z coordinate file:: /home/himesb/tmp/coords_prepare_no_upsample.txt +Input image file:: image.mrc +Output star file:: /home/himesb/tmp/from_coords.star +Output particle stack:: /home/himesb/tmp/from_coords.mrc +Box size for particles (px.):: 256 +Pixel size of image (A):: 1.0 +Average defocus 1 (A):: 5000.0 +Average defocus 2 (A):: 5000.0 +Average defocus angle (deg):: 0.0 +Beam energy (keV):: 300.0 +Spherical aberration (mm):: 2.7 +Amplitude contrast:: 0.07 diff --git a/.devcontainer/.currentMakeParticleStack.dff b/.devcontainer/.currentMakeParticleStack.dff new file mode 100644 index 00000000..e69de29b diff --git a/.devcontainer/.currentMakeTemplateResult.dff b/.devcontainer/.currentMakeTemplateResult.dff new file mode 100644 index 00000000..e69de29b diff --git a/.devcontainer/check-version.sh b/.devcontainer/check-version.sh new file mode 100755 index 00000000..e1df522d --- /dev/null +++ b/.devcontainer/check-version.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash +# Check if devcontainer image version matches CONTAINER_VERSION_TOP +# Fails by default - only passes on explicit version match + +# EARLY MARKER: Write immediately to prove script was invoked +{ + echo "==========================================" + echo "INVOKED: $(date '+%Y-%m-%d %H:%M:%S')" + echo "PWD: $(pwd)" + echo "Script path: $0" + echo "BASH_SOURCE: ${BASH_SOURCE[0]}" + echo "==========================================" +} >> /tmp/devcontainer-check-invoked.log + +set -e # Exit on any error + +# Setup logging - write to file directly at each echo +LOG_FILE="/tmp/devcontainer-version-check-$(date +%s).log" + +# Helper function to log to both stdout and file +log() { + #echo "$@" + echo "$@" >> "${LOG_FILE}" +} + +# Initialize log file +echo "Script started: $(date '+%Y-%m-%d %H:%M:%S')" > "${LOG_FILE}" + +# Get the directory where this script is located +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +WORKSPACE_ROOT="$(dirname "$SCRIPT_DIR")" + +log "==================================================" +log "DevContainer Version Check" +log "==================================================" +log "Timestamp: $(date '+%Y-%m-%d %H:%M:%S')" +log "Log file: ${LOG_FILE}" +log "" +log "📂 Workspace root: ${WORKSPACE_ROOT}" + +VERSION_FILE="${WORKSPACE_ROOT}/.vscode/CONTAINER_VERSION_TOP" +REPO_FILE="${WORKSPACE_ROOT}/.vscode/CONTAINER_REPO_NAME" +DEVCONTAINER_FILE="${WORKSPACE_ROOT}/.devcontainer/devcontainer.json" + +log "📄 Checking version file: ${VERSION_FILE}" +log "📄 Checking repo file: ${REPO_FILE}" +log "📄 Checking devcontainer file: ${DEVCONTAINER_FILE}" + +# Verify required files exist +if [ ! -f "$VERSION_FILE" ]; then + log "" + log "❌ ERROR: CONTAINER_VERSION_TOP file not found at: $VERSION_FILE" + log " This file is required to validate container version." + exit 1 +fi + +if [ ! -f "$REPO_FILE" ]; then + log "" + log "❌ ERROR: CONTAINER_REPO_NAME file not found at: $REPO_FILE" + log " This file is required to validate container repository." + exit 1 +fi + +if [ ! -f "$DEVCONTAINER_FILE" ]; then + log "" + log "❌ ERROR: devcontainer.json not found at: $DEVCONTAINER_FILE" + exit 1 +fi + +# Read expected values from config files +EXPECTED_VERSION=$(cat "$VERSION_FILE" | tr -d '[:space:]') +EXPECTED_REPO=$(cat "$REPO_FILE" | tr -d '[:space:]') + +if [ -z "$EXPECTED_VERSION" ]; then + log "" + log "❌ ERROR: CONTAINER_VERSION_TOP file is empty" + log " The version file must contain a valid version number." + exit 1 +fi + +if [ -z "$EXPECTED_REPO" ]; then + log "" + log "❌ ERROR: CONTAINER_REPO_NAME file is empty" + log " The repo file must contain a valid repository path." + exit 1 +fi + +# Construct expected image string +EXPECTED_IMAGE="${EXPECTED_REPO}:v${EXPECTED_VERSION}" + +# Extract current image from devcontainer.json +CURRENT_IMAGE=$(grep -oP '"image":\s*"\K[^"]+' "$DEVCONTAINER_FILE") + +log "" +log "📋 CONTAINER_REPO_NAME specifies: ${EXPECTED_REPO}" +log "📋 CONTAINER_VERSION_TOP specifies: v${EXPECTED_VERSION}" +log "📋 Expected image: ${EXPECTED_IMAGE}" +log "" +log "🔧 devcontainer.json image field: ${CURRENT_IMAGE}" + +log "" +log "🔍 Comparing images..." +log " Expected: ${EXPECTED_IMAGE}" +log " Current: ${CURRENT_IMAGE}" + +# Compare images +if [ "$EXPECTED_IMAGE" != "$CURRENT_IMAGE" ]; then + log "" + log "❌ ERROR: Image mismatch detected" + log " devcontainer.json uses: ${CURRENT_IMAGE}" + log " Expected image: ${EXPECTED_IMAGE}" + log "" + log " Action required:" + log " Update devcontainer.json image field to: ${EXPECTED_IMAGE}" + exit 1 +fi + +# Success case - images match +log "" +log "✅ Container image check passed: ${EXPECTED_IMAGE}" +log "==================================================" +exit 0 diff --git a/.devcontainer/configure-git.sh b/.devcontainer/configure-git.sh new file mode 100755 index 00000000..1c58326d --- /dev/null +++ b/.devcontainer/configure-git.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# +# Configure git identity inside the devcontainer. +# +# Priority: +# 1. GIT_AUTHOR_NAME / GIT_AUTHOR_EMAIL env vars (if set) +# 2. Fall through — let VS Code's default forwarding handle it +# +# Called by devcontainer.json postCreateCommand. + +configure_if_unset() { + local current_name current_email + + current_name="$(git config --global user.name 2>/dev/null)" + current_email="$(git config --global user.email 2>/dev/null)" + + # If git identity is already configured (e.g., VS Code forwarded it), done + if [ -n "$current_name" ] && [ -n "$current_email" ]; then + echo "Git identity already configured: ${current_name} <${current_email}>" + return 0 + fi + + # Try env vars + if [ -n "$GIT_AUTHOR_NAME" ] && [ -n "$GIT_AUTHOR_EMAIL" ]; then + git config --global user.name "$GIT_AUTHOR_NAME" + git config --global user.email "$GIT_AUTHOR_EMAIL" + echo "Git identity set from env: ${GIT_AUTHOR_NAME} <${GIT_AUTHOR_EMAIL}>" + return 0 + fi + + echo "WARNING: Git identity not configured." + echo " Set GIT_AUTHOR_NAME and GIT_AUTHOR_EMAIL in devcontainer.json remoteEnv," + echo " or configure git globally on your local machine." + return 0 # Non-fatal — don't block container creation +} + +configure_if_unset diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..a69aa5e6 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,69 @@ +{ + "name": "cisTEMx-${localEnv:USER}-${localWorkspaceFolderBasename}", + "workspaceMount": "source=${localWorkspaceFolder},target=${localWorkspaceFolder},type=bind,consistency=cached", + "workspaceFolder": "${localWorkspaceFolder}", + // NOTE: Container image must match .vscode/CONTAINER_REPO_NAME and CONTAINER_VERSION_TOP + // Update all three when changing container version or repository + "image": "ghcr.io/stochasticanalytics/cistem_build_env:v4.0.0", + "initializeCommand": ".devcontainer/check-version.sh", + "postCreateCommand": "bash .devcontainer/configure-git.sh && source ~/.nvm/nvm.sh && npm install -g @anthropic-ai/sandbox-runtime pyright typescript-language-server typescript", + "remoteUser": "cisTEMdev", + "hostRequirements": { + "gpu": "optional" + }, + "remoteEnv": { + // We need to set this so we can use wxmctrl -lpx | grep ${WORKSPACE_BASENAME} | awk '{print $1}' to find window IDs + // then xdotool get_desktop_for_window to get the desktop number + // then later compare to xdotool get_desktop to see if the window is on the current desktop, if not we can run a hook to push notify if claude is read. + // FIXME this should just come from name + "WORKSPACE_BASENAME": "${localWorkspaceFolderBasename}", + "WORKSPACE_CONTAINER_NAME": "cisTEMx-${localEnv:USER}-${localWorkspaceFolderBasename}", + // Persist Claude Code sessions, plans, and memories across container rebuilds + "CLAUDE_CONFIG_DIR": "/sa_shared/.claude" + }, + // To add mounts, enable smudge/clean filters, define variable in bashrc ... cistem_mounts=("source=/scratch,target=/scratch,type=bind" "source=/sa_shared,target=/sa_shared,type=bind")" + // Leave the following line comment, it is used as a placeholder for the devcontainer-filter.sh script to insert the mounts. + // begin_mounts + "mounts": [ + "source=/scratch,target=/scratch,type=bind", + "source=/sa_shared,target=/sa_shared,type=bind" + ], + // end_mounts + // UID is 1000 in base container (default) + "updateRemoteUserUID": false, + // Network configuration: Using host networking for maximum compatibility. + // Whisper Assistant server runs on separate whisper-net bridge network with port 4444 exposed to host. + // This devcontainer accesses whisper via localhost:4444 through host network. + "runArgs": [ + "--gpus", + "all", + "--security-opt", + "apparmor=unconfined", + "-it", + "--net", + "host", + "-e", + "DISPLAY=${env:DISPLAY}", + "-v", + "${env:XAUTHORITY}:/home/cisTEMdev/.Xauthority", + "-v", + "/tmp/.X11-unix:/tmp/.X11-unix" + ], + "customizations": { + "vscode": { + "extensions": [ + "maelvalais.autoconf", + "ms-python.autopep8", + "ms-vscode.cpptools", + "ms-vscode.cpptools-extension-pack", + "cschlosser.doxdocgen", + "eamodio.gitlens", + "DavidAnson.vscode-markdownlint", + "ms-python.vscode-pylance", + "ms-python.python", + "frinkr.vscode-tabify", + "jomeinaster.bracket-peek", + "onnovalkering.vscode-singularity" + ] + } + } \ No newline at end of file diff --git a/.devcontainer/launch-container.py b/.devcontainer/launch-container.py new file mode 100755 index 00000000..1e80384a --- /dev/null +++ b/.devcontainer/launch-container.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Launch cisTEMx development container and attach Cursor. + +This script replicates the functionality of devcontainer.json for editors +that don't support the Dev Containers extension (like Cursor). + +Default behavior: Start the container and launch Cursor attached to it. +""" + +import argparse +import os +import subprocess +import sys +from pathlib import Path + + +def get_workspace_info() -> dict: + """Determine workspace paths and container name. + + Returns: + dict with keys: workspace_root, workspace_basename, container_name + """ + # Script is in .devcontainer/, workspace is parent + script_dir = Path(__file__).resolve().parent + workspace_root = script_dir.parent + workspace_basename = workspace_root.name + + user = os.environ.get("USER", "unknown") + container_name = f"cisTEMx-{user}-{workspace_basename}" + + return { + "workspace_root": workspace_root, + "workspace_basename": workspace_basename, + "container_name": container_name, + "user": user, + } + + +def read_image_config(workspace_root: Path) -> str: + """Read image version from .vscode/ config files. + + Args: + workspace_root: Path to workspace root directory + + Returns: + Full image name with tag (e.g., ghcr.io/stochasticanalytics/cistem_build_env:v3.1.0) + """ + version_file = workspace_root / ".vscode" / "CONTAINER_VERSION_TOP" + repo_file = workspace_root / ".vscode" / "CONTAINER_REPO_NAME" + + if not version_file.exists(): + print(f"Error: Version file not found: {version_file}", file=sys.stderr) + sys.exit(1) + if not repo_file.exists(): + print(f"Error: Repo file not found: {repo_file}", file=sys.stderr) + sys.exit(1) + + version = version_file.read_text().strip() + repo = repo_file.read_text().strip() + + return f"{repo}:v{version}" + + +def run_version_check(workspace_root: Path) -> bool: + """Execute the existing check-version.sh validation. + + Args: + workspace_root: Path to workspace root directory + + Returns: + True if validation passes, False otherwise + """ + check_script = workspace_root / ".devcontainer" / "check-version.sh" + + if not check_script.exists(): + print(f"Warning: Version check script not found: {check_script}", file=sys.stderr) + return True # Don't fail if script doesn't exist + + result = subprocess.run( + [str(check_script)], + cwd=str(workspace_root), + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print("Version check failed:", file=sys.stderr) + print(result.stderr, file=sys.stderr) + return False + + return True + + +def get_container_status(container_name: str) -> str: + """Check if container is running, stopped, or non-existent. + + Args: + container_name: Name of the Docker container + + Returns: + One of: "running", "stopped", "none" + """ + # Check if container exists at all + result = subprocess.run( + ["docker", "inspect", "--format", "{{.State.Status}}", container_name], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + return "none" + + status = result.stdout.strip() + if status == "running": + return "running" + else: + return "stopped" + + +def stop_container(container_name: str) -> bool: + """Stop the running container. + + Args: + container_name: Name of the Docker container + + Returns: + True if stopped successfully or wasn't running + """ + status = get_container_status(container_name) + + if status == "none": + print(f"Container '{container_name}' does not exist.") + return True + + if status == "stopped": + print(f"Container '{container_name}' is already stopped.") + return True + + print(f"Stopping container '{container_name}'...") + result = subprocess.run( + ["docker", "stop", container_name], + capture_output=True, + text=True, + ) + + if result.returncode != 0: + print(f"Failed to stop container: {result.stderr}", file=sys.stderr) + return False + + print(f"Container '{container_name}' stopped.") + return True + + +def remove_container(container_name: str) -> bool: + """Remove a stopped container. + + Args: + container_name: Name of the Docker container + + Returns: + True if removed successfully or didn't exist + """ + result = subprocess.run( + ["docker", "rm", container_name], + capture_output=True, + text=True, + ) + return result.returncode == 0 + + +def start_container(info: dict, image: str, interactive: bool = False) -> bool: + """Start the container with all mounts and configuration. + + Args: + info: Workspace info dict from get_workspace_info() + image: Full Docker image name with tag + interactive: If True, run interactively with TTY + + Returns: + True if container is running after this call + """ + container_name = info["container_name"] + workspace_root = info["workspace_root"] + workspace_basename = info["workspace_basename"] + + status = get_container_status(container_name) + + # If running and not interactive, we're done + if status == "running" and not interactive: + print(f"Container '{container_name}' is already running.") + return True + + # If stopped, start it + if status == "stopped": + print(f"Starting stopped container '{container_name}'...") + result = subprocess.run( + ["docker", "start", container_name], + capture_output=True, + text=True, + ) + if result.returncode != 0: + print(f"Failed to start container: {result.stderr}", file=sys.stderr) + return False + print(f"Container '{container_name}' started.") + return True + + # Container doesn't exist, create it + print(f"Creating container '{container_name}'...") + + # Build docker run command + cmd = ["docker", "run"] + + if interactive: + cmd.extend(["-it", "--rm"]) + else: + cmd.append("-d") + + # Container name + cmd.extend(["--name", container_name]) + + # Network + cmd.extend(["--network", "host"]) + + # User (don't use --user, let the container handle it via entrypoint) + # The container's default user is cisTEMdev + + # Environment variables + display = os.environ.get("DISPLAY", ":0") + cmd.extend(["-e", f"DISPLAY={display}"]) + cmd.extend(["-e", f"WORKSPACE_BASENAME={workspace_basename}"]) + cmd.extend(["-e", f"WORKSPACE_CONTAINER_NAME={container_name}"]) + + # Workspace mount + cmd.extend(["-v", f"{workspace_root}:/workspaces/cisTEMx"]) + + # Additional mounts from devcontainer.json + cmd.extend(["-v", "/scratch:/scratch"]) + cmd.extend(["-v", "/sa_shared:/sa_shared"]) + + # X11 forwarding + xauthority = os.environ.get("XAUTHORITY", os.path.expanduser("~/.Xauthority")) + cmd.extend(["-v", f"{xauthority}:/home/cisTEMdev/.Xauthority"]) + cmd.extend(["-v", "/tmp/.X11-unix:/tmp/.X11-unix"]) + + # Working directory inside container + cmd.extend(["-w", "/workspaces/cisTEMx"]) + + # Image + cmd.append(image) + + # For detached mode, run a long-lived process + if not interactive: + cmd.extend(["sleep", "infinity"]) + + print(f"Running: {' '.join(cmd)}") + + if interactive: + # For interactive, replace current process + os.execvp("docker", cmd) + else: + # Don't capture output - let Docker show pull progress and other messages + result = subprocess.run(cmd) + if result.returncode != 0: + print(f"Failed to create container (exit code {result.returncode})", file=sys.stderr) + return False + print(f"Container '{container_name}' created and running.") + return True + + +def attach_cursor(info: dict) -> None: + """Launch Cursor attached to the running container. + + Args: + info: Workspace info dict from get_workspace_info() + """ + container_name = info["container_name"] + + # Hex-encode the container name for the URI + hex_name = container_name.encode().hex() + + # Build the remote URI + # Format: vscode-remote://attached-container+/path + remote_uri = f"vscode-remote://attached-container+{hex_name}/workspaces/cisTEMx" + + print(f"Launching Cursor attached to '{container_name}'...") + print(f"URI: {remote_uri}") + + # Launch Cursor with the remote URI + subprocess.Popen( + ["cursor", "--folder-uri", remote_uri], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + print("Cursor launched. It may take a moment to connect.") + + +def show_status(info: dict) -> None: + """Display container status information. + + Args: + info: Workspace info dict from get_workspace_info() + """ + container_name = info["container_name"] + status = get_container_status(container_name) + + print(f"Container: {container_name}") + print(f"Status: {status}") + + if status == "running": + # Get additional info + result = subprocess.run( + ["docker", "inspect", "--format", + "Created: {{.Created}}\nImage: {{.Config.Image}}", + container_name], + capture_output=True, + text=True, + ) + if result.returncode == 0: + print(result.stdout) + + +def main() -> int: + """Parse arguments and execute requested action.""" + parser = argparse.ArgumentParser( + description="Launch cisTEMx development container for Cursor", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + %(prog)s Start container and launch Cursor + %(prog)s --no-cursor Start container only + %(prog)s --stop Stop the container + %(prog)s --restart Restart container and launch Cursor + %(prog)s --shell Start container and get interactive shell + %(prog)s --status Show container status + """, + ) + + parser.add_argument( + "--no-cursor", + action="store_true", + help="Start container without launching Cursor", + ) + parser.add_argument( + "--stop", + action="store_true", + help="Stop the running container", + ) + parser.add_argument( + "--restart", + action="store_true", + help="Restart the container (stop, remove, start fresh)", + ) + parser.add_argument( + "--status", + action="store_true", + help="Show container status", + ) + parser.add_argument( + "--shell", + action="store_true", + help="Start container and attach an interactive shell (implies --no-cursor)", + ) + + args = parser.parse_args() + + # Get workspace info + info = get_workspace_info() + + # Handle status command + if args.status: + show_status(info) + return 0 + + # Handle stop command + if args.stop: + return 0 if stop_container(info["container_name"]) else 1 + + # Read image configuration + image = read_image_config(info["workspace_root"]) + print(f"Using image: {image}") + + # Run version check + if not run_version_check(info["workspace_root"]): + print("Version check failed. Please fix the configuration.", file=sys.stderr) + return 1 + + # Handle restart command + if args.restart: + stop_container(info["container_name"]) + remove_container(info["container_name"]) + + # Handle shell command (interactive mode) + if args.shell: + start_container(info, image, interactive=True) + # If we get here, exec failed + return 1 + + # Start container (default) + if not start_container(info, image): + return 1 + + # Launch Cursor unless --no-cursor specified + if not args.no_cursor: + attach_cursor(info) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github-workflows-example.yml b/.github-workflows-example.yml new file mode 100644 index 00000000..5f8b27b9 --- /dev/null +++ b/.github-workflows-example.yml @@ -0,0 +1,94 @@ +# Example GitHub Actions workflow for the new linting setup +# This replaces the old black/isort/flake8/mypy workflow + +name: Code Quality + +on: + push: + branches: [ main, develop, ctf3d_work ] + pull_request: + branches: [ main, develop ] + +jobs: + lint-and-type-check: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8, 3.9, "3.10", "3.11"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install ruff pyright bandit[toml] safety + pip install -e .[dev] + + # Fast linting and formatting with Ruff + - name: Lint with Ruff + run: | + # Check for lint issues + ruff check python/ --output-format=github + # Check formatting + ruff format python/ --check + + # Static type checking with Pyright + - name: Type check with Pyright + run: pyright python/ + + # Security linting with Bandit + - name: Security check with Bandit + run: bandit -r python/ -c pyproject.toml -f json -o bandit-report.json + continue-on-error: true + + # Dependency vulnerability scanning with Safety + - name: Check dependencies with Safety + run: safety check --json --output safety-report.json + continue-on-error: true + + # Upload security reports as artifacts + - name: Upload security reports + uses: actions/upload-artifact@v3 + if: always() + with: + name: security-reports-${{ matrix.python-version }} + path: | + bandit-report.json + safety-report.json + + # Optional: Run more comprehensive checks occasionally + comprehensive-analysis: + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + + - name: Install analysis tools + run: | + python -m pip install --upgrade pip + pip install ruff pyright bandit[toml] safety pylint radon + pip install -e .[dev] + + - name: Run comprehensive analysis + run: | + # Ruff with all rules enabled + ruff check python/ --select ALL --ignore D203,D212,PLR0913,PLR2004,T201 + + # Code complexity analysis + radon cc python/ -a -nc + radon mi python/ -nc + + # Deep code quality analysis with Pylint (advisory) + pylint python/ --reports=y --exit-zero || true diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml new file mode 100644 index 00000000..c94e734a --- /dev/null +++ b/.github/workflows/code-style.yml @@ -0,0 +1,88 @@ +name: Code Style + +on: + push: + branches: + - main + - '*_ci' # CI branches automatically get style fixes applied + paths: + # Only run when Python code or style config changes + - 'python/**/*.py' # Python source files only + - 'pyproject.toml' # Ruff configuration + - '.github/workflows/code-style.yml' # This workflow file + pull_request: + branches: + - main + paths: + # Same path filters for PRs to avoid unnecessary runs + - 'python/**/*.py' + - 'pyproject.toml' + - '.github/workflows/code-style.yml' + workflow_dispatch: # Manual trigger bypasses path restrictionsn: + +jobs: + style-check: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' # Match project target version + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-ruff-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-ruff- + + - name: Install Ruff + run: python -m pip install --upgrade pip ruff + + - name: Run Ruff linting + run: | + echo "::group::Ruff Linting" + ruff check python/ --output-format=github # GitHub annotations + echo "::endgroup::" + + - name: Run Ruff formatting check + run: | + echo "::group::Ruff Formatting" + ruff format --check --diff python/ # Show what would change + echo "::endgroup::" + + # Auto-fix for CI branches - helps maintain code quality automatically + auto-fix: + runs-on: ubuntu-latest + if: github.event_name == 'push' && endsWith(github.ref_name, '_ci') + needs: style-check # Only run if style check completes + + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Ruff + run: pip install ruff + + - name: Auto-fix with Ruff + run: | + ruff check python/ --fix # Fix what can be auto-fixed + ruff format python/ # Apply consistent formatting + + - name: Commit fixes + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add python/ + git diff --staged --quiet || git commit -m "Auto-fix: Apply Ruff formatting and linting fixes" + git push diff --git a/.github/workflows/gpu-tests.yml b/.github/workflows/gpu-tests.yml new file mode 100644 index 00000000..3eef07f0 --- /dev/null +++ b/.github/workflows/gpu-tests.yml @@ -0,0 +1,283 @@ +name: emClarity GPU Tests + +on: + push: + branches: + - main + - '*_ci' # CI branches automatically get style fixes applied + paths: + - 'python/cuda_ops/**' + - 'python/masking/**' # Contains GPU-accelerated functions + - '.github/workflows/gpu-tests.yml' + pull_request: + branches: + - main + - '*_ci' # CI branches automatically get style fixes applied + paths: + - 'python/cuda_ops/**' + - 'python/masking/**' + workflow_dispatch: # Allow manual triggering + +jobs: + gpu-tests: + # Use self-hosted runners with GPU support for private repo + runs-on: [self-hosted] + + steps: + - uses: actions/checkout@v4 + + - name: Check GPU availability and environment + run: | + echo "=== GPU Environment Check ===" + nvidia-smi || (echo "ERROR: nvidia-smi not found" && exit 1) + echo "" + echo "CUDA environment:" + echo "CUDA_VISIBLE_DEVICES: ${CUDA_VISIBLE_DEVICES:-not set}" + nvcc --version || echo "Warning: nvcc not found" + echo "" + echo "Setting environment variables..." + export HAS_GPU=true + echo "HAS_GPU=true" >> $GITHUB_ENV + + - name: Set up Python 3.12 + uses: actions/setup-python@v4 + with: + python-version: '3.12' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + cd python + + echo "=== Installing core dependencies ===" + pip install numpy scipy mrcfile jsonschema matplotlib pillow + pip install pytest pytest-cov + + echo "=== Installing CuPy for GPU support ===" + # Try CUDA 12.x first, then 11.x as fallback + # Version constraint >=12.0.0 matches pyproject.toml [gpu] extras + if pip install 'cupy-cuda12x>=12.0.0'; then + echo "✓ CuPy CUDA 12.x installed successfully" + elif pip install 'cupy-cuda11x>=12.0.0'; then + echo "✓ CuPy CUDA 11.x installed successfully" + else + echo "❌ Failed to install CuPy" + exit 1 + fi + + echo "=== Verifying GPU setup ===" + python -c " + import cupy as cp + print(f'CuPy version: {cp.__version__}') + print(f'CUDA version: {cp.cuda.runtime.runtimeGetVersion()}') + print(f'GPU count: {cp.cuda.runtime.getDeviceCount()}') + + # Get GPU name using correct CuPy API + with cp.cuda.Device(0): + props = cp.cuda.runtime.getDeviceProperties(0) + gpu_name = props['name'].decode('utf-8') + print(f'GPU 0 name: {gpu_name}') + print(f'GPU 0 memory: {props[\"totalGlobalMem\"] / 1024**3:.1f} GB') + " + + - name: Test CUDA operations + run: | + cd python + echo "=== Testing basic CUDA operations ===" + python -c " + import cupy as cp + import numpy as np + + print('Testing basic CuPy operations...') + a = cp.array([1, 2, 3, 4, 5]) + b = cp.array([2, 3, 4, 5, 6]) + c = a + b + print(f'✓ GPU array addition: {c}') + + print('Testing CUDA memory management...') + large_array = cp.random.rand(1000, 1000, dtype=cp.float32) + result = cp.sum(large_array) + print(f'✓ Large array sum: {result:.2f}') + + print('Testing emClarity CUDA modules...') + try: + from cuda_ops.emc_cuda_basic_ops import CudaBasicOps + ops = CudaBasicOps() + if ops.is_ready(): + print('✓ CudaBasicOps loaded successfully') + # Test array addition + test_a = cp.array([1.0, 2.0, 3.0], dtype=cp.float32) + test_b = cp.array([4.0, 5.0, 6.0], dtype=cp.float32) + result = ops.add_arrays(test_a, test_b) + expected = test_a + test_b + if cp.allclose(result, expected): + print('✓ CudaBasicOps array addition test passed') + else: + print('❌ CudaBasicOps array addition test failed') + exit(1) + else: + print('❌ CudaBasicOps not ready') + exit(1) + except ImportError as e: + print(f'â„šī¸ CudaBasicOps not available: {e}') + print('This is expected if CUDA modules are not compiled yet') + " + + echo "=== Testing cuda_ops module ===" + python -c " + try: + from cuda_ops.basic_array_ops import BasicArrayOps + ops = BasicArrayOps() + print('✓ BasicArrayOps module loaded successfully') + except ImportError as e: + print(f'â„šī¸ BasicArrayOps not available: {e}') + print('This is expected if CUDA modules are not compiled yet') + " + + - name: Run GPU unit tests + run: | + cd python + + echo "=== Running CUDA operations tests ===" + if ! python -m pytest cuda_ops/tests/ -v --tb=short; then + echo "❌ CUDA operations tests FAILED" + exit 1 + fi + echo "✅ CUDA operations tests PASSED" + + echo "=== Running GPU masking tests ===" + if python -m pytest masking/tests/ -v --tb=short -k "gpu or GPU" --collect-only -q | grep -q "no tests collected"; then + echo "â„šī¸ No GPU masking tests found, skipping" + else + if ! python -m pytest masking/tests/ -v --tb=short -k "gpu or GPU"; then + echo "❌ GPU masking tests FAILED" + exit 1 + fi + echo "✅ GPU masking tests PASSED" + fi + + echo "=== Running all available tests with GPU markers ===" + if python -m pytest -v --tb=short -k "gpu or cuda or GPU or CUDA" --collect-only -q | grep -q "no tests collected"; then + echo "â„šī¸ No additional GPU tests found" + else + if ! python -m pytest -v --tb=short -k "gpu or cuda or GPU or CUDA"; then + echo "❌ Additional GPU tests FAILED" + exit 1 + fi + echo "✅ Additional GPU tests PASSED" + fi + + - name: Performance benchmarks + run: | + cd python + echo "=== GPU vs CPU Performance Benchmarks ===" + python -c " + import time + import numpy as np + import cupy as cp + + # Memory info + print('GPU Memory Info:') + mempool = cp.get_default_memory_pool() + print(f' Used: {mempool.used_bytes() / 1024**3:.2f} GB') + print(f' Total: {mempool.total_bytes() / 1024**3:.2f} GB') + print() + + try: + from masking.emc_pad_zeros_3d import emc_pad_zeros_3d + + # Test different sizes + sizes = [(32, 32, 32), (64, 64, 64), (128, 128, 128)] + + for size in sizes: + print(f'Testing size {size}:') + test_image = np.random.rand(*size).astype(np.float32) + pad_width = [8, 8, 8] + + # CPU timing + start = time.time() + result_cpu = emc_pad_zeros_3d(test_image, pad_width, method='CPU') + cpu_time = time.time() - start + + # GPU timing + test_image_gpu = cp.asarray(test_image) + start = time.time() + result_gpu = emc_pad_zeros_3d(test_image_gpu, pad_width, method='GPU') + gpu_time = time.time() - start + + # Results + speedup = cpu_time / gpu_time if gpu_time > 0 else float('inf') + diff = np.abs(cp.asnumpy(result_gpu) - result_cpu).max() + + print(f' CPU time: {cpu_time:.4f}s') + print(f' GPU time: {gpu_time:.4f}s') + print(f' Speedup: {speedup:.2f}x') + print(f' Max diff: {diff:.2e}') + + if diff < 1e-5: + print(' ✓ Results match') + else: + print(' âš ī¸ Results differ') + print() + + except ImportError as e: + print(f'Masking module not available for benchmarks: {e}') + + # Basic CuPy benchmarks + print('Basic CuPy operations:') + sizes = [1000, 5000, 10000] + for n in sizes: + # Matrix multiplication benchmark + a_cpu = np.random.rand(n, n).astype(np.float32) + b_cpu = np.random.rand(n, n).astype(np.float32) + + # CPU + start = time.time() + c_cpu = np.dot(a_cpu, b_cpu) + cpu_time = time.time() - start + + # GPU + a_gpu = cp.asarray(a_cpu) + b_gpu = cp.asarray(b_cpu) + cp.cuda.Device().synchronize() # Ensure GPU is ready + + start = time.time() + c_gpu = cp.dot(a_gpu, b_gpu) + cp.cuda.Device().synchronize() # Wait for completion + gpu_time = time.time() - start + + speedup = cpu_time / gpu_time if gpu_time > 0 else float('inf') + print(f' Matrix {n}x{n}: CPU {cpu_time:.4f}s, GPU {gpu_time:.4f}s, Speedup {speedup:.2f}x') + " || echo "âš ī¸ Benchmarks completed with some issues (non-critical)" + + # Summary job - always runs to provide clear CI status + gpu-tests-summary: + runs-on: ubuntu-latest + needs: [gpu-tests] + if: always() + steps: + - name: GPU Tests Summary + run: | + echo "=== emClarity GPU Tests Summary ===" + echo "Status: ${{ needs.gpu-tests.result }}" + echo "Timestamp: $(date)" + echo "" + + if [ "${{ needs.gpu-tests.result }}" = "success" ]; then + echo "✅ All GPU tests passed successfully!" + echo "GPU-accelerated operations are working correctly." + elif [ "${{ needs.gpu-tests.result }}" = "failure" ]; then + echo "❌ GPU tests encountered failures." + echo "Check the logs above for specific issues." + echo "This may indicate problems with:" + echo " - CUDA/CuPy installation" + echo " - GPU hardware issues" + echo " - emClarity CUDA module compilation" + exit 1 + elif [ "${{ needs.gpu-tests.result }}" = "cancelled" ]; then + echo "âš ī¸ GPU tests were cancelled." + exit 1 + else + echo "❓ GPU tests completed with unknown status." + exit 1 + fi diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml new file mode 100644 index 00000000..aa7039b4 --- /dev/null +++ b/.github/workflows/security-scan.yml @@ -0,0 +1,92 @@ +name: Security Scan + +on: + push: + branches: + - main + - '*_ci' # Include CI branches for security validation + paths: + # Run when code or dependencies change that could affect security + - 'python/**/*.py' # Python source files + - 'pyproject.toml' # Project configuration + - 'requirements*.txt' # Dependency files + - '.github/workflows/security-scan.yml' # This workflow file + pull_request: + branches: + - main + paths: + # Same efficient filtering for PRs + - 'python/**/*.py' + - 'pyproject.toml' + - 'requirements*.txt' + - '.github/workflows/security-scan.yml' + workflow_dispatch: # Manual security scans always allowed + schedule: + # Weekly security audit regardless of code changes + - cron: '0 9 * * 1' # Mondays at 9 AM UTC + +jobs: + security-scan: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-security-${{ hashFiles('pyproject.toml', 'python/requirements*.txt') }} + restore-keys: | + ${{ runner.os }}-security- + + - name: Install security tools + run: | + python -m pip install --upgrade pip + pip install bandit[toml] safety + + - name: Run code security scan with Bandit + run: | + echo "::group::Code Security Analysis" + bandit -r python/ -f json -o bandit-report.json || true + bandit -r python/ -f txt + echo "::endgroup::" + + - name: Run dependency vulnerability scan + run: | + echo "::group::Dependency Vulnerability Scan" + safety check --json --output safety-report.json || true + safety check + echo "::endgroup::" + + - name: Upload security reports + uses: actions/upload-artifact@v4 + if: always() + with: + name: security-reports-${{ github.run_number }} + path: | + bandit-report.json + safety-report.json + retention-days: 90 # Keep security reports longer + + - name: Create security summary + if: always() + run: | + echo "## Security Scan Summary" >> $GITHUB_STEP_SUMMARY + echo "### Bandit (Code Security)" >> $GITHUB_STEP_SUMMARY + if [ -f bandit-report.json ]; then + echo "✅ Bandit scan completed - check artifacts for details" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Bandit scan failed" >> $GITHUB_STEP_SUMMARY + fi + echo "### Safety (Dependencies)" >> $GITHUB_STEP_SUMMARY + if [ -f safety-report.json ]; then + echo "✅ Safety scan completed - check artifacts for details" >> $GITHUB_STEP_SUMMARY + else + echo "❌ Safety scan failed" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/type-checking.yml b/.github/workflows/type-checking.yml new file mode 100644 index 00000000..29ca1240 --- /dev/null +++ b/.github/workflows/type-checking.yml @@ -0,0 +1,50 @@ +name: Type Checking + +on: + push: + branches: + - main + - '*_ci' # CI branches for testing workflows + paths: + # Only run when Python code or type config changes + - 'python/**' # Any Python files + - 'pyproject.toml' # Pyright configuration + - '.github/workflows/type-checking.yml' # This workflow file + pull_request: + branches: + - main + paths: + # Same path filters for PRs - efficiency over completeness + - 'python/**' + - 'pyproject.toml' + - '.github/workflows/type-checking.yml' + workflow_dispatch: # Allow manual trigger regardless of paths + +jobs: + type-check: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Cache pip dependencies + uses: actions/cache@v4 + with: + path: ~/.cache/pip + key: ${{ runner.os }}-pyright-${{ hashFiles('pyproject.toml') }} + restore-keys: | + ${{ runner.os }}-pyright- + + - name: Install Pyright + run: python -m pip install --upgrade pip pyright + + - name: Run type checking + run: | + echo "::group::Type Checking with Pyright" + pyright python/ + echo "::endgroup::" diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 00000000..f5117e84 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,163 @@ +name: Unit Tests + +# Focused on testing functionality only +# No linting - that's handled by dedicated workflows + +on: + push: + branches: [ main, ctf3d_work, develop ] # Development branches + paths: + # Broader path matching since tests validate overall functionality + - 'python/**' # Any Python code/config changes + - '.github/workflows/**' # Workflow changes affect testing + - 'requirements*.txt' # Dependency changes + pull_request: + branches: [ main, ctf3d_work ] + paths: + # Same broad matching for comprehensive PR validation + - 'python/**' + - '.github/workflows/**' + - 'requirements*.txt' + +jobs: + unit-tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.12'] + + steps: + - uses: actions/checkout@v4 + with: + # Fetch full history for proper version detection + fetch-depth: 0 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Cache pip dependencies + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ubuntu-latest-pip-3.12-${{ hashFiles('python/requirements*.txt') }} + restore-keys: | + ubuntu-latest-pip-3.12- + ubuntu-latest-pip- + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip setuptools wheel + cd python + # Install CPU-only versions for CI + pip install numpy scipy matplotlib pillow jsonschema mrcfile + pip install -r requirements-dev.txt + + - name: Check Python environment + run: | + cd python + python -c " + import sys + print(f'Python version: {sys.version}') + + # Test core imports (excluding CUDA) + packages = ['numpy', 'scipy', 'mrcfile', 'jsonschema', 'matplotlib'] + for pkg in packages: + try: + module = __import__(pkg) + version = getattr(module, '__version__', 'unknown') + print(f'✓ {pkg} {version}') + except ImportError as e: + print(f'❌ {pkg}: {e}') + sys.exit(1) + + # Test PIL import + try: + from PIL import Image + print(f'✓ pillow {Image.__version__}') + except ImportError as e: + print(f'❌ pillow: {e}') + sys.exit(1) + + print('Core dependencies verified!') + " + + - name: Run unit tests + run: | + # Run pytest from repo root with proper Python path + # This ensures relative imports work correctly + python -m pytest python/metaData/tests/ -v --tb=short \ + --cov=python/metaData \ + --cov-report=xml --cov-report=term-missing + + # Run inline tests in utils modules (they have embedded test functions) + echo "Running utils inline tests..." + cd python/utils + python emc_str2double.py || echo "emc_str2double tests completed" + python parameter_parser.py || echo "parameter_parser tests completed" + cd ../.. + + # Run masking tests (CPU only for CI) + python -m pytest python/masking/tests/ -v --tb=short -k "not gpu" || echo "Some masking tests failed - this is expected" + + # Run basic CUDA tests (will skip CUDA if not available) + python -m pytest python/cuda_ops/tests/ -v --tb=short -k "not gpu and not cuda" || echo "CUDA tests completed" + env: + PYTHONPATH: ${{ github.workspace }}/python + + - name: Test imports and package structure + run: | + cd python + python -c " + # Test package imports + try: + from metaData import ParameterConverter + print('✓ metaData.ParameterConverter import successful') + except ImportError as e: + print(f'❌ metaData import failed: {e}') + exit(1) + + try: + from masking import emc_pad_zeros_3d + print('✓ masking.emc_pad_zeros_3d import successful') + except ImportError as e: + print(f'❌ masking import failed: {e}') + exit(1) + + try: + from utils import gpu_context, validate_array_dimensions + print('✓ utils imports successful') + except ImportError as e: + print(f'❌ utils import failed: {e}') + exit(1) + + # Test utils inline test functions + try: + from utils.emc_str2double import emc_str2double + from utils.parameter_parser import parse_parameter_file + print('✓ utils function imports successful') + except ImportError as e: + print(f'❌ utils function import failed: {e}') + exit(1) + + # Test parameter conversion functionality + try: + converter = ParameterConverter() + print('✓ ParameterConverter instantiation successful') + except Exception as e: + print(f'❌ ParameterConverter failed: {e}') + exit(1) + + print('All package imports verified!') + " + + - name: Upload coverage to Codecov + if: always() # Always upload coverage from our single test configuration + uses: codecov/codecov-action@v3 + with: + file: python/coverage.xml + flags: unittests + name: codecov-umbrella + fail_ci_if_error: false diff --git a/.gitignore b/.gitignore index 7d947cf8..bae0d0e9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -mexFiles/ mexFiles/compiled +mexFiles/logFile/ !mexFiles/compiled/emC_autoAlign.sh bin/ lib/ @@ -7,7 +7,26 @@ testScripts/EMC_test/logTest/ testScripts/EMC_test/logPerf/ testScripts/EMC_test/fixtures *.orig +*.old +*.bak .vscode +.claude/ +.DS_Store +._.DS_Store +.nfs* + +# Nested repos (managed separately) +autonomous-build/ +autonomous-build-worktrees/ + +*.db +*.png +*.tif* +*.mrc +*.st +*.jpg + +python/metaData/*.star # Things leftover from partial builds testScripts/emClarity_* @@ -17,3 +36,135 @@ testScripts/mccExcludedFiles.log testScripts/readme.txt testScripts/requiredMCRProducts.txt testScripts/unresolvedSymbols.txt + +# Python virtual environment +.venv/ +venv/ +env/ +.env + +# Python cache and compiled files +__pycache__/ +*.py[cod] +*$py.class +*.so + +# Python distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +Pipfile.lock + +# poetry +poetry.lock + +# pdm +.pdm.toml + +# PEP 582 +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +.idea/ + +# Pre-commit +.pre-commit-cache/ + +# ignore log files in any subdirectory named logFile +**/logFile/ +logFile/ +testScripts/*.logy + +CLAUDE.md +.claude +dot-claude diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..f26ac247 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,42 @@ +# Pre-commit configuration for emClarity Python code quality +# See https://pre-commit.com for more information + +repos: + # Ruff - Fast Python linter and formatter + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.12.11 + hooks: + # Run the linter + - id: ruff + args: [--fix] + files: '^python/.*\.py$' + # Run the formatter + - id: ruff-format + files: '^python/.*\.py$' + + # Type checking with Pyright + - repo: https://github.com/RobertCraigie/pyright-python + rev: v1.1.405 + hooks: + - id: pyright + files: '^python/.*\.py$' + + # Security linting with Bandit + - repo: https://github.com/pycqa/bandit + rev: 1.8.6 + hooks: + - id: bandit + args: ["-c", "pyproject.toml"] + files: '^python/.*\.py$' + + # Basic file checks + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v6.0.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + args: ['--maxkb=1000'] + - id: check-merge-conflict + - id: debug-statements diff --git a/@MRCImage/SAVE_IMG.m b/@MRCImage/SAVE_IMG.m index 5b16da71..3e709aa7 100644 --- a/@MRCImage/SAVE_IMG.m +++ b/@MRCImage/SAVE_IMG.m @@ -38,7 +38,7 @@ if nargin > 1 newFilename = varargin{2}; - mRCImage = close(mRCImage); + mRCImage = close(mRCImage); mRCImage.filename = newFilename; % Check to see if the file exists @@ -93,7 +93,12 @@ else flgComplex = false; end - + % if isa(mRCImage.volume, 'half') + % mrcImage.volume = typecast(mRCImage.volume, 'uint16'); + % end + if strcmp(modeStr, 'half') + modeStr = 'uint16'; + end nElements = numel(mRCImage.volume); if mRCImage.header.mode == 0 && getWriteBytesAsSigned(mRCImage) @@ -129,7 +134,16 @@ end else % normal (not complex) data - if (mRCImage.header.minDensity == 0.0 && mRCImage.header.maxDensity ==0 ) + try + do_fix_header = mRCImage.header.minDensity == 0.0 && ... + mRCImage.header.maxDensity == 0.0; + catch + mRCImage.header.minDensity + mRCImage.header.maxDensity + error('Failed to read header min/max density values. Check the file is not corrupt.'); + end + + if (do_fix_header ) if numel(mRCImage.volume) < 768^3 mRCImage.header.minDensity = min(min(min(mRCImage.volume))); mRCImage.header.maxDensity = max(max(max(mRCImage.volume))); diff --git a/@MRCImage/checkFullFile.m b/@MRCImage/checkFullFile.m new file mode 100644 index 00000000..b5011911 --- /dev/null +++ b/@MRCImage/checkFullFile.m @@ -0,0 +1,14 @@ +function is_file_the_expected_size = checkFullFile(mrc_image_obj) + + + bytesPerElement = getModeBytes(mrc_image_obj); + dimensions = getDimensions(mrc_image_obj); + + expected_size = dimensions(1) * dimensions(2) * dimensions(3) * bytesPerElement + 1024 + mrc_image_obj.header.nBytesExtended; + + + is_file_the_expected_size = getFileNBytes(mrc_image_obj) == expected_size; + if ~is_file_the_expected_size + fprintf('File is not the expected size: %d instead is %d\n', expected_size, getFileNBytes(mrc_image_obj)); + end +end \ No newline at end of file diff --git a/@MRCImage/getModeString.m b/@MRCImage/getModeString.m index d010e9d1..a13e00ae 100644 --- a/@MRCImage/getModeString.m +++ b/@MRCImage/getModeString.m @@ -35,6 +35,10 @@ modeString = 'int16*2'; % used for complex short ints case 4 modeString = 'float32*2'; % used for complex floating point + case 6 + modeString = 'uint16'; + case 12 + modeString = 'half'; % 16-bit floating point otherwise PEETError('Unsupported MRCImage mode %d!', mRCImage.header.mode); end diff --git a/@MRCImage/getVolume.m b/@MRCImage/getVolume.m index 4db188a3..fc1316a7 100644 --- a/@MRCImage/getVolume.m +++ b/@MRCImage/getVolume.m @@ -163,6 +163,11 @@ else flgComplex = 0; end +% fread doesn't yet recognize "half" +% read as uint16 and then typecast to half later +if strcmp(modeStr, 'half') + modeStr = 'uint16'; +end % Changed counters and index variables to be clear to me BAH 2017-11-22 % l --> nSlice, k --> iSlice @@ -187,6 +192,11 @@ wordLength = (flgComplex*1 + 1) .* [nImageElements]; flgReCast = 1; + +readX = length(iIndex); +readY = length(jIndex); +imgSize = readX*readY; + switch mode % Make a string to tell fread how long each "value is. BAH 2017-11-22 case 0 @@ -199,7 +209,7 @@ case 1 - precisionString = 'int16'; + precisionString = 'int16'; nToRead = sprintf('%d*int16=>int16',wordLength); nToSkip = 2*nPixelsBetween * (flgComplex*1 + 1); % Allocate the output matrix - NOTE: always single precision at end @@ -215,11 +225,15 @@ % 2x as fast to recast the whole array, than to either read in as % single or to recast each slice. case 6 - precisionString = 'uint16'; + precisionString = 'uint16'; nToRead = sprintf('%d*uint16=>uint16',wordLength); nToSkip = 2*nPixelsBetween * (flgComplex*1 + 1); - otherwise + case 12 + precisionString = 'uint16'; + nToRead = sprintf('%d*uint16=>uint16',wordLength); + nToSkip = 2*nPixelsBetween * (flgComplex*1 + 1); + otherwise error('did not recognize mode value %d\n', mode) end @@ -234,9 +248,6 @@ vol = zeros(length(iIndex), length(jIndex), length(kIndex), precisionString); end -readX = length(iIndex); -readY = length(jIndex); -imgSize = readX*readY; nSlice = 1; for iSlice = kIndex @@ -275,11 +286,14 @@ % vol(~topHalf) = vol(~topHalf) - 128; %end + if ( flgReCast ) if ( flgComplex ) vol = complex(single(vol{1}),single(vol{2})); else - vol = single(vol); + if mRCImage.header.mode == 12 + vol = emc_halfcast(vol); + end end else if ( flgComplex ) @@ -294,5 +308,7 @@ mRCImage.fid = []; end + + end diff --git a/@MRCImage/loadVolume.m b/@MRCImage/loadVolume.m index e4cb3e09..753ceb9b 100644 --- a/@MRCImage/loadVolume.m +++ b/@MRCImage/loadVolume.m @@ -31,6 +31,11 @@ nVoxels = mRCImage.header.nX * mRCImage.header.nY * mRCImage.header.nZ; modeStr = getModeString(mRCImage); +% fread doesn't yet recognize "half" +% read as uint16 and then typecast to half later +if strcmp(modeStr, 'half') + modeStr = 'uint16'; +end if strcmp(modeStr, 'int16*2') || strcmp(modeStr, 'float32*2') % handle reading complex volume modeStr = modeStr(1 : end - 2); @@ -62,7 +67,10 @@ mRCImage.flgVolume = 1; - +if mRCImage.header.mode == 12 + % FIXME: should the header mode be changed? + mRCImage.volume = emc_halfcast(mRCImage.volume); +end mRCImage.volume = reshape(mRCImage.volume, ... mRCImage.header.nX, ... mRCImage.header.nY, ... diff --git a/@MRCImage/OPEN_IMG.m b/@MRCImage/open_img.m similarity index 100% rename from @MRCImage/OPEN_IMG.m rename to @MRCImage/open_img.m diff --git a/@MRCImage/private/getModeBytes.m b/@MRCImage/private/getModeBytes.m index e9c670b4..d5282f2c 100644 --- a/@MRCImage/private/getModeBytes.m +++ b/@MRCImage/private/getModeBytes.m @@ -27,15 +27,19 @@ switch mRCImage.header.mode case 0 - nBytes = 1; + nBytes = 1; % byte case 1 - nBytes = 2; + nBytes = 2; % int16 case 2 - nBytes = 4; + nBytes = 4; % single case 3 - nBytes = 4; + nBytes = 4; % complex int16 case 4 - nBytes = 8; + nBytes = 8; % complex single + case 6 + nBytes = 2; % uint16 + case 12 + nBytes = 2; % half (16-bit float) otherwise nBytes = -1; end diff --git a/@MRCImage/private/readHeader.m b/@MRCImage/private/readHeader.m index 9c54f7c9..74658910 100644 --- a/@MRCImage/private/readHeader.m +++ b/@MRCImage/private/readHeader.m @@ -59,6 +59,12 @@ [fname, perm, fileEndianFormat] = fopen(mRCImage.fid); %#ok if strcmp('ieee-be', fileEndianFormat) == 1 mRCImage.endianFormat = 'ieee-le'; +elseif strcmp('ieee-be.l64', fileEndianFormat) == 1 + mRCImage.endianFormat = 'ieee-le.l64'; + elseif strcmp('ieee-le', fileEndianFormat) == 1 + mRCImage.endianFormat = 'ieee-be'; + elseif strcmp('ieee-le.l64', fileEndianFormat) == 1 + mRCImage.endianFormat = 'ieee-be.l64'; else mRCImage.endianFormat = 'ieee-be'; end @@ -104,12 +110,8 @@ mRCImage.header.minDensity = fread(mRCImage.fid, 1, 'float32'); mRCImage.header.maxDensity = fread(mRCImage.fid, 1, 'float32'); mRCImage.header.meanDensity = fread(mRCImage.fid, 1, 'float32'); -mRCImage.header.spaceGroup = fread(mRCImage.fid, 1, 'int16'); -mRCImage.header.nSymmetryBytes = fread(mRCImage.fid, 1, 'int16'); -if debug - fprintf(debugFD, 'nSymmetry bytes %d\n', mRCImage.header.nSymmetryBytes); -end - +mRCImage.header.spaceGroup = fread(mRCImage.fid, 1, 'int32'); +% nBytesExtended is called nsymbt in the 2014 MRC Standard mRCImage.header.nBytesExtended = fread(mRCImage.fid, 1, 'int32'); if debug fprintf(debugFD, 'nBytesExtended %d\n', mRCImage.header.nBytesExtended); @@ -117,7 +119,7 @@ % MRC EXTRA section mRCImage.header.creatorID = fread(mRCImage.fid, 1, 'int16'); -junk = fread(mRCImage.fid, 30, 'uchar'); %#ok +mRCImage.header.extraInfo1 = fread(mRCImage.fid, 30, 'uchar'); mRCImage.header.nBytesPerSection = fread(mRCImage.fid, 1, 'int16'); if debug fprintf(debugFD, 'nBytesPerSection %d\n', mRCImage.header.nBytesPerSection); @@ -126,7 +128,7 @@ if debug fprintf(debugFD, 'serialEMType %d\n', mRCImage.header.serialEMType); end -junk = fread(mRCImage.fid, 20, 'uchar'); %#ok +mRCImage.header.extraInfo2 = fread(mRCImage.fid, 20, 'uchar'); % IMOD stamp / flags for deciding whether to read / write mode 0 files % as signed or unsigned bytes (added in PEET 1.8.0) diff --git a/@MRCImage/private/setVolumeAndHeaderFromVolume.m b/@MRCImage/private/setVolumeAndHeaderFromVolume.m index 6ba7ae79..5c5279e7 100644 --- a/@MRCImage/private/setVolumeAndHeaderFromVolume.m +++ b/@MRCImage/private/setVolumeAndHeaderFromVolume.m @@ -19,38 +19,45 @@ if isa(volume, 'uint8') mRCImage.header.mode = 0; - mRCImage.volume = volume; -end -if isa(volume, 'int16') +elseif isa(volume, 'int16') if isreal(volume) mRCImage.header.mode = 1; else mRCImage.header.mode = 3; end - mRCImage.volume = volume; -end -if isa(volume, 'single') +elseif isa(volume, 'single') if isreal(volume) mRCImage.header.mode = 2; else mRCImage.header.mode = 4; end - mRCImage.volume = volume; -end -if isa(volume, 'double') +elseif isa(volume, 'double') if ~isreal(volume) - PEETError('Double precision complex images are not supported!'); + PEETError('Double precision complex volumes are not supported!'); end mRCImage.header.mode = 2; - mRCImage.volume = single(volume); +elseif isa(volume, 'uint16') + if isreal(volume) + mRCImage.header.mode = 6; + else + PEETError('Complex uint16 volumes are not supported!'); + end +elseif isa(volume, 'half') + if isreal(volume) + mRCImage.header.mode = 12; + else + PEETError('Complex half-precision volumes are not supported!'); + end +else + PEETError('Volume must be uint8, int16, unit16, half, single, or double!'); end -% Added BAH - setting mRCImage.volume is just a pointer UNTIL we work with it. -% The volume will be duplicated in memory later, so clear "volume" in order -% to avoid wasting large chunks of memory. Change subsequent references from -% "volume" to "mRCImage.volume" -clear volume - + +if isa(volume, 'double') + mRCImage.volume = single(volume); % Store float*64 as float*32 +else + mRCImage.volume = volume; +end % TODO: is this the correct/best way to set the values mRCImage.header.nX = size(mRCImage.volume, 1); mRCImage.header.nY = size(mRCImage.volume, 2); @@ -79,8 +86,10 @@ mRCImage.header.imodFlags = writeBytesAsSigned; mRCImage.header.creatorID = int16(0); +mRCImage.header.extraInfo1 = char(zeros(1, 30, 'uint8')); mRCImage.header.nBytesPerSection = int16(0); mRCImage.header.serialEMType = int16(0); +mRCImage.header.extraInfo2 = char(zeros(1, 20, 'uint8')); mRCImage.header.idtype = 0; mRCImage.header.lens = 0; mRCImage.header.ndl = 0; diff --git a/@MRCImage/private/writeHeader.m b/@MRCImage/private/writeHeader.m index 2108769b..35a8149e 100644 --- a/@MRCImage/private/writeHeader.m +++ b/@MRCImage/private/writeHeader.m @@ -51,7 +51,7 @@ status = fseek(mRCImage.fid, 0, 'bof'); if status disp('Could not move the file pointer to the begining '); - PEETError('Could not seek to beginning of file id %d!', mRCImage.fid); + PEETError('Could not seek to beginning of file id %d', mRCImage.fid); end % Write out the dimensions of the data @@ -78,20 +78,19 @@ writeAndCheck(mRCImage.fid, mRCImage.header.maxDensity, 'float32'); writeAndCheck(mRCImage.fid, mRCImage.header.meanDensity, 'float32'); -writeAndCheck(mRCImage.fid, mRCImage.header.spaceGroup, 'int16'); -writeAndCheck(mRCImage.fid, mRCImage.header.nSymmetryBytes, 'int16'); +writeAndCheck(mRCImage.fid, mRCImage.header.spaceGroup, 'int32'); writeAndCheck(mRCImage.fid, mRCImage.header.nBytesExtended, 'int32'); % MRC EXTRA section writeAndCheck(mRCImage.fid, mRCImage.header.creatorID, 'int16'); -writeAndCheck(mRCImage.fid, char(zeros(1, 30)), 'uchar'); +writeAndCheck(mRCImage.fid, mRCImage.header.extraInfo1, 'uchar'); writeAndCheck(mRCImage.fid, mRCImage.header.nBytesPerSection, 'int16'); writeAndCheck(mRCImage.fid, mRCImage.header.serialEMType, 'int16'); -writeAndCheck(mRCImage.fid, char(zeros(1, 20)), 'uchar'); +writeAndCheck(mRCImage.fid, mRCImage.header.extraInfo2, 'uchar'); mRCImage.header.imodStamp = defaultIMODStamp(); writeAndCheck(mRCImage.fid, mRCImage.header.imodStamp, 'int32'); -if getWriteBytesAsSigned(mRCImage); +if getWriteBytesAsSigned(mRCImage) mRCImage.header.imodFlags = ... int32(bitor(uint32(mRCImage.header.imodFlags), 1)); end @@ -126,12 +125,12 @@ end % If there's room, add a label indicating writing by PEET -%if mRCImage.header.nLabels < 10 -% msg = ['Written by PEET / MatTomo ' datestr(now)]; -% writeAndCheck(mRCImage.fid, msg, 'uchar'); -% writeAndCheck(mRCImage.fid, char(blanks(80 - length(msg))), 'uchar'); -% mRCImage.header.nLabels = mRCImage.header.nLabels + 1; -%end +if mRCImage.header.nLabels < 10 +msg = ['Written by PEET / MatTomo ' datestr(now)]; +writeAndCheck(mRCImage.fid, msg, 'uchar'); +writeAndCheck(mRCImage.fid, char(blanks(80 - length(msg))), 'uchar'); +mRCImage.header.nLabels = mRCImage.header.nLabels + 1; +end % Use blank messages for the remainder for iJunk = mRCImage.header.nLabels+1:10 @@ -146,7 +145,11 @@ % Simple error checking write function writeAndCheck(fid, matrix, precision) nElements = numel(matrix); - count = fwrite(fid, matrix, precision); + if strcmp(precision, 'half') + count = fwrite(fid, matrix, 'uint16'); + else + count = fwrite(fid, matrix, precision); + end if count ~= nElements error('Matrix contains %d elements, but only wrote %d',nElements, count); end diff --git a/@MRCImage/save.m b/@MRCImage/save.m index 7ed2d0d1..d3bde741 100644 --- a/@MRCImage/save.m +++ b/@MRCImage/save.m @@ -47,7 +47,16 @@ % Write out the volume if it is not already on the disk if mRCImage.flgVolume - modeStr = getModeString(mRCImage); + modeStr = getModeString(mRCImage); + if strcmp(modeStr, 'half') + %PEETError('Sorry, writing half-precision files is not supported!') + + % fwrite doesn't yet recognize "half" + % typecast to uint16 before writing, then typecast back later + modeStr = 'uint16'; + mRCImage.volume = typecast(mRCImage.volume, 'uint16'); + + end if strcmp(modeStr, 'int16*2') || strcmp(modeStr, 'float32*2') modeStr = modeStr(1 : end - 2); flgComplex = true; @@ -91,10 +100,18 @@ else % normal (not complex) data count = fwrite(mRCImage.fid, mRCImage.volume, modeStr); if count ~= nElements + % if mRCImage.header.mode == 12 + % mRCImage.volume = typecast(mRCImage.volume, 'half'); + % end fprintf('Matrix contains %d but only wrote %d elements\n', ... nElements, count); PEETError('Failed writing matrix!'); end end end +if mRCImage.header.mode == 12 + % FIXME: should the header mode be changed? + mRCImage.volume = emc_halfcast(mRCImage.volume); +end + close(mRCImage); diff --git a/@MRCImage/showHeader.m b/@MRCImage/showHeader.m index 0262e1d4..fe49f21e 100644 --- a/@MRCImage/showHeader.m +++ b/@MRCImage/showHeader.m @@ -58,10 +58,15 @@ function showHeader(mRCImage) fprintf('rms density:\t\t\t%d\n', mRCImage.header.densityRMS); fprintf('space group:\t\t\t%d\n', mRCImage.header.spaceGroup); -fprintf('# symmetry bytes:\t\t%d\n', mRCImage.header.nSymmetryBytes); - fprintf('# extended header bytes:\t%d\n', mRCImage.header.nBytesExtended); fprintf('creator ID:\t\t\t%d\n', mRCImage.header.creatorID); + +if strcmp(mRCImage.header.extraInfo1, char(zeros(1, 30, 'uint8'))) + fprintf('extended header info1:'); + fprintf('%x ', mRCImage.header.extraInfo1); + fprintf('\n'); +end + fprintf('Extended header bytes/section:\t%d\n', ... mRCImage.header.nBytesPerSection); fprintf('Serial EM data type:\t\t%d\n', mRCImage.header.serialEMType); @@ -70,6 +75,12 @@ function showHeader(mRCImage) fprintf('IMOD flags: \t\t\t%d\n', mRCImage.header.imodFlags); end +if strcmp(mRCImage.header.extraInfo2, char(zeros(1, 20, 'uint8'))) + fprintf('extended header info2:'); + fprintf('%x ', mRCImage.header.extraInfo2); + fprintf('\n'); +end + fprintf('X origin:\t\t\t%d\n', mRCImage.header.xOrigin); fprintf('Y origin:\t\t\t%d\n', mRCImage.header.yOrigin); fprintf('Z origin:\t\t\t%d\n', mRCImage.header.zOrigin); diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 00000000..f26326f1 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +dot-claude/CLAUDE.md \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 00000000..9821d4e8 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +recursive-include python *.cu *.cuh +recursive-include python *.npy +recursive-include python/ctf/tests/fixtures *.json diff --git a/PYTHON_STYLE_GUIDE.md b/PYTHON_STYLE_GUIDE.md new file mode 100644 index 00000000..febac0bb --- /dev/null +++ b/PYTHON_STYLE_GUIDE.md @@ -0,0 +1,317 @@ +# Python Code Style Guide for emClarity + +This document outlines the coding standards and linting rules for the emClarity Python codebase to maintain consistency and prevent CI failures. + +## Overview + +The emClarity project uses automated linting tools to enforce code quality: +- **Black** for code formatting +- **isort** for import sorting +- **flake8** for style and syntax checking +- **mypy** for type checking + +## Pre-Commit Setup + +### Install Development Dependencies +```bash +pip install black isort flake8 mypy autopep8 +``` + +### Pre-Commit Hook (Recommended) +Create `.git/hooks/pre-commit`: +```bash +#!/bin/bash +cd python/ +echo "Running code quality checks..." + +# Format code +black . +isort . + +# Check for issues +black --check . || exit 1 +isort --check-only . || exit 1 +flake8 . --max-line-length=100 --ignore=E203,W503 || exit 1 + +echo "✅ All checks passed!" +``` + +## Code Style Rules + +### 1. Import Organization (isort) + +**✅ Correct:** +```python +# Standard library imports +import os +import sys +from pathlib import Path +from typing import Any, Dict, List, Optional + +# Third-party imports +import numpy as np +import pandas as pd + +# Local imports +from .utils import helper_function +from .models import DataModel +``` + +**❌ Incorrect:** +```python +import numpy as np +import os +from .utils import helper_function +import pandas as pd +``` + +### 2. Code Formatting (Black) + +**✅ Correct:** +```python +def long_function_name( + parameter_one: str, + parameter_two: int, + parameter_three: Optional[bool] = None, +) -> Dict[str, Any]: + """Function with properly formatted parameters.""" + result = { + "key1": parameter_one, + "key2": parameter_two, + "key3": parameter_three, + } + return result +``` + +**❌ Incorrect:** +```python +def long_function_name(parameter_one: str, parameter_two: int, parameter_three: Optional[bool] = None) -> Dict[str, Any]: + result = {"key1": parameter_one, + "key2": parameter_two, "key3": parameter_three} + return result +``` + +### 3. String Formatting + +**✅ Correct:** +```python +# Simple f-strings +name = "World" +message = f"Hello, {name}!" + +# Complex expressions - avoid nested f-strings +text = widget.get("text", "") +text_part = f': "{text}"' if text else "" +summary = f"- {widget_type}{text_part}" +``` + +**❌ Incorrect:** +```python +# Nested f-strings with backslashes (syntax error) +summary = f"- {widget_type}{f': \"{text}\"' if text else ''}" + +# Backslashes in f-strings +path = f"C:\\Users\\{username}\\Documents" # Use raw strings or Path +``` + +### 4. Line Length and Spacing + +**✅ Correct:** +```python +# Max 100 characters per line +very_long_variable_name = some_function_with_long_name( + parameter_one, parameter_two, parameter_three +) + +# Proper spacing +class MyClass: + """Class docstring.""" + + def __init__(self): + """Initialize the class.""" + self.value = 42 + + def method(self) -> int: + """Method with proper spacing.""" + return self.value +``` + +**❌ Incorrect:** +```python +# Line too long +very_long_variable_name = some_function_with_long_name(parameter_one, parameter_two, parameter_three, parameter_four) + +# Improper spacing +class MyClass: + def __init__(self): + self.value = 42 + def method(self) -> int: + return self.value +``` + +### 5. Error Handling + +**✅ Correct:** +```python +try: + result = risky_operation() +except ValueError as e: + logger.error(f"Value error occurred: {e}") + raise +except Exception as e: + logger.error(f"Unexpected error: {e}") + raise +``` + +**❌ Incorrect:** +```python +try: + result = risky_operation() +except: # Bare except clause + pass +``` + +### 6. Type Hints + +**✅ Correct:** +```python +from typing import Any, Dict, List, Optional, Union + +def process_data( + items: List[str], + config: Dict[str, Any], + timeout: Optional[int] = None +) -> Union[str, None]: + """Process data with proper type hints.""" + if not items: + return None + return "processed" +``` + +**❌ Incorrect:** +```python +def process_data(items, config, timeout=None): # No type hints + if not items: + return None + return "processed" +``` + +### 7. Unused Imports and Variables + +**✅ Correct:** +```python +import logging +from typing import Dict + +logger = logging.getLogger(__name__) + +def process_config(config: Dict[str, str]) -> bool: + """All imports and variables are used.""" + logger.info("Processing configuration") + return len(config) > 0 +``` + +**❌ Incorrect:** +```python +import logging +import os # Unused import +from typing import Dict, List # List is unused + +logger = logging.getLogger(__name__) + +def process_config(config: Dict[str, str]) -> bool: + unused_var = "not used" # Unused variable + logger.info("Processing configuration") + return len(config) > 0 +``` + +## Common Fixes + +### Auto-Fix Commands +Run these before committing: +```bash +cd python/ +isort . # Fix import sorting +black . # Fix formatting +autopep8 --in-place --recursive --max-line-length=100 . # Additional fixes +``` + +### Manual Fixes Needed +- Remove unused imports and variables +- Fix undefined variable references +- Resolve type checking errors +- Add missing docstrings + +## IDE Configuration + +### VS Code Settings (.vscode/settings.json) +```json +{ + "python.formatting.provider": "black", + "python.sortImports.provider": "isort", + "python.linting.enabled": true, + "python.linting.flake8Enabled": true, + "python.linting.mypyEnabled": true, + "python.linting.flake8Args": [ + "--max-line-length=100", + "--ignore=E203,W503" + ], + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true + } +} +``` + +### PyCharm Settings +1. File → Settings → Tools → External Tools +2. Add Black, isort, and flake8 as external tools +3. Enable "Format on Save" in Code Style settings +4. Configure import optimization in Code Style → Python → Imports + +## CI Integration + +The CI pipeline runs these checks on every push: +```yaml +- name: Code Quality Checks + run: | + cd python/ + black --check --diff . + isort --check-only --diff . + flake8 . --max-line-length=100 --ignore=E203,W503 + mypy . --ignore-missing-imports +``` + +## Quick Reference + +### Before Committing (Every Time) +```bash +cd python/ +black . && isort . && flake8 . --max-line-length=100 +``` + +### Common flake8 Error Codes +- `E501`: Line too long (>100 characters) +- `F401`: Imported but unused +- `F841`: Local variable assigned but never used +- `E302`: Expected 2 blank lines +- `W291`: Trailing whitespace +- `E722`: Do not use bare except + +### Quick Fixes +- **Trailing whitespace**: Remove extra spaces at line ends +- **Import order**: Let isort handle it automatically +- **Line length**: Break long lines at logical points +- **Unused imports**: Remove or move to comments if needed for typing + +## Resources + +- [Black Documentation](https://black.readthedocs.io/) +- [isort Documentation](https://pycqa.github.io/isort/) +- [flake8 Documentation](https://flake8.pycqa.org/) +- [mypy Documentation](https://mypy.readthedocs.io/) +- [PEP 8 Style Guide](https://pep8.org/) + +--- + +Following these guidelines will ensure your code passes CI checks and maintains consistency across the emClarity codebase. diff --git a/README.md b/README.md index 754103d1..c193f852 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,12 @@ # emClarity + +[![Unit Tests](https://github.com/StochasticAnalytics/emClarity/actions/workflows/unit-tests.yml/badge.svg?branch=main)](https://github.com/StochasticAnalytics/emClarity/actions/workflows/unit-tests.yml) +[![Code Style](https://github.com/StochasticAnalytics/emClarity/actions/workflows/code-style.yml/badge.svg?branch=main)](https://github.com/StochasticAnalytics/emClarity/actions/workflows/code-style.yml) +[![Type Checking](https://github.com/StochasticAnalytics/emClarity/actions/workflows/type-checking.yml/badge.svg?branch=main)](https://github.com/StochasticAnalytics/emClarity/actions/workflows/type-checking.yml) +[![Security Scan](https://github.com/StochasticAnalytics/emClarity/actions/workflows/security-scan.yml/badge.svg?branch=main)](https://github.com/StochasticAnalytics/emClarity/actions/workflows/security-scan.yml) +[![GPU Tests](https://github.com/StochasticAnalytics/emClarity/actions/workflows/gpu-tests.yml/badge.svg?branch=main)](https://github.com/StochasticAnalytics/emClarity/actions/workflows/gpu-tests.yml) + Cleaning out the repo to make room for the source code. Binaries will be distributed through an external web site, linked in the wiki. Good things are happening! -Ben diff --git a/alignment/BH_alignClassRotAvg3d.m b/alignment/BH_alignClassRotAvg3d.m deleted file mode 100755 index 7be7a7d0..00000000 --- a/alignment/BH_alignClassRotAvg3d.m +++ /dev/null @@ -1,670 +0,0 @@ -function [ ] = BH_alignClassRotAvg3d(PARAMETER_FILE, CYCLE) - - -%Extract and align class averages and references from 4D montages derived. -% -% Input variables: -% -% IMAGE = 4d volume, or a string specifing a volume to read in. -% -% CLASSES = Align subset of class averages. [1, 2, 5, 6] -% -% CLASS_NAME = a number (e.g. 64) that refers to the class/montage to use. -% -% REFERENCES a list same as CLASSES with the class id, and symmetry to apply. -% -% REF_NAME = a number (e.g. 8) that refers to the class/montage to draw the -% references from. -% -% REAL_MASK = {maskType, maskSize, maskRadius, maskCenter} -% -% BANDPASS = [HIGH_THRESH, HIGH_CUT, LOW_CUT, PIXEL_SIZE] applied to the -% particle prior to interpolation. -% -% ANGLE_SEARCH = [a1 a2 a3 a4 ] the angular search is a grid searched -% designed to exhaustively sample a unit sphere over the range -% you specify. Out of plane +/- a1 in steps of size a2, in -% plane +/- a3 in steps of a4. These together define a set of -% "latitudes" if you will, and the in plane sampling at each -% point sampled on that latitude, while the longitudinal -% sampling is calculated to be evenly sampled at the same rate -% as the latitude. The smaller the out of plane step size, the -% more longitudinal sampling points, and also the larger the -% absolute out of plane angle, the greater number of steps it -% takes to make a full revolution. -% -% PEAK = [x y z] = *RADIUS* of peak search -% PEAK_MASS = [x y z] = *RADIUS* for center of mass search around max peak -% Set to a zero to ignore either option. -% -% REFERENCES = a cell with list of images to use as references. -% -% GEOMETRY = A structure with tomogram names as the field names, and geometry -% information in a 26 column array. -% Additionally, a field called 'source_path' has a value with the -% absolute path to the location of the tomograms. -% -% The input is a string 'Geometry_templatematching.mat' for -% example, and it is expected that the structure is saved as the -% variable named geometry. -% -% -% OUTPUT_PREFIX = String to prepend to output volumes. -% -% -% Output variables: -% -% None = files are written to disk in the current directory. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% Goals & Limitations: -% -% Align class averages to a reference. Here it is implicitly assumed that the -% references and the volumes they will be aligned against are the same -% dimension. I am not going to remove all of the information and steps related -% to binning, as these will be needed when writing the function to handle -% alignment of raw subTomos, but later I will clean this up. -% -% Assumed to run on GPU. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% TODO - -% - Update geometry to record % sampling -% - deal with assumption that 256 256 256 is sufficient for volumes. -% - Add error check for mask size/ v radius -% - Change angular searches to allow for a translational only search. -% - Work through angular sampling to be sure it is doing what you think it -% is. -% - Read sampling from metadata to deal with bandpass. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -if (nargin ~= 2) - error('args = PARAMETER_FILE, CYCLE') -end - -PRECISION = 'single'; - -startTime = clock; -CYCLE = EMC_str2double(CYCLE); - - -cycleNumber = sprintf('cycle%0.3u', CYCLE); - -pBH = BH_parseParameterFile(PARAMETER_FILE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - -maxGoldStandard = subTomoMeta.('maxGoldStandard'); - -bFactor = pBH.('Fsc_bfactor'); - -try - flgRotAvgRef = pBH.('flgRotAvgRef'); -catch - flgRotAvgRef = 0; -end -samplingRate = pBH.('Ali_samplingRate'); -pixelSize = pBH.('PIXEL_SIZE').*samplingRate.*10^10; -if pBH.('SuperResolution') - pixelSize = pixelSize * 2; -end -try - scaleCalcSize = pBH.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end - -angleSearch = pBH.('Cls_angleSearch'); -refName = pBH.('Cls_className'); %pBH.('Ref_className'); -className = pBH.('Cls_className'); -peakSearch = floor(pBH.('particleRadius')./pixelSize) -peakCOM = [1,1,1].*peakCOM; -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); - - - -flgAngleShift{1}= pBH.('ref_AngleShift_odd'); -flgTransShift{1}= pBH.('ref_TransShift_odd'); -flgRefRef{1} = pBH.('ref_Ref_odd'); -refVectorFull{1}= pBH.('Ref_references_odd'); -classVector{1} = pBH.('Cls_classes_odd')(1,:); -features{1} = pBH.('Pca_coeffs_odd'); -vol_geometry{1} = subTomoMeta.(cycleNumber).('ClusterResults').( ... - sprintf('%s_%d_%d_nClass_%d_ODD', ... - outputPrefix,features{1}(1,1), ... - features{1}(1,end), className)); - -flgAngleShift{2}= pBH.('ref_AngleShift_eve'); -flgTransShift{2}= pBH.('ref_TransShift_eve'); -flgRefRef{2} = pBH.('ref_Ref_eve'); -refVectorFull{2}= pBH.('Ref_references_eve'); -classVector{2} = pBH.('Cls_classes_eve')(1,:); -features{2} = pBH.('Pca_coeffs_eve'); -vol_geometry{2} = subTomoMeta.(cycleNumber).('ClusterResults').( ... - sprintf('%s_%d_%d_nClass_%d_EVE', ... - outputPrefix,features{2}(1,1), ... - features{2}(1,end), className)); -% Merge alignments that could come from using different feature vectors in the -% classification, back into one metadata. -vol_geometry = BH_mergeClassGeometry(vol_geometry{1},vol_geometry{2}); - - - - - - -refVector = cell(2,1); -refGroup = cell(2,1); -refSym = cell(2,1); - -for iGold = 1:2 - % Sort low to high, because order is rearranged as such unstack - refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; - % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) - % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) - % axial symmetry to apply, negative value indicates creating a mirrored ref - % accros the corresponding axis - refSym{iGold} = refVectorFull{iGold}(2,:) -end - - -pathList= subTomoMeta.mapPath; -extList = subTomoMeta.mapExt; -masterTM = subTomoMeta; clear subTomoMeta - -% make sure the number of references match the unique groups in the classVector -% and also that the class/group pairs match the class/ref pairs. -nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})] - -uniqueSym = cell(2,1); -for iGold = 1:2 - [~,uniqueGroup,~] = unique(refGroup{iGold}); - uniqueSym{iGold} = refSym{iGold}(uniqueGroup) -end - - - - - - -[ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Cls', samplingRate); -[ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) -padREF = [0,0,0;0,0,0]; -if any(peakSearch > maskRadius) - fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') - peakSearch( (peakSearch > maskRadius) ) = ... - maskRadius( (peakSearch > maskRadius) ); -end - - -% Read in the references. -refIMG = cell(2,1); -imgCounts = cell(2,1); - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet); - weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', refName, halfSet); - - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - [ refIMG{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - - [ refWDG{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... - sizeCalc); - - - sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}; - sizeREF = sizeREF(2:2:6)' - - -end - -[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, maxGoldStandard); - - -% % % [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc, padREF ] = ... -% % % BH_multi_validArea( maskRadius, sizeREF ) -% optimize the fft for the given size. Padding to the next power of 2 is usually -% slower given the dimensionality of the volume data. -fftPlanner = rand(sizeCalc); -fftw('planner', 'exhaustive'); -fftn(fftPlanner); -clear fftPlanner - -% Make a mask, and apply to the average motif && save a masked, binned copy of -% the average for inspection. - - -[ volMask ] = gpuArray(BH_mask3d(maskType, sizeMask, maskRadius, maskCenter)); -volBinary = (volMask > 0.01); - -[ peakMask] = gpuArray(BH_mask3d(maskType, sizeMask, peakSearch, maskCenter)); -peakBinary = (peakMask > 0.01); - -bandpassFilt = cell(nReferences(1),1); - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'cpu', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); -for iRef = 1:nReferences(1) - - - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); - - % The class averages have roughly the same SNR as the references so apply any - % bFactor to them as well. - [ ~, bandpassFilt{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, bFactor,1 ); - bandpassFilt{iRef} = gpuArray(bandpassFilt{iRef}); -end - -bestAnglesResults = cell(2,1); - - -try - EMC_parpool(2) -catch - delete(gcp('nocreate')) - EMC_parpool(2) -end - -parfor iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - - ref_FT = zeros([sizeWindow,nReferences(iGold)], PRECISION, 'gpuArray'); - refRotAvg_FT = zeros([sizeWindow,nReferences(iGold)], PRECISION, 'gpuArray'); - - - refRotAvg = refIMG{iGold}; - refTrans = cell(length(refIMG{iGold}).*2); - nRefOut = 1; - for iRef = 1:nReferences(iGold) - - - if (flgRotAvgRef) - refRotAvg{iRef} = gather( ... - BH_axialSymmetry(gpuArray(refIMG{iGold}{iRef}),120, 0, ... - 'GPU', [0,0,0])); - - % For later stages where the angles are very small, still scan only - % azimuthal and out of plane, but don't rotationally average the in-plane - % angles. - else - refRotAvg{iRef} = refIMG{iGold}{iRef}; - - - end - - padTransTrim = padWindow + padREF; - refTransTrim = refIMG{iGold}{iRef}(padTransTrim(1,1)+1 : end - padTransTrim(2,1), ... - padTransTrim(1,2)+1 : end - padTransTrim(2,2), ... - padTransTrim(1,3)+1 : end - padTransTrim(2,3) ); - - refTrans{nRefOut} = real(ifftn( BH_bandLimitCenterNormalize(refTransTrim.*volMask, ... - bandpassFilt{iRef}, volBinary,padCalc,'double'))); - - refTrans{nRefOut} = gather(refTrans{nRefOut}(padCalc(1,1) + 1: end - padCalc(2,1),... - padCalc(1,2) + 1: end - padCalc(2,2),... - padCalc(1,3) + 1: end - padCalc(2,3)) .*volMask); - - refTrans{nRefOut+1} = real(ifftn( BH_bandLimitCenterNormalize(refTransTrim.*peakMask, ... - bandpassFilt{iRef}, peakBinary,padCalc,'double'))); - - refTrans{nRefOut+1} = gather(refTrans{nRefOut+1}(padCalc(1,1) + 1: end - padCalc(2,1),... - padCalc(1,2) + 1: end - padCalc(2,2),... - padCalc(1,3) + 1: end - padCalc(2,3)) .*peakMask); - - nRefOut = nRefOut +2; - - -iGold -iRef -size(refIMG{iGold}{iRef}) -size(ref_FT) - %winRefDiff = padREF - padWindow; - ref_FT(:,:,:,iRef) = refIMG{iGold}{iRef}( ... - padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - refRotAvg_FT(:,:,:,iRef) = refRotAvg{iRef}( ... - padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - - - - - end - refMontage = BH_montage4d(refTrans,''); - SAVE_IMG(MRCImage(refMontage),sprintf('%s_class_refFiltered_%s.mrc',cycleNumber,halfSet)); - % Read in the class averages. -% refMontage{iGold} = BH_montage4d(refTrans,''); - imgClassNAME = sprintf('class_%d_Locations_%s_%s_NoWgt', className, 'Cls', halfSet); - wdgClassNAME = sprintf('class_%d_Locations_%s_%s_Wgt', className, 'Cls', halfSet); - - [ classIMG ] = BH_unStackMontage4d(classVector{iGold}, ... - masterTM.(cycleNumber).(imgClassNAME){1}, ... - masterTM.(cycleNumber).(imgClassNAME){2},sizeWindow); - - [ classWDG ] = BH_unStackMontage4d(classVector{iGold}, ... - masterTM.(cycleNumber).(imgClassNAME){1}, ... - masterTM.(cycleNumber).(imgClassNAME){2},sizeWindow); - - - nClassesPossible = length(classVector{iGold}) - nClasses = length(classIMG) - - %%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't - %%%%%%%%%%%%%%%%%%%%% search at all in that dimension. - [ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch) - - fprintf('%d ',inPlaneSearch); - fprintf('\n'); - - % Store the cross correlation score, peak location, and wedge weight - bestAnglesTotal = zeros(nClassesPossible,10); - nCount = 1; - - - for iClass = classVector{iGold} - - tic; - - - % Load the class into gpu, center and normalize - iClassImg = gpuArray(classIMG{iClass}( ... - padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) )); - - - iClassWdg = ifftshift(gpuArray(classWDG{iClass}( ... - padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ))); - - - - % [ iClassImg ] = BH_padZeros3d(iClassImg, padPre, padPost, 'GPU', 'single'); - %[ iClassImg ] = BH_bandLimitCenterNormalize(iClassImg, bandpassFilt, volMask); - % iClassImg = real(ifftn(iClassImg)); - % First loop over all out of plane, no in plane, with rotationally averaged - % reference. - cccStorage1 = []; - cccStorage2 = []; - cccStorage3 = []; - cccStorage4 = []; - % Out of plane search - if any(angleStep(2:4)) - flgSearchDepth = 3; - elseif any(angleStep(5)) - % in plane - flgSearchDepth = 2; - peakListTop10(angleStep(5).*nReferences(iGold),6) = gpuArray(0); - - nPeak = 1; - for iRef = 1 - for iPsi = inPlaneSearch - peakListTop10(nPeak,1) = iRef; - nPeak = nPeak + 1; - end - end - else - error('specify at least an in plane search') - end - - - if (flgSearchDepth == 3) - % Note the rotationally averaged ref is passed as main ref - % - [ cccStorage1 ] = BH_multi_angularSearch( angleStep, 0, 0, ... - iClassImg, iClassWdg, ... - refRotAvg_FT, NaN, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padWindow,... - peakMask, peakCOM, iClass, ... - uniqueSym{iGold}); - - - cccStorage1(1:10,:) - - - % Second loop over top 10 peaks now using the non-rotationally averaged - % reference and including out of plane angles. - - % peakList is # rows = top peaks - % reference, phi, theta - % put zero peak at top of list - zeroPeak = sortrows(cccStorage1,[3, 4, 5]); - zeroPeak = [zeroPeak(1,:) ; cccStorage1 ]; - % if zero peak was already there, remove it so no duplicate - zeroPeak = unique(zeroPeak, 'stable', 'rows'); - peakListTop10 = [zeroPeak(1:10,1),zeroPeak(1:10,3:4),zeroPeak(1:10,8:10)] - end - - [ cccStorage2 ] = BH_multi_angularSearch( angleStep, peakListTop10, ... - inPlaneSearch, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG{iGold}, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM,iClass, ... - uniqueSym{iGold}); - - - cccStorage2 = unique(cccStorage2((cccStorage2(:,6) ~= 0),:), 'stable','rows'); - if size(cccStorage2, 1) > 9 - cccStorage2(1:10,:) - else - cccStorage2 - end - % This is to save time assuming that we can get a good estimate of the - % particles shift by taking the average of the higher ranking alignments. In - % testing this was always within ~ half a pixel. If CCC scores are strangely - % low, suspect this as a break point. - - - % Get the top three peaks with unique phi, and theta - % Return [ref,phi,theta,psi,phistep,thetastep,psistep] - % Stable prevents any sorting - - if (flgSearchDepth== 3) - [~,ia,~] = unique(cccStorage2(:,3:4),'stable' ,'rows'); - peakListTop3 = zeros(3,10); - else - [~,ia,~] = unique(cccStorage2(:,3:5),'stable' ,'rows'); - peakListTop3 = zeros(3,10); - end - - if numel(ia) >= 10 - TOP = 10; - else - TOP = numel(ia); - end - - for top3 = 1:TOP - outOfPlaneAngle = cccStorage2(ia(top3),4); - angleIndex = find(angleStep(:,1)==outOfPlaneAngle,1,'first'); - if (flgSearchDepth == 3 ) - % Search around top 3 +/- 0.5 the original out of plane angular increment - peakListTop3(top3,:) = [cccStorage2(ia(top3),1), ... - cccStorage2(ia(top3),3:5),... - angleStep(angleIndex,3)./4,... - angleStep(angleIndex,4)./2,... - angleStep(angleIndex,5)./2, cccStorage2(ia(top3),8:10)]; - else - % Search around top 3 +/- 0.5 the original out of plane angular increment - peakListTop3(top3,:) = [cccStorage2(ia(top3),1), ... - cccStorage2(ia(top3),3:5),... - 0,... - 0,... - angleStep(1,5)./2, cccStorage2(ia(top3),8:10)]; - end - end - peakListTop3 - [ cccStorage3 ] = BH_multi_angularSearch( angleStep, peakListTop3, ... - 0, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG{iGold}, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask,peakCOM,iClass, ... - uniqueSym{iGold}); - - - cccStorage3 = unique(cccStorage3((cccStorage3(:,6) ~= 0),:), 'stable','rows'); - cccStorage3(1:10,:) - - if (flgSearchDepth == 3 ) - % Use previous increments/2 - - % Search around the top peak +/- 0.25 the orginal angular increment - topPeak = [cccStorage3(1,1), ... - cccStorage3(1,3:5), ... - cccStorage3(1,11:13)./2, ... - cccStorage3(1,8:10)] - else - topPeak = [cccStorage3(1,1), ... - cccStorage3(1,3:5), ... - 0, ... - 0,... - angleStep(1,5)./3, cccStorage3(1,8:10)] - end - - - [ cccStorage4 ] = BH_multi_angularSearch( angleStep, topPeak, ... - 0, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG{iGold}, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padWindow,... - peakMask, peakCOM,iClass, ... - uniqueSym{iGold}); - - cccStorage4(1,:) - - - bestAnglesTotal(iClass,:) = gather(cccStorage4(1,1:10)); - - timeClass = toc; - fprintf('finished working on %d/%d classes...%fs\n',nCount,nClassesPossible,timeClass) - nCount = nCount + 1; - - % Save alignment incase of crash, long runs can be resumed. - -%%% save('bestAnglesTotalClass.mat', 'bestAnglesTotal'); - - - - - end % loop over (classes) - - bestAnglesResults{iGold} = gather(bestAnglesTotal); - % Save a text copy of results. - resultsOut = fopen(sprintf('%s_bestAngles_%s.txt', cycleNumber, halfSet),'w'); - fprintf(resultsOut,'%d %d %3.3f %3.3f %3.3f %1.6f %d %4.4f %4.4f %4.4f\n',bestAnglesTotal'); - fclose(resultsOut); -end - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - imgClassNAME = sprintf('class_%d_Locations_%s_%s', className, 'Cls', halfSet); - [ classIMG ] = BH_unStackMontage4d(classVector{iGold}, ... - masterTM.(cycleNumber).(imgClassNAME){1}, ... - masterTM.(cycleNumber).(imgClassNAME){2},sizeWindow); - - [ vol_geometry ] = BH_classAlignmentsApply( vol_geometry, bestAnglesResults{iGold},... - samplingRate,classVector{iGold},iGold,halfSet); - - - - % SAVE_IMG(MRCImage(refMontage{iGold}),sprintf('%s_class_refFiltered_%s.mrc',cycleNumber,halfSet)); - % Max a montage of the applied corrections for viewing. - % Insert blank images so subsets are still corresponding. - classInterp = cell(className,1); - emptyImg = zeros(sizeWindow, 'single'); - for iClass = 1:className - if ismember(iClass, classVector{iGold}) - iClass - - iClassImg = single(gpuArray(classIMG{iClass})); - sizeMask - size(iClassImg) - %figure, imshow3D(gather(iClassImg)) ; pause(2) ; close(gcf) - % Note 'forward' affects the shifts but not the angles, a value of 10 means the - % class was 10 away from the ref - idx = find(bestAnglesResults{iGold}(:,2) == iClass); - iClassImg = BH_resample3d(iClassImg, bestAnglesResults{iGold}(idx, 3:5), ... - bestAnglesResults{iGold}(idx, 8:10), ... - 'Bah', 'GPU', 'inv'); - else - - iClassImg = emptyImg; - end - - classInterp{iClass} = gather(single(iClassImg(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3)) )); - - - end - - - try - [montOUT] = BH_montage4d(classInterp, ''); - imout = sprintf('%s_class%d_%s_aligned.mrc',outputPrefix, className,halfSet); - SAVE_IMG(MRCImage(gather(montOUT)), imout); - catch - fprintf('error in montaging the aligned classes line 603.\n') - end -end - -masterTM.(cycleNumber).('ClassAlignment') = vol_geometry; -subTomoMeta = masterTM; -save(pBH.('subTomoMeta'), 'subTomoMeta'); - -fprintf('Total execution time : %f seconds\n', etime(clock, startTime)); -gpuDevice(1); - -try - delete(gcp('nocreate')) -catch -end -end % end of average3d function - - diff --git a/alignment/BH_alignRaw3d.m b/alignment/BH_alignRaw3d.m deleted file mode 100755 index 87d2e20a..00000000 --- a/alignment/BH_alignRaw3d.m +++ /dev/null @@ -1,1676 +0,0 @@ - function [ ] = BH_alignRaw3d(PARAMETER_FILE, CYCLE, varargin) - -%Extract and align class averages and references from 4D montages derived. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% Goals & Limitations: -% -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% TODO -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -global bh_global_print_shifts_in_particle_basis; -if isempty(bh_global_print_shifts_in_particle_basis) - bh_global_print_shifts_in_particle_basis = true; -end - -global bh_global_zero_lag_score; -if isempty(bh_global_zero_lag_score) - bh_global_zero_lag_score = false -end - -if (nargin ~= 2 && nargin ~= 3) - error('args = PARAMETER_FILE, CYCLE, [1,abs(ccc),2,weighted,3,abs(weighted)]') -else - parentFunc = mfilename; - resumeVars = struct(); -end - -if nargin == 3 - - flgWeightCCC = EMC_str2double(varargin{1}); -else - % default to linear ccc (which is actually weighted by the SNR though) - flgWeightCCC = 0; -end - -% Explicit reference to location of variables in main memory, or on the GPU. As -% in pcaPub, looking ahead to re-write in c++ for cuda, no cells allowed. -cpuVar = struct(); -GPUVar = struct(); - -startTime = clock; -CYCLE = EMC_str2double(CYCLE); -cycle_numerator = ''; -cycle_denominator =''; - flgStartThird = 0; - flgReverseOrder = 0; -if numel(CYCLE) == 3 - cycle_numerator = CYCLE(2); - cycle_denominator = CYCLE(3); - CYCLE = CYCLE(1); - flgStartThird = true; -elseif CYCLE < 0 - % Simple option to process in reverse order so that the load can be run on two - % physically distinct systems at once. - flgReverseOrder = 1; - flgStartThird = 0; - CYCLE = abs(CYCLE); - - -end - - - -pBH = BH_parseParameterFile(PARAMETER_FILE); -cycleNumber = sprintf('cycle%0.3u', CYCLE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); -mapBackIter = subTomoMeta.currentTomoCPR; -reconScaling = 1; -try - nPeaks = pBH.('nPeaks'); -catch - nPeaks = 1; -end - -try - flgCutOutVolumes=pBH.('flgCutOutVolumes') -catch - flgCutOutVolumes=0 -end - -% TODO decide on a "reasonable" padding based on expected shifts. -try - CUTPADDING = subTomoMeta.('CUTPADDING') -catch - CUTPADDING=20 -end - -maxGoldStandard = subTomoMeta.('maxGoldStandard'); - - -nGPUs = pBH.('nGPUs') - - -flgClassify= pBH.('flgClassify'); -try - flgMultiRefAlignment=pBH.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end -try - flgCenterRefCOM = pBH.('flgCenterRefCOM'); -catch - flgCenterRefCOM = 1; -end - -try - flgSymmetrizeSubTomos = pBH.('flgSymmetrizeSubTomos'); -catch - flgSymmetrizeSubTomos = 0; -end -flgRaw_shapeMask = 0;%= pBH.('experimentalOpts')(3) -samplingRate = pBH.('Ali_samplingRate'); - -pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; -if pBH.('SuperResolution') - pixelSize = pixelSize * 2; -end - -flgPrecision = 'single'; %pBH.('flgPrecision'); -angleSearch = pBH.('Raw_angleSearch'); -peakSearch = (pBH.('particleRadius')./pixelSize); -peakCOM = [1,1,1].*3; -className = pBH.('Raw_className'); - -try - loadTomo = pBH.('loadTomo') -catch - loadTomo = 0; -end -try - eraseMaskType = pBH.('Peak_mType'); - eraseMaskRadius = pBH.('Peak_mRadius')./pixelSize; - fprintf('Further restricting peak search to radius %f %f %f\n',... - eraseMaskRadius); - eraseMask = 1; -catch - eraseMask = 0; - fprintf('Using particle radius for peak search\n'); -end - -rotConvention = 'Bah'; -% Check and override the rotational convention to get helical averaging. -% Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = pBH.('doHelical'); -catch - doHelical = 0; -end -if ( doHelical ) - rotConvention = 'Helical' -end - -rotConvention - -bFactor = pBH.('Fsc_bfactor'); -if length(bFactor) > 1 - fprintf('multiple bFactors specified, using the first for alignment.\n'); - bFactor = bFactor(1); -end - -try - scaleCalcSize = pBH.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end -% % % % if (flgClassify || flgMultiRefAlignment) -if (flgClassify) - refName = pBH.('Ref_className'); -else - refName = pBH.('Raw_className'); -end - -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); - - - -classVector{1} = pBH.('Raw_classes_odd')(1,:); -classSymmetry{1}= pBH.('Raw_classes_odd')(2,:); - - -classVector{2} = pBH.('Raw_classes_eve')(1,:); -classSymmetry{2}= pBH.('Raw_classes_eve')(2,:); - - -% % % % if (flgClassify || flgMultiRefAlignment) -if (flgClassify) - geometry = subTomoMeta.(cycleNumber).ClassAlignment; - refVectorFull{1}= [pBH.('Ref_references_odd');1] - refVectorFull{2}= [pBH.('Ref_references_eve');1] -elseif (flgMultiRefAlignment) - geometry = subTomoMeta.(cycleNumber).ClusterRefGeom; - refVectorFull{1}= [pBH.('Raw_classes_odd');classVector{1} ] - refVectorFull{2}= [pBH.('Raw_classes_eve');classVector{2} ] -else - geometry = subTomoMeta.(cycleNumber).Avg_geometry; - refVectorFull{1} = [pBH.('Raw_classes_odd');1]; - refVectorFull{2} = [pBH.('Raw_classes_eve');1]; -end - - -% % % pathList= subTomoMeta.mapPath; -% % % extList = subTomoMeta.mapExt; -masterTM = subTomoMeta; clear subTomoMeta - - - - -refVector = cell(2,1); -refGroup = cell(2,1); -refSym = cell(2,1); - -for iGold = 1:2 - % Sort low to high, because order is rearranged as such unstack - refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; - % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) - % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) - % axial symmetry to apply, negative value indicates creating a mirrored ref - % accros the corresponding axis - refSym{iGold} = refVectorFull{iGold}(2,:) -end - -% make sure the number of references match the unique groups in the classVector -% and also that the class/group pairs match the class/ref pairs. -nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})] - - -nRefOut(1:2) = [length(unique(refGroup{1})) + sum(( refSym{1} < 0 )),... - length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; - - -%%%%%%%%%%%%%%%%%%%%%%% - -% Get the number of tomograms to process. -tomoList = fieldnames(geometry); -nTomograms = length(tomoList); -tiltList = masterTM.tiltGeometry; -ctfGroupList = masterTM.('ctfGroupSize'); - - - -% mask defines area for angular search, peakRADIUS restricts translational - - -[ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Ali', pixelSize) - -[ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) - - -try - flgLimitToOneProcess = pBH.('flgLimitToOneProcess'); -catch - flgLimitToOneProcess = 0; -end - -if ( loadTomo ) - limitToOne = loadTomo; - if (flgLimitToOneProcess) - limitToOne = min(limitToOne, flgLimitToOneProcess); - end -elseif (flgLimitToOneProcess) - limitToOne = flgLimitToOneProcess; -else - limitToOne = pBH.('nCpuCores'); -end - -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); -if ( flgReverseOrder ) - % Flip the order for reverse processing on a second machine. This will also disable saving of - % of the metadata so there aren't conflicts. - for iParProc = 1:nParProcesses - iterList{iParProc} = flip(iterList{iParProc}); - end - -elseif ( flgStartThird ) - - % Shift to start at one third through to process on a third machine. This will also disable saving of - % of the metadata so there aren't conflicts. - for iParProc = 1:nParProcesses - % Note the use of floor is more like ceiling here (rounds away from - % zero) - nParts = ceil(length(iterList{iParProc}) ./ cycle_denominator); - fIDX = 1+(cycle_numerator - 1)*nParts; - lIDX = min(cycle_numerator*nParts,length(iterList{iParProc})); - iterList{iParProc} = iterList{iParProc}(fIDX:lIDX); - end - -end - -if any(peakSearch > maskRadius) - fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') - peakSearch( (peakSearch > maskRadius) ) = ... - maskRadius( (peakSearch > maskRadius) ); -end - -if ( any(classSymmetry{1}~=1) || any(classSymmetry{2}~=1) ) && flgSymmetrizeSubTomos - flgSymmetry = true -else - flgSymmetry = false -end - -% Read in the references. -% Read in the references. -refIMG = cell(2,1); -refWGT = cell(2,1); -refWgtROT = cell(2,1); -imgCounts = cell(2,1); -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) - - - weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', refName, halfSet); - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - - [ refTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - - [ wdgTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... - sizeCalc); - - sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}(2:2:6)'; - - if (flgCenterRefCOM) -% % % % % % % [ comMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); - [ comMask ] = EMC_maskShape(maskType, sizeMask, maskRadius, 'gpu', {'shift', maskCenter}); - end - - % get boxSize - n = 1 ; tIMG = cell(numel(refVector{iGold})); tWDG = cell(numel(refVector{iGold}));tWDG_r = tWDG; - for iP = 1:numel(refTMP) - if ~isempty(refTMP{iP}) - tIMG{n} = refTMP{iP}; refTMP{iP} = []; - if (flgCenterRefCOM) - % Not sure if this is always the best approach, but it may be - % useful in some cases. -% % % % % % % [~,iCOM] = BH_mask3d(gpuArray(tIMG{n}).*comMask,pixelSize,'','',1); - - [~, ~, ~,iCOM] = EMC_maskReference(gpuArray(tIMG{n}).*comMask, pixelSize, {'fsc',true; 'com', true}); - fprintf('centering ref %d on COM %3.3f %3.3f %3.3f \n',n,iCOM); - - tIMG{n} = BH_resample3d(tIMG{n},[0,0,0],gather(iCOM), ... - {'Bah',1,'spline'},'cpu','inv'); - - end - tWDG{n} = wdgTMP{iP}; wdgTMP{iP} = []; - tWDG{n} = tWDG{n} - min(tWDG{n}(:)) + 1e-6; - tWDG{n} = tWDG{n} ./ max(tWDG{n}(:)); - n = n + 1; - end - end - - wdgPAD = BH_multi_padVal(size(tWDG{1}), sizeCalc) - for iWdg = 1:n-1 - tWDG_r{iWdg} = BH_padZeros3d(tWDG{iWdg},wdgPAD(1,:),wdgPAD(2,:),... - 'cpu',flgPrecision); - tWDG{iWdg} = ifftshift(tWDG_r{iWdg}); - end - - refIMG{iGold} = tIMG ; clear tIMG refTMP - refWGT{iGold} = tWDG; clear tWDG wdgTMP - refWgtROT{iGold} = tWDG_r; clear tWDG_r - - clear comMask - -end - -[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, maxGoldStandard ); - - - -% optimize the fft for the given size. Padding to the next power of 2 is usually -% slower given the dimensionalityl of the volume data. -fftPlanner = rand(sizeCalc); -fftw('planner', 'exhaustive'); -fftn(fftPlanner); -clear fftPlanner - -% Make a mask, and apply to the average motif && save a masked, -% binned copy of the average for inspection. -% % % -% % % [ volMask ] = gather(BH_mask3d(maskType, sizeMask, maskRadius, maskCenter)); -% % % volBinary = (volMask >= 0.01); - - -% In principle the window and mask could be different sizes, however, I -% think I am currently forcing them to be the same. - - - % make rotationally invariant -% % % % % % % [ peakMask] = gather(BH_mask3d('sphere', sizeWindow, [1,1,1].*max(peakSearch), maskCenter)); - [ peakMask ] = gather(EMC_maskShape('sphere', sizeWindow, [1,1,1].*floor(max(peakSearch)), 'gpu', {'shift', maskCenter})); - - if (eraseMask) - % Mask could be smaller than the normal taper would allow, so instead - % of thresholding a normal mask, take this alt route. - eraseMask = ones(ceil(2.*eraseMaskRadius),'single'); - padEraseMask = BH_multi_padVal(size(eraseMask),sizeCalc); - eraseMask = BH_padZeros3d(eraseMask,padEraseMask(1,:),padEraseMask(2,:),'cpu','single'); - eraseMask = single(find(eraseMask < 1)); - else - eraseMask = []; - end - - - if ( flgRaw_shapeMask ) -% % % % % % % [ volMask ] = BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter); - [ volMask ] = EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter}); - - % Currently not set up for mult-ref alignment - iRef = 1; - % Use the geometric mean so that excluded areas mask out -% % % % % % % [ volMask ] = gather(sqrt(volMask .* ... -% % % % % % % BH_mask3d(refIMG{1}{iRef}+refIMG{2}{iRef},pixelSize,'',''))); - - [ volMask ] = gather(sqrt(volMask .* ... - EMC_maskReference(refIMG{1}{iRef}+refIMG{2}{iRef}, pixelSize, {'fsc', true}))); - - else -% % % % % % % [ volMask ] = gather(BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter)); - [ volMask ] = gather(EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter})); - - end - -% % % [ peakMask] = gather(BH_mask3d(maskType, sizeMask, peakSearch, maskCenter)); -% % % peakBinary = (peakMask >= 0.01); - -% [ refInterp] = gather(BH_mask3d(maskType, sizeREF, peakSearch, maskCenter)); -% refInterp = (refInterp >= 0.01); - - bandpassFilt = cell(nReferences(1),1); - bandpassFiltREF = bandpassFilt; - wCCC = cell(nReferences(1),1); - for iWccc = 1:length(nReferences(1)); - wCCC{iWccc} = 0; - end - if (flgClassify || flgMultiRefAlignment) - for iRef = 1:nReferences(1) - if (flgClassify) - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); - else - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Raw%d',iRef)); % % % % - end - - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - % returns a cpu array - if (flgWeightCCC) - [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1); - else - [ bandpassFilt{iRef}, ~] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1); - end - - - bandpassFiltREF{iRef} = 1; - - end - else - - - - for iRef = 1 - fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - % returns a cpu array - if (flgWeightCCC) - [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1 ); - else - [ bandpassFilt{iRef},~ ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1 ); - end - - bandpassFiltREF{iRef} = 1; - - - end - - end - -% if (flgWeightCCC) -% for i = 1:length(wCCC{1}) -% i -% length(wCCC{1}{i}) -% end -% end - - % This is just used to limit the interpolation search so use the most - % permissive bandpass, while the appropriate bandpass (given a multi-ref - % alignment) will still be applied. - mostPermissive = zeros(1,nReferences(1)); - for iRef = 1:nReferences(1) - mostPermissive(iRef) = sum(bandpassFilt{iRef}(:)); - end - [~,mPidx] = max(mostPermissive); - - wdgBinary = single(find(fftshift(bandpassFilt{mPidx} > 10^-2))); - - -ref_FT1 = cell(2,1); -ref_FT2 = cell(2,1); - - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - nOut = 1; - refOUT = cell(2.*nReferences(iGold),2); - - for iRef = 1:nReferences(iGold) - - refTMP_2 = refIMG{iGold}{iRef}; refIMG{iGold}{iRef} = []; - refTMP = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); - - - % if not using a weighted average (adapted SPW filter), apply an - % approximation the cRef from Rosenthal/Henderson. This is currently always set to one - % and is just doing the masking and normalization. It should be okay to just apply the mask - % and rely on the normalization during the CCC calc. TODO - ref_FT1{iGold}{iRef} = gather(conj(BH_bandLimitCenterNormalize(... - refTMP.*volMask, bandpassFiltREF{iRef}, (volMask>0.01), padCalc, flgPrecision))); - - - - - ref_FT2{iGold}{iRef} = gather(refTMP_2); - % Trim for output reference - refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); - - % Overwrite a copy of the filtered, bandpassed ref for output - refOUT{nOut} = real(ifftn(conj(ref_FT1{iGold}{iRef}))); - refOUT{nOut} = gather(refOUT{nOut}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); - - - - refOUT{nOut} = refOUT{nOut}.*volMask; - - refOUT{nOut+1} = real(ifftn(BH_bandLimitCenterNormalize(... - refTMP_2.*peakMask, '', (peakMask > 0.01), padCalc, 'single'))); - refOUT{nOut+1} = gather(refOUT{nOut+1}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) .* peakMask ); - nOut = nOut + 2; - - refOUT{nOut} = refOUT{nOut} - mean(refOUT{nOut}(:)); - refOUT{nOut} = refOUT{nOut} ./ rms(refOUT{nOut}(:)); - - refOUT{nOut+1} = refOUT{nOut+1} - mean(refOUT{nOut+1}(:)); - refOUT{nOut+1} = refOUT{nOut+1} ./ rms(refOUT{nOut+1}(:)); - end - - - % Save a montage of the masked reference & shape masks if requested. - - maskedOUTFILE = sprintf('%s_maskedRef-mont_%s.mrc',outputPrefix,halfSet); - [ maskedReferences, ~ ] = BH_montage4d(refOUT, ''); - SAVE_IMG(MRCImage(single(maskedReferences)), maskedOUTFILE); - - -end - -clear refIMG refWDG refOUT iRef - -%%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't -%%%%%%%%%%%%%%%%%%%%% search at all in that dimension. - -[ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch) - -[masterTM] = BH_recordAngularSampling( masterTM, cycleNumber, angleStep, inPlaneSearch); - -% set truth value for refinement during out of plane search - -if any(angleStep(:,1)) - flgRefine = true; - fprintf('flgRefine set to %s','True'); -else - flgRefine = false; - fprintf('flgRefine set to %s','False'); -end - -angleStep(:,1) -any(angleStep(:,1)) -nCount = 1; - -firstLoop = true; -nIgnored = 0; - -bestAnglesResults = cell(nParProcesses,1); -geometryResults = cell(nParProcesses,1); - - - -try - EMC_parpool(nParProcesses+1) -catch - delete(gcp('nocreate')) - EMC_parpool(nParProcesses+1) -end - -size(ref_FT2) - -system('mkdir -p alignResume'); - -system(sprintf('mkdir -p alignResume/%s',outputPrefix)); -softenWeight = 1/sqrt(samplingRate); -for iParProc = 1:nParProcesses - - % Caclulating weights takes up a lot of memory, so do all that are necessary - % prior to the main loop -- CHANGE THE CHECK TO JUST READ THE HEADER NOT LOAD - % THE WEIGHT INTO GPU MEMORY - iParProc - iterList{iParProc} - for iTomo = iterList{iParProc} - - BH_multi_loadOrCalcWeight(masterTM,ctfGroupList,tomoList{iTomo},samplingRate ,... - sizeCalc,geometry,flgPrecision,1); - - - end -end - -% Clear all of the GPUs prior to entering the main processing loop -for iGPU = 1:nGPUs - g = gpuDevice(iGPU); - fprintf('\n\nClear gpu %d mem prior to main loop, %3.3e available\n\n',iGPU,g.AvailableMemory); - clear g -end - -parVect = 1:nParProcesses; -parfor iParProc = parVect -%for iParProc = 1:nParProcesses -%profile on - bestAngles_tmp = struct(); - geometry_tmp = geometry; - -% % % % Get the gpuIDX assigned to this process -% % % iGPUidx = gpuDevice(); -% % % iGPUidx = iGPUidx.Index; - gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; - iGPUidx = gpuIDXList(iParProc); - gpuDevice(iGPUidx); - fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); - for iTomo = iterList{iParProc} - - - - nCtfGroups = ctfGroupList.(tomoList{iTomo})(1); - % Check for interupted alignment. - previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); - if exist(previousAlignment,'file') - % Sometimes when multiple nodes are used, an extra line is added. - % TODO fix this workaround - system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s_clean; mv %s_clean %s',... - previousAlignment,previousAlignment,previousAlignment,previousAlignment)); - bestAngles_tmp.(tomoList{iTomo}) = load(previousAlignment); - fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); - else - % There is some memory leak somewhere that I haven't been able to figure - % out. I am clearing all vars but output in the children functions ... this - % isn't ideal, but for now is an acceptable stop gap. - %D = gpuDevice(gpuList(iGPU)); - - % shake up the random number generator for phi and theta - rng('shuffle'); - - bandpassFilt_tmp = cell(nReferences(1),1); - bandpassFiltREF_tmp = cell(nReferences(1),1); - for iRef = 1:nReferences(1) - if flgMultiRefAlignment <= 2 - bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); - else - bandpassFilt_tmp{iRef} = (bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = (bandpassFiltREF{iRef}); - end - end - - - - ref_FT1_tmp = cell(2,1); - ref_FT2_tmp = cell(2,1); - ref_WGT_tmp = cell(2,1); - ref_WGT_rot = cell(2,1); - - - volMask_tmp = gpuArray(volMask); - volBinary_tmp = single(find( volMask_tmp > 0.01 )); - peakMask_tmp = gpuArray(peakMask); - peakBinary_tmp = single(find( peakMask_tmp > 0.01 )); - wdgBinary_tmp = gpuArray(wdgBinary); - eraseMask_tmp = gpuArray(eraseMask); - - wCCC_tmp = cell(length(wCCC)); - - - - for iRef = 1:nReferences(1) - for iWccc = 1:length(wCCC{iRef}) - if (flgWeightCCC) - wCCC_tmp{iRef}{iWccc} = gpuArray(wCCC{iRef}{iWccc}); - else - % The check in xcf_rotational looks for a cell - wCCC_tmp{iRef} = 0; - end - end - end - - - - for iGold = 1:2 - for iRef = 1:nReferences(iGold) - if flgMultiRefAlignment <= 2 - ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = gpuArray(refWgtROT{iGold}{iRef}); - else - % Temp workaround, six big ribo refs crashing - ref_FT1_tmp{iGold}{iRef} = (ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = (ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = (refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = (refWgtROT{iGold}{iRef}); - end - end - end - - - sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) - tic; - - % Load the tomo into gpu - tomoName = tomoList{iTomo}; - %fprintf('gpu %d working on tomoName %s\n', iGPU, tomoName); - - tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); - % Load in the geometry for the tomogram, and get number of subTomos. - positionList = geometry_tmp.(tomoList{iTomo}); - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); - -% [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); - binShift = [0,0,0]; - nSubTomos = size(positionList,1); - - - - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; - wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); -% wgtName = sprintf('cache/%s_bin%d.wgt', tomoList{iTomo},... -% samplingRate); - maxWedgeMask = BH_unStackMontage4d(1:nCtfGroups,wgtName,... - ceil(sqrt(nCtfGroups)).*[1,1],''); - maxWedgeIfft = maxWedgeMask; - - for iWdg = 1:length(maxWedgeMask) - if ~isempty(maxWedgeMask{iWdg}) - maxWedgeIfft{iWdg} = ifftshift(maxWedgeIfft{iWdg}.^softenWeight); - maxWedgeMask{iWdg} = maxWedgeMask{iWdg}.^softenWeight; - end - end - - - fprintf('loaded %s.\n',wgtName); - - - % Can't clear inside the parfor, but make sure we don't have two tomograms - % in memory at once. - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - - if (flgCutOutVolumes) - volumeData = []; - else - [ volumeData, ~ ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,loadTomo); - if ( loadTomo ) - volHeader = struct(); - volHeader.('nX') = size(volumeData,1); - volHeader.('nY') = size(volumeData,2); - volHeader.('nZ') = size(volumeData,3); - else - volHeader = getHeader(volumeData); - end - end - - - % For now, set up for full grid-search only, as I intend to just do - % translational and in-plane searches for now anyhow. - - [~,iv1,iv2,iv3] = BH_resample3d(volMask_tmp,eye(3),[0,0,0],... - {'Bah',1,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv'); - inputVectors = {iv1,iv2,iv3}; - iv1 = []; iv2 = []; iv3 = []; - cccStorageBest = cell(nPeaks,1); - cccStorageRefine = cell(nPeaks,1); - for iPeak = 1:nPeaks - cccStorageBest{iPeak} = zeros(nSubTomos,10); - cccStorageRefine{iPeak}= zeros(nSubTomos,10); - end - % reset for each tomogram - wdgIDX = 0; - - for iSubTomo = 1:nSubTomos - breakPeak = 0; % for try catch on cut out vols - if (wdgIDX ~= positionList(iSubTomo,9)) - % Geometry is sorted on this value so that tranfers are minimized, - % as these can take up a lot of mem. For 9 ctf Groups on an 80s - % ribo at 2 Ang/pix at full sampling ~ 2Gb eache. - wdgIDX = positionList(iSubTomo,9); - fprintf('pulling the wedge %d onto the GPU\n',wdgIDX); - % Avoid temporar - iMaxWedgeMask = []; iMaxWedgeIfft = []; - iMaxWedgeMask = gpuArray(maxWedgeMask{wdgIDX}); - iMaxWedgeIfft = gpuArray(maxWedgeIfft{wdgIDX}); - end - - - [~,iw1,iw2,iw3] = BH_resample3d(iMaxWedgeMask, eye(3), [0,0,0], ... - {'Bah',1,'linear',1,wdgBinary_tmp}, ... - 'GPU', 'inv'); - inputWgtVectors = {iw1,iw2,iw3}; - iw1 = []; iw2 = []; iw3 = []; - - for iPeak = 1:nPeaks - if (breakPeak) - continue; - end - getInitialCCC = 1; - cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); - cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); - powerOut = zeros(nAngles(1).*nReferences(1),1,'gpuArray'); - - - - % Used in refinment loop - angCount = 1; - - % Check that the given subTomo is not to be ignored - classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); - particleIDX = positionList(iSubTomo, 4); - iGold = positionList(iSubTomo, 7); - - - if classVector{iGold}(1,:) == 0 - classPosition = 1; - flgAllClasses = true; - else - classPosition = find(classVector{iGold}(1,:) == classIDX); - flgAllClasses = false; - end - - - - if (classIDX ~= -9999) && ... % All previously ignored particles - ( flgAllClasses || ismember(classIDX, classVector{iGold}(1,:)) ) - - - center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; - angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); - - % Find range to extract, and check for domain error. - if (flgCutOutVolumes) - % Need some check that the windowsize has not changed! TODO TODO - - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(2*CUTPADDING+sizeWindow, ... - sizeWindow,maskRadius, center); - else - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... - sizeWindow,maskRadius, center); - end - - - - - if ischar(indVAL) - fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) - nIgnored = nIgnored + 1; - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - else - - - if (flgCutOutVolumes) - % Test with some generic padding , only to be used on bin 1 at - % first!!! TODO add a flag to check this. - try - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iparticle = gpuArray(getVolume(MRCImage(particleOUT_name),[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - catch - fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - breakPeak = 1; - continue; - end - else - - if ( loadTomo ) - iparticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3))); - - else - iparticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - end - - end - [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... - padVAL(2,1:3), 'GPU', 'singleTaper'); - - - - for iAngle = 1:size(angleStep,1) - - theta = angleStep(iAngle,1); - thetaInc = angleStep(iAngle,4); - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. - - phiInc = angleStep(iAngle,3); - - % To prevent only searching the same increments each time in a limited - % grid search, radomly offset the azimuthal angle by a random number - % between 0 and 1/2 the azimuthal increment. - - azimuthalRandomizer = (rand(1)-0.5)*phiInc; - - for iAzimuth = 0:angleStep(iAngle,2) - phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); - - for iInPlane = inPlaneSearch - psi = iInPlane; - psiInc = angleStep(iAngle,5); - %[phi,theta,psi-phi]; - - - RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - cccStorageTrans= zeros(1.*nReferences(1),10,'gpuArray'); - - for alignLoop = 1:2 - - - switch alignLoop - - case 1 - % This takes care of non-inter shift in the origin that is - % ignored during the windowing of the particle. - estPeakCoord = shiftVAL; - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - - case 2 - - - bestOfRefs = sortrows(gather(cccStorageTrans), -6); - %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); - - estPeakCoord = bestOfRefs(1,8:10); - - - - - % Assuming if class specific symmetry, then some not just 1 - if (flgSymmetry) - symmetry = classSymmetry{iGold}(1, classPosition); - %fprintf('Symmetry confirmation %d\n',symmetry); - [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... - estPeakCoord,... - {'Bah',symmetry,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv',inputVectors); - - - - if (getInitialCCC) - [ iTrimInitial ] = BH_resample3d(iparticle, ... - reshape(angles,3,3),... - shiftVAL,... - {'Bah',symmetry,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv',inputVectors); - - - -% % % powerInitial = sum(abs(iTrimInitial(volBinary_tmp))).^2; -% % % - - iWedgeInitial = BH_resample3d(iMaxWedgeMask, reshape(angles,3,3), [0,0,0], ... - {'Bah',symmetry,'linear',1,wdgBinary_tmp}, ... - 'GPU', 'inv',inputWgtVectors); - - - end - - iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - {'Bah',symmetry,'linear',1,wdgBinary_tmp}, ... - 'GPU', 'inv',inputWgtVectors); - - - - - - else - symmetry = 1; - % Transform the particle, and then trim to motif size - - [ iTrimParticle ] = BH_resample3d(iparticle, RotMat, ... - estPeakCoord, {'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'inv',inputVectors); - iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'inv',inputWgtVectors); - - - - if (getInitialCCC) - - iTrimInitial = BH_resample3d(iparticle, reshape(angles,3,3),... - shiftVAL,{'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'inv',inputVectors); - - - - iWedgeInitial = BH_resample3d(iMaxWedgeMask, reshape(angles,3,3), [0,0,0], ... - {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'inv',inputWgtVectors); - - end - end % Symmetry or not + interpolation - - -% % % powerOut(angCount) = sum(abs(iTrimParticle(volBinary_tmp))).^2; - - end - - - switch flgMultiRefAlignment - case 0 - refToAlign = 1; - case 1 - refToAlign = 1:max(nReferences(:)); - case 2 - refToAlign = classIDX; - otherwise - error('flgMultiRefAlignment is not 0,1,2') - end - - for iRef = refToAlign % 1:max(nReferences(:)) - - switch alignLoop - - case 1 - - - % use transpose of RotMat - - iRotRef = BH_resample3d(ref_FT2_tmp{iGold}{iRef}, RotMat', ... - estPeakCoord, {'Bah',1,'linear',1,peakBinary_tmp}, 'GPU', 'forward',inputVectors); - - iRotWdg = BH_resample3d(ref_WGT_rot{iGold}{iRef}, RotMat', ... - [0,0,0], {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - -% iRotRef = ... -% iRotRef(padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - % maybe I should be rotating peak mask here in case it has - % an odd shape, since we are leaving the proper frame - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef.*peakMask_tmp,... - bandpassFiltREF_tmp{iRef} ,peakBinary_tmp,... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*peakMask_tmp,... - bandpassFilt_tmp{iRef} ,peakBinary_tmp,padCalc,flgPrecision); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - peakMask_tmp, peakCOM,eraseMask_tmp); - - - cccStorageTrans(iRef,:) = [iRef, particleIDX, ... - phi, theta, psi - phi, ... - 0, 0, ... - peakCoord + estPeakCoord]; - case 2 - - % get starting point - if (getInitialCCC) - - initialRotPart_FT = BH_bandLimitCenterNormalize(... - iTrimInitial.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( initialRotPart_FT, ... - ref_FT1_tmp{iGold}{iRef}, ... - ifftshift(iWedgeInitial),... - ref_WGT_tmp{iGold}{iRef}, ... - wCCC_tmp{iRef}); - - - - - cccInitial(iRef,:) = [iRef, particleIDX, ... - 0,0,0, ... - iCCC, 1, ... - shiftVAL]; - - - initialRotPart_FT = []; - - - end - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - - - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{iGold}{iRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{iGold}{iRef}, ... - wCCC_tmp{iRef}); - - - - - - - % Note that no new translational estimate is made, so no - % need to multiply by RotMat - cccStorage2(angCount,:) = ... - [iRef, particleIDX, ... - phi, theta, psi , ... - iCCC, 1, ... - estPeakCoord]; - - - angCount = angCount + 1; - end - - - end % loop over references. - - - - end - % This volume won't be needed until the next subTomo is considered, - % which is also where getInitialCCC Boolean is set to True again. - iTrimInitial = []; - getInitialCCC = 0; - - end % in plane angles - end % azimuth - end % polar - -% % % fprintf('Power ratio is %3.3f\n',powerOut./powerInitial); - - cccPreRefineSort = sortrows(gather(cccStorage2),-6); - - if (length(refToAlign) > 1) - cccInitial = sortrows(gather(cccInitial), -6); - cccInitial = cccInitial(1,:); - else - cccInitial = gather(cccInitial(refToAlign,:)); - - end - - if cccInitial(1,6 ) > cccPreRefineSort(1,6) - cccPreRefineSort(1,:) = cccInitial(1,:); - end - - - - % This only seems to be a problem with cut out volumes. - % Normalization maybe? - if ~any(cccPreRefineSort(1,:)) - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); - continue - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - if (flgRefine) - - - - % Get the results from just this subTomo and sort on CCC - - rRef = cccPreRefineSort(1,1); - rPart = cccPreRefineSort(1,2); - rPhi = cccPreRefineSort(1,3); - rPhiInc = phiInc / 4; - rTheta= cccPreRefineSort(1,4); - rTheInc = thetaInc /2; - rPsi = cccPreRefineSort(1,5); - rPsiInc = psiInc /2; - % Confirm shiftVAL is doing what it should be - rXYZest = cccPreRefineSort(1,8:10); - - if (rTheInc) - % For a larger out of plane step, search a larger range in plane - psiRefineStep = floor(sqrt(rTheInc)); - else - psiRefineStep = 1; - end - - thetaRefineStep =1; - phiRefineStep=2; - totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; - totalRefineStep = prod((2.*totalRefineStep)+1); - - cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); - - if (rPsiInc == 0) - inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; - else - inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; - end - polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; - azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; - - searchList = zeros(totalRefineStep,3); - nSearch = 1; - for iPhi = azimuthalRefine - for iTheta = polarRefine - for iPsi = inPlaneRefine - % best iPsi is origin Psi - Phi, no need to subtract here. - - searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; - - nSearch = nSearch + 1; - end - end - end % end of building angle list - - for iRefine = 1:nSearch-1 - for alignLoop = 1:2 - if alignLoop == 1 - rXYZ = rXYZest; - elseif alignLoop == 2 - rXYZ = cccStorage3(iRefine,8:10); - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - - RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - - - - switch alignLoop - % This keeps seperate shifts due to windowing and binning from - % shifts found in CCC - case 1 - - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - case 2 - - % Assuming if class specific symmetry, then some not just 1 - if (flgSymmetry) - symmetry = classSymmetry{iGold}(1, classPosition); - - [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... - rXYZ,... - {'Bah',symmetry,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv',inputVectors); - - -% iTrimParticle = iTrimParticle(... -% padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - {'Bah',symmetry,'linear',1,wdgBinary_tmp},... - 'GPU', 'inv',inputWgtVectors); - - - else - symmetry = 1; - % Transform the particle, and then trim to motif size - - [ iTrimParticle ] = BH_resample3d(iparticle, RotMat, ... - rXYZ, {'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'inv',inputVectors); - - - iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'inv'); - - -% iTrimParticle = ... -% iTrimParticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - - end % Symmetry or not + interpolation - - - - - - end - - - - if alignLoop == 1 - - % use transpose of RotMat - try - iRotRef = BH_resample3d(ref_FT2_tmp{iGold}{rRef}, RotMat', ... - rXYZ, {'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); - catch - cccPreRefineSort(1,1) - end - iRotWdg = BH_resample3d(ref_WGT_rot{iGold}{rRef}, RotMat', ... - [0,0,0], {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - - - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef.*peakMask_tmp,... - bandpassFiltREF_tmp{rRef},peakBinary_tmp,... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*peakMask_tmp,... - bandpassFilt_tmp{rRef} ,peakBinary_tmp,padCalc,flgPrecision); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - peakMask_tmp, peakCOM,eraseMask_tmp); - - - % 2016-11-11 also took out (+ rXYZ) - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - 1, 1, ... - peakCoord+rXYZ]; - else - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{rRef},volBinary_tmp,... - padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{iGold}{rRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{iGold}{rRef}, ... - wCCC_tmp{iRef}); - - - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - iCCC, 1, ... - rXYZ] ; - end - end - - end - - sortRef = sortrows(gather(cccStorage3),-6); - cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); - - end % end of refinement loop - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % Get the final translational shift for the best scoring angular - % match. - try - if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) - bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); -% % % % Get the negative slope of the top ten CCC scores. -% % % topTen = fit([.1:.1:1]',sortRef(1:10,6),'linear'); -% % % bestRotPeak(1,7) = topTen(100)-topTen(101); - - else - bestRotPeak = cccPreRefineSort(1,:); - bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); -% % % rowNum = min(size(cccPreRefineSort,1),10*nPeaks); -% % % topX = 1- 0.1.*(10-rowNum); -% % % % Get the negative slope of the top ten CCC scores. -% % % topTen = fit([.1:.1:topX]',cccPreRefineSort(1:rowNum,6),'linear'); -% % % bestRotPeak(1,7) = topTen(100)-topTen(101); - - end - catch - fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); - cccStorageRefine{iPeak}(iSubTomo,:) - cccPreRefineSort(1,:) -% % % rowNum = min(size(cccPreRefineSort,1),10*nPeaks) -% % % topX = 1- 0.1.*(10-rowNum) -% % % fprintf('\nNow check the fits, first and second clause\n'); -% % % topTen = fit([.1:.1:1]',sortRef(1:10,6),'linear') -% % % fprintf('\nSecond\n'); -% % % topTen = fit([.1:.1:topX]',cccPreRefineSort(1:rowNum,6),'linear') -% % % error('Error in sorting the best peak in alignRaw'); - end - - finalRef = bestRotPeak(1,1); - finalPart = bestRotPeak(1,2); - finalPhi = bestRotPeak(1,3); - finalTheta= bestRotPeak(1,4); - finalPsi = bestRotPeak(1,5); - % Confirm shiftVAL is doing what it should be - finalrXYZest = bestRotPeak(1,8:10); - - RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - - - - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - - % use transpose of RotMat - %%% 2016-11-11 estPeakCoord should have been finalrXYZest in - %%% the last writing, but now switching to zeros - try - iRotRef = BH_resample3d(ref_FT2_tmp{iGold}{finalRef}, RotMat', ... - finalrXYZest, {'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); - iRotWdg = BH_resample3d(ref_WGT_rot{iGold}{finalRef}, RotMat', ... - [0,0,0], {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - catch - fprintf('\n\nFinal ref,part,phi,theta,psi %f %f %f %f %f\n\n',... - bestRotPeak(:,1:5)); - bestRotPeak(1,1:5) - fprintf('BreakPeak %d\n',breakPeak); - error('errrorsoedfsdf') - end - -% iRotRef = ... -% iRotRef(padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef.*peakMask_tmp,... - bandpassFiltREF_tmp{finalRef} ,peakBinary_tmp,... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*peakMask_tmp,... - bandpassFilt_tmp{finalRef} ,peakBinary_tmp,padCalc,flgPrecision ); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - peakMask_tmp, peakCOM,eraseMask_tmp); - -% % % end - - - - - % Subtract shiftVAL since this is due to windowing, not the actual - % position. - cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... - peakCoord + finalrXYZest - shiftVAL]) ; - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - % It is probably more useful see the shifts in the particle - % reference frame vs. the avg which was the original - if (bh_global_print_shifts_in_particle_basis) - printShifts = zeros(3,3); - printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); - printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); - printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); - else - printShifts = [cccInitial(1,end-2:end); ... - cccPreRefineSort(1,end-2:end);... - cccStorageBest{iPeak}(iSubTomo,end-2:end)]; - end - - % Print out in Angstrom - printShifts = printShifts .* pixelSize; - - - deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); - if (deltaCCC < 0) && (abs(deltaCCC) > 0.15*cccInitial(1,6)) - fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); - fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - - end - - if (flgRefine) - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX,cccInitial(1,1:end-3),printShifts(1,:), ... - 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... - cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... - 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - - else - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX,cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - - end - - - end % if condition on newly ignored particles - - end - - if ~(rem(iSubTomo,100)) - timeClass = toc; - fprintf('\nworking on %d/%d subTomo from %s...%fs\n',... - iSubTomo,nSubTomos,tomoName,timeClass); - tic; - end - - - iParticle = []; - iSymParti = []; - iTrimParticle = []; - iAsym = []; - iTrimAsym = []; - iWedgeMask = []; - rotPart_FT = []; - rotParticle = []; - end % end loop over possible peaks - end % loop over subTomos - - - for iPeak = 1:nPeaks - - % Get rid of any zero entries left over from pre-initialization - if iPeak == 1 - nonZeroInits = ( cccStorageBest{iPeak}(:,2) ~= 0 ); - cccStorageBest{1}=cccStorageBest{1}(nonZeroInits,:); - sortCCC = zeros(size(cccStorageBest{1},1),10*nPeaks); - else - cccStorageBest{iPeak}=cccStorageBest{iPeak}(nonZeroInits,:); - end - - sortCCC(:,1+10*(iPeak-1):10+10*(iPeak-1)) = cccStorageBest{iPeak}; - end - -% % % % I think this is redundant now, but leaving until I double check. -% % % save('sortCCC.mat','sortCCC'); - [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); - - cccSortedandUnique = sortCCC(a,:); -% % % save('cccSortedandUnique.mat','cccSortedandUnique'); -% % % g = gather(geometry); -% % % save('TBL_geom.mat','g'); - - bestAngles_tmp.(tomoList{iTomo}) = gather(cccSortedandUnique); - - % save doesn't work in a parfor, so write out the results for each tomogram so that a - % run may be resumed if cancelled. - angOut = fopen(sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}),'w'); - - for iRow = 1:size( bestAngles_tmp.(tomoList{iTomo}),1) - for iPeak = 1:nPeaks - fprintf(angOut,'%d %d %6.3f %6.3f %6.3f %6.6f %6.6f %6.3f %6.3f %6.3f ', ... - bestAngles_tmp.(tomoList{iTomo})(iRow,1+10*(iPeak-1):10+10*(iPeak-1))); - end - fprintf(angOut,'\n'); - end - fclose(angOut); - - end % if clause to check for previous alignment - end % loop over tomos - bestAnglesResults{iParProc} = bestAngles_tmp; - geometryResults{iParProc} = geometry_tmp; -%profile off -%profsave -end % parfor - - - -save('bestAnglesResults.mat', 'bestAnglesResults'); -bestAngles = struct(); -for iParProc = 1:nParProcesses - for iTomo = iterList{iParProc} - geometry.(tomoList{iTomo}) = geometryResults{iParProc}.(tomoList{iTomo}); - bestAngles.(tomoList{iTomo}) = bestAnglesResults{iParProc}.(tomoList{iTomo}); - end -end -% save('bestAnglesTemp.mat', 'bestAngles'); - save('bestAngles.mat', 'bestAngles'); - - [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, nPeaks,rotConvention ); - masterTM.(cycleNumber).('RawAlign') = rawAlign; - masterTM.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); - -clear bestAngles rawAlign -subTomoMeta = masterTM; - -if ( flgReverseOrder || flgStartThird ) - fprintf('This reverse run will not write the metaData\n'); -else - save(pBH.('subTomoMeta'), 'subTomoMeta'); -end - -delete(gcp('nocreate')) -for iGPU = 1:nGPUs - gpuDevice(iGPU); -end - -end % end of alignRaw3d diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 1d935aff..02d7b2a5 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -1,5 +1,5 @@ - function [ ] = BH_alignRaw3d_v2(PARAMETER_FILE, CYCLE, varargin) - +function [ ] = BH_alignRaw3d_v2(PARAMETER_FILE, CYCLE, varargin) + %Extract and align class averages and references from 4D montages derived. % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -12,15 +12,7 @@ % TODO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -global bh_global_print_shifts_in_particle_basis; -if isempty(bh_global_print_shifts_in_particle_basis) - bh_global_print_shifts_in_particle_basis = true; -end -global bh_global_zero_lag_score; -if isempty(bh_global_zero_lag_score) - bh_global_zero_lag_score = false; -end if (nargin ~= 2 && nargin ~= 3) error('args = PARAMETER_FILE, CYCLE, [1,abs(ccc),2,weighted,3,abs(weighted)]') @@ -41,12 +33,12 @@ cpuVar = struct(); GPUVar = struct(); -startTime = clock; +startTime = datetime("now"); CYCLE = EMC_str2double(CYCLE); cycle_numerator = ''; cycle_denominator =''; - flgStartThird = 0; - flgReverseOrder = 0; +flgStartThird = 0; +flgReverseOrder = 0; if numel(CYCLE) == 3 cycle_numerator = CYCLE(2); cycle_denominator = CYCLE(3); @@ -58,127 +50,45 @@ flgReverseOrder = 1; flgStartThird = 0; CYCLE = abs(CYCLE); - + end - - -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); cycleNumber = sprintf('cycle%0.3u', CYCLE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); -mapBackIter = subTomoMeta.currentTomoCPR; +% Load using wrapper +subTomoMeta = BH_loadSubTomoMeta(emc.('subTomoMeta'), emc.('metadata_format')); +mapBackIter = subTomoMeta.currentTomoCPR; reconScaling = 1; -try - nPeaks = pBH.('nPeaks'); -catch - nPeaks = 1; -end - -try - track_stats = pBH.('track_stats'); -catch - track_stats = false; -end - -try - flgCutOutVolumes=pBH.('flgCutOutVolumes') -catch - flgCutOutVolumes=0 -end - -% TODO decide on a "reasonable" padding based on expected shifts. -try - CUTPADDING = subTomoMeta.('CUTPADDING') -catch - CUTPADDING=20 -end - -try - use_v2_SF3D = pBH.('use_v2_SF3D') -catch - use_v2_SF3D = true; -end -try - symmetry_op = pBH.('symmetry'); -catch - error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); -end -try - use_new_grid_search = pBH.('use_new_grid_search'); -catch - use_new_grid_search = true; -end -try - force_no_symmetry = pBH.('force_no_symmetry'); -catch - force_no_symmetry = false; -end -if (force_no_symmetry) - symmetry_op='C1' - fprintf('\nWarning, overriding symmetry in the alignment. THis is just for benchmarking\n'); -end maxGoldStandard = subTomoMeta.('maxGoldStandard'); -nGPUs = pBH.('nGPUs') +nGPUs = emc.('nGPUs'); -flgClassify= pBH.('flgClassify'); -try - flgMultiRefAlignment=pBH.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end -try - updateClassByBestReferenceScore = pBH.('updateClassByBestReferenceScore'); -catch - updateClassByBestReferenceScore = false; -end -if (~flgMultiRefAlignment) - updateClassByBestReferenceScore = false; -end -try - flgCenterRefCOM = pBH.('flgCenterRefCOM'); -catch - flgCenterRefCOM = 1; -end +samplingRate = emc.('Ali_samplingRate'); -try - flgSymmetrizeSubTomos = pBH.('flgSymmetrizeSubTomos'); -catch - flgSymmetrizeSubTomos = 0; -end -flgRaw_shapeMask = 0;%= pBH.('experimentalOpts')(3) -samplingRate = pBH.('Ali_samplingRate'); +emc.pixel_size_angstroms = emc.pixel_size_angstroms.*samplingRate; -pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; -if pBH.('SuperResolution') - pixelSize = pixelSize * 2; -end -flgPrecision = 'single'; %pBH.('flgPrecision'); -angleSearch = pBH.('Raw_angleSearch'); -peakSearch = (pBH.('particleRadius')./pixelSize); +flgPrecision = 'single'; %emc.('flgPrecision'); +angleSearch = emc.('Raw_angleSearch'); +peakSearch = (emc.('particleRadius')./emc.pixel_size_angstroms); peakCOM = [1,1,1].*3; -className = pBH.('Raw_className'); +className = emc.('Raw_className'); -try - loadTomo = pBH.('loadTomo') -catch - loadTomo = 0; -end -try - eraseMaskType = pBH.('Peak_mType'); - eraseMaskRadius = pBH.('Peak_mRadius')./pixelSize; + +try + eraseMaskType = emc.('Peak_mType'); + eraseMaskRadius = emc.('Peak_mRadius')./emc.pixel_size_angstroms; fprintf('Further restricting peak search to radius %f %f %f\n',... - eraseMaskRadius); + eraseMaskRadius); eraseMask = 1; catch eraseMask = 0; @@ -188,71 +98,43 @@ rotConvention = 'Bah'; % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = pBH.('doHelical'); -catch - doHelical = 0; -end -if ( doHelical ) - rotConvention = 'Helical' -end -rotConvention -try - bFactor = pBH.('Fsc_bfactor'); -catch - bFactor = 0; -end -if length(bFactor) > 1 - fprintf('multiple bFactors specified, using the first for alignment.\n'); - bFactor = bFactor(1); -end +% if (emc.classification) +% refName = emc.('Ref_className'); +% else + refName = emc.('Raw_className'); +% end -try - scaleCalcSize = pBH.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end -% % % % if (flgClassify || flgMultiRefAlignment) -if (flgClassify) - refName = pBH.('Ref_className'); -else - refName = pBH.('Raw_className'); -end - -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); +outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); -classVector{1} = pBH.('Raw_classes_odd')(1,:); +classVector{1} = emc.('Raw_classes_odd')(1,:); -classVector{2} = pBH.('Raw_classes_eve')(1,:); +classVector{2} = emc.('Raw_classes_eve')(1,:); -% % % % if (flgClassify || flgMultiRefAlignment) -if (flgClassify) - geometry = subTomoMeta.(cycleNumber).ClassAlignment; - refVectorFull{1}= [pBH.('Ref_references_odd');1] - refVectorFull{2}= [pBH.('Ref_references_eve');1] -elseif (flgMultiRefAlignment) +% % % % if (emc.classification || emc.multi_reference_alignment) +% if (emc.classification) +% geometry = subTomoMeta.(cycleNumber).ClassAlignment; +% refVectorFull{1}= [emc.('Ref_references_odd');1] +% refVectorFull{2}= [emc.('Ref_references_eve');1] +% else if +if (emc.multi_reference_alignment) geometry = subTomoMeta.(cycleNumber).ClusterRefGeom; - refVectorFull{1}= [pBH.('Raw_classes_odd');classVector{1} ] - refVectorFull{2}= [pBH.('Raw_classes_eve');classVector{2} ] + refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ] + refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ] else geometry = subTomoMeta.(cycleNumber).Avg_geometry; - refVectorFull{1} = [pBH.('Raw_classes_odd');1]; - refVectorFull{2} = [pBH.('Raw_classes_eve');1]; + refVectorFull{1} = [emc.('Raw_classes_odd');1]; + refVectorFull{2} = [emc.('Raw_classes_eve');1]; end % % % pathList= subTomoMeta.mapPath; % % % extList = subTomoMeta.mapExt; -masterTM = subTomoMeta; clear subTomoMeta - - - refVector = cell(2,1); refGroup = cell(2,1); @@ -262,206 +144,171 @@ % Sort low to high, because order is rearranged as such unstack refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) + refVector{iGold} = refVectorFull{iGold}(1,:); % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) + refGroup{iGold} = refVectorFull{iGold}(3,:); % axial symmetry to apply, negative value indicates creating a mirrored ref % accros the corresponding axis - refSym{iGold} = refVectorFull{iGold}(2,:) + refSym{iGold} = refVectorFull{iGold}(2,:); end % make sure the number of references match the unique groups in the classVector % and also that the class/group pairs match the class/ref pairs. nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})] +nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]; nRefOut(1:2) = [length(unique(refGroup{1})) + sum(( refSym{1} < 0 )),... - length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; - - -%%%%%%%%%%%%%%%%%%%%%%% + length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; % Get the number of tomograms to process. tomoList = fieldnames(geometry); nTomograms = length(tomoList); -tiltList = masterTM.tiltGeometry; -ctfGroupList = masterTM.('ctfGroupSize'); +tiltList = subTomoMeta.tiltGeometry; % Sort the list by number of active subtomos to improve parallelism sortedTomoList = zeros(nTomograms,1); for iTomo = 1:nTomograms sortedTomoList(iTomo) = sum(geometry.(tomoList{iTomo})(:,26)~=-9999); + + % If we have previously aligned this tomo, it will be skipped, so don't count it + % Previously, only the existence of the file was checked in the parfor loop, but this + % adds a check that the number of lines matches the number of subtomos, so that if the alignment was + previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); + if exist(previousAlignment,'file') + % Sometimes when multiple nodes are used, an extra line is added. + % In the parfor loop, multiple processes are prevented from writing this from the final iterlist but here we would have to use a random tmp file + tmp_filename = tempname; + system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s; ', previousAlignment, tmp_filename)); + test_load = importdata(tmp_filename); + if (size(test_load,1) ~= sortedTomoList(iTomo)) + fprintf('Warning: Number of lines in %s (%d) does not match number of subtomos for %s (%d)\n', previousAlignment, size(test_load,1), tomoList{iTomo}, sortedTomoList(iTomo)); + pause(1); + delete(previousAlignment); + else + fprintf('Skipping previously aligned tomogram %s\n', tomoList{iTomo}); + sortedTomoList(iTomo) = 0; + end + end end -[~, sortedTomoIDX] = sort(sortedTomoList,'descend') +[~, sortedTomoIDX] = sort(sortedTomoList,'descend'); +tomoList = tomoList(sortedTomoIDX); % mask defines area for angular search, peakRADIUS restricts translational [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Ali', pixelSize) + BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms); [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); -try - flgLimitToOneProcess = pBH.('flgLimitToOneProcess'); -catch - flgLimitToOneProcess = 0; -end - -if ( loadTomo ) - limitToOne = loadTomo; - if (flgLimitToOneProcess) - limitToOne = min(limitToOne, flgLimitToOneProcess); - end -elseif (flgLimitToOneProcess) - limitToOne = flgLimitToOneProcess; +if (flgStartThird) + [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, sizeCalc(1), emc.nCpuCores, [cycle_numerator,cycle_denominator]); else - limitToOne = pBH.('nCpuCores'); + [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, sizeCalc(1), emc.nCpuCores); end - - -nParProcesses = 0; -iterList = {}; if ( flgReverseOrder ) - fprintf('nCpuCores is %d\n', limitToOne); - [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); - for iParProc = 1:nParProcesses - iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' - end - % Flip the order for reverse processing on a second machine. This will also disable saving of + % Flip the order for reverse processing on a second machine. This will also disable saving of % of the metadata so there aren't conflicts. for iParProc = 1:nParProcesses iterList{iParProc} = flip(iterList{iParProc}); end - -elseif ( flgStartThird ) - fprintf('nCpuCores is %d\n', limitToOne); - [ nParProcesses, iterList_full] = BH_multi_parallelJobs(nTomograms,nGPUs*cycle_denominator, sizeCalc(1),limitToOne*cycle_denominator); - - for iParProc = 1:nParProcesses - iterList_full{iParProc} = sortedTomoIDX(iterList_full{iParProc})'; - end - - % Need to scale this back down - nParProcesses = limitToOne; - - % Shift to start at one third through to process on a third machine. This will also disable saving of - % of the metadata so there aren't conflicts. - iterList = {}; - for iParProc = 1:nParProcesses - idx = cycle_numerator + (iParProc-1)*cycle_denominator; - if (idx <= length(iterList_full)) - iterList{iParProc} = iterList_full{idx}; - end - end - -else - fprintf('nCpuCores is %d\n', limitToOne); - [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); - for iParProc = 1:nParProcesses - iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' - end - - end + if any(peakSearch > maskRadius) fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') peakSearch( (peakSearch > maskRadius) ) = ... - maskRadius( (peakSearch > maskRadius) ); + maskRadius( (peakSearch > maskRadius) ); end - - % Read in the references. % Read in the references. refIMG = cell(2,1); refWGT = cell(2,1); refWgtROT = cell(2,1); imgCounts = cell(2,1); +ref_to_class_idx = cell(2,1); for iGold = 1:2 - + if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - - - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) - - weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', refName, halfSet); - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - + + imgNAME = sprintf('class_%d_Locations_Ref_%s', refName, halfSet) + + + weightNAME = sprintf('class_%d_Locations_Ref_%s_Wgt', refName, halfSet); + imgCounts{iGold} = subTomoMeta.(cycleNumber).(imgNAME){3}; + % Get the class index for each reference + ref_to_class_idx{iGold} = subTomoMeta.(cycleNumber).(imgNAME){3}(1,:); + [ refTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - + subTomoMeta.(cycleNumber).(imgNAME){1}, ... + subTomoMeta.(cycleNumber).(imgNAME){2},... + sizeWindow); + [ wdgTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... - sizeCalc); - - sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}(2:2:6)'; - - if (flgCenterRefCOM) -% % % % % % % [ comMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); - [ comMask ] = EMC_maskShape(maskType, sizeMask, maskRadius, 'gpu', {'shift', maskCenter}); + subTomoMeta.(cycleNumber).(weightNAME){1},... + subTomoMeta.(cycleNumber).(weightNAME){2},... + sizeCalc); + + sizeREF = subTomoMeta.(cycleNumber).(imgNAME){2}{1}(2:2:6)'; + + if (emc.move_reference_by_com) + % % % % % % % [ comMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); + [ comMask ] = EMC_maskShape(maskType, sizeMask, maskRadius, 'gpu', {'shift', maskCenter}); end - - % get boxSize + + % get boxSize n = 1 ; tIMG = cell(numel(refVector{iGold})); tWDG = cell(numel(refVector{iGold}));tWDG_r = tWDG; for iP = 1:numel(refTMP) if ~isempty(refTMP{iP}) tIMG{n} = refTMP{iP}; refTMP{iP} = []; - if (flgCenterRefCOM) - % Not sure if this is always the best approach, but it may be - % useful in some cases. -% % % % % % % [~,iCOM] = BH_mask3d(gpuArray(tIMG{n}).*comMask,pixelSize,'','',1); + if (emc.move_reference_by_com) - [~, ~, ~,iCOM] = EMC_maskReference(gpuArray(tIMG{n}).*comMask, pixelSize, {'fsc',true; 'com', true}); + [~, ~, ~,iCOM] = EMC_maskReference(gpuArray(tIMG{n}).*comMask, emc.pixel_size_angstroms, {'fsc',true; 'com', true}); fprintf('centering ref %d on COM %3.3f %3.3f %3.3f \n',n,iCOM); - + tIMG{n} = BH_resample3d(tIMG{n},[0,0,0],gather(iCOM), ... - {'Bah',1,'spline'},'cpu','inv'); - + {'Bah',1,'spline'},'cpu','inv'); + end - - tWDG{n} = wdgTMP{iP}; wdgTMP{iP} = []; - tWDG{n} = tWDG{n} - min(tWDG{n}(:)) + 1e-6; - tWDG{n} = tWDG{n} ./ max(tWDG{n}(:)); - + + tWDG{n} = wdgTMP{iP}; wdgTMP{iP} = []; + tWDG{n} = tWDG{n} - min(tWDG{n}(:)) + 1e-6; + tWDG{n} = tWDG{n} ./ max(tWDG{n}(:)); + n = n + 1; end end - - - wdgPAD = BH_multi_padVal(size(tWDG{1}), sizeCalc); - for iWdg = 1:n-1 - tWDG_r{iWdg} = BH_padZeros3d(tWDG{iWdg},wdgPAD(1,:),wdgPAD(2,:),... - 'cpu',flgPrecision); - tWDG{iWdg} = ifftshift(tWDG_r{iWdg}); - end - - refWGT{iGold} = tWDG; clear tWDG wdgTMP - refWgtROT{iGold} = tWDG_r; clear tWDG_r - + + + wdgPAD = BH_multi_padVal(size(tWDG{1}), sizeCalc); + for iWdg = 1:n-1 + tWDG_r{iWdg} = BH_padZeros3d(tWDG{iWdg},wdgPAD(1,:),wdgPAD(2,:),... + 'cpu',flgPrecision); + tWDG{iWdg} = ifftshift(tWDG_r{iWdg}); + end + + refWGT{iGold} = tWDG; clear tWDG wdgTMP + refWgtROT{iGold} = tWDG_r; clear tWDG_r + refIMG{iGold} = tIMG ; clear tIMG refTMP - - + + clear comMask end -[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, maxGoldStandard ); +[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, emc.pixel_size_angstroms, maxGoldStandard ); @@ -474,163 +321,154 @@ - - - stat_mask = []; - if (eraseMask) - peakMask = EMC_maskShape(eraseMaskType,sizeCalc,floor(eraseMaskRadius),'cpu',{'kernel',false}); - if track_stats - stat_mask = single(find(peakMask > 0.95)); - end - - else - if track_stats - stat_mask = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter}); - stat_mask = single(find(stat_mask > 0.95)); - end - [ peakMask ] = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter;'kernel',false}); + +stat_mask = []; +if (eraseMask) + peakMask = EMC_maskShape(eraseMaskType,sizeCalc,floor(eraseMaskRadius),'cpu',{'kernel',false}); + + if ( emc.track_stats ) + stat_mask = single(find(peakMask > 0.95)); end - - if ( flgRaw_shapeMask ) - - [ volMask ] = gather(sqrt(volMask .* ... - EMC_maskReference(refIMG{1}{iRef}+refIMG{2}{iRef}, pixelSize, {'fsc', true}))); - - else -% % % % % % % [ volMask ] = gather(BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter)); - [ volMask ] = gather(EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter})); - - end - - - - bandpassFilt = cell(nReferences(1),1); - bandpassFiltREF = bandpassFilt; - wCCC = cell(nReferences(1),1); - for iWccc = 1:length(nReferences(1)); - wCCC{iWccc} = 0; +else + if ( emc.track_stats ) + stat_mask = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter}); + stat_mask = single(find(stat_mask > 0.95)); end - if (flgClassify || flgMultiRefAlignment) - for iRef = 1:nReferences(1) - if (flgClassify) - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); - else - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Raw%d',iRef)); % % % % - end + [ peakMask ] = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter;'kernel',false}); +end - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - % returns a cpu array - if (flgWeightCCC) - [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1); - else - [ bandpassFilt{iRef}, ~] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1); - end - - bandpassFiltREF{iRef} = 1; +[ volMask ] = gather(EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter})); - end - else - - for iRef = 1 - fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - % returns a cpu array - if (flgWeightCCC) - [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1 ); - else - [ bandpassFilt{iRef},~ ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1 ); - end +bandpassFilt = cell(nReferences(1),1); +bandpassFiltREF = bandpassFilt; +wCCC = cell(nReferences(1),1); +for iWccc = 1:length(nReferences(1)); + wCCC{iWccc} = 0; +end +% if (emc.classification || emc.multi_reference_alignment) +if ( emc.multi_reference_alignment) - bandpassFiltREF{iRef} = 1; + for iRef = 1:nReferences(1) - + fscINFO = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Ref%d',iRef)); + + [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... + 'GPU', {'none'}, 1, 0, 1 ); + radialGrid = single(radialGrid./emc.pixel_size_angstroms); + % returns a cpu array + if (flgWeightCCC) + [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1, 1); + else + [ bandpassFilt{iRef}, ~] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1); end - + + + bandpassFiltREF{iRef} = 1; + + end +else + + + + for iRef = 1 + + fscINFO = subTomoMeta.(cycleNumber).('fitFSC').('Ref1'); + [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... + 'GPU', {'none'}, 1, 0, 1 ); + radialGrid = single(radialGrid./emc.pixel_size_angstroms); + % returns a cpu array + if (flgWeightCCC) + [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1, 1 ); + else + [ bandpassFilt{iRef},~ ] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1 ); + end + + bandpassFiltREF{iRef} = 1; + + end +end + % if (flgWeightCCC) % for i = 1:length(wCCC{1}) % i % length(wCCC{1}{i}) % end % end - - % This is just used to limit the interpolation search so use the most - % permissive bandpass, while the appropriate bandpass (given a multi-ref - % alignment) will still be applied. - mostPermissive = zeros(1,nReferences(1)); - for iRef = 1:nReferences(1) - mostPermissive(iRef) = sum(bandpassFilt{iRef}(:)); - end - [~,mPidx] = max(mostPermissive); - - wdgBinary = single(find(fftshift(bandpassFilt{mPidx} > 10^-2))); - + +% This is just used to limit the interpolation search so use the most +% permissive bandpass, while the appropriate bandpass (given a multi-ref +% alignment) will still be applied. +mostPermissive = zeros(1,nReferences(1)); +for iRef = 1:nReferences(1) + mostPermissive(iRef) = sum(bandpassFilt{iRef}(:)); +end +[~,mPidx] = max(mostPermissive); + +wdgBinary = single(find(fftshift(bandpassFilt{mPidx} > 10^-2))); + ref_FT1 = cell(2,1); ref_FT2 = cell(2,1); - -for iGold = 1:2 +for iGold = 1:2 + if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - + nOut = 1; refOUT = cell(2.*nReferences(iGold),2); - + for iRef = 1:nReferences(iGold) refTMP_2 = refIMG{iGold}{iRef}; refIMG{iGold}{iRef} = []; refTMP = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); - - + padWindow(1,2) + 1: end - padWindow(2,2), ... + padWindow(1,3) + 1: end - padWindow(2,3)); + + % if not using a weighted average (adapted SPW filter), apply an % approximation the cRef from Rosenthal/Henderson. This is currently always set to one % and is just doing the masking and normalization. It should be okay to just apply the mask % and rely on the normalization during the CCC calc. TODO - ref_FT1{iGold}{iRef} = gather(conj(BH_bandLimitCenterNormalize(... - refTMP.*volMask, bandpassFiltREF{iRef}, (volMask>0.01), padCalc, flgPrecision))); - - - + ref_FT1{iGold}{iRef} = gather(conj(BH_bandLimitCenterNormalize(... + refTMP.*volMask, bandpassFiltREF{iRef}, (volMask>0.01), padCalc, flgPrecision))); + + + ref_FT2{iGold}{iRef} = gather(refTMP_2); % Trim for output reference - refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); + refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... + padWindow(1,2) + 1: end - padWindow(2,2), ... + padWindow(1,3) + 1: end - padWindow(2,3)); % Overwrite a copy of the filtered, bandpassed ref for output refOUT{nOut} = real(ifftn(conj(ref_FT1{iGold}{iRef}))); refOUT{nOut} = gather(refOUT{nOut}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); - - + padCalc(1,2) + 1: end - padCalc(2,2), ... + padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); + + refOUT{nOut} = refOUT{nOut}.*volMask; refOUT{nOut+1} = real(ifftn(BH_bandLimitCenterNormalize(... - refTMP_2, '', '', padCalc, 'single'))); + refTMP_2, '', '', padCalc, 'single'))); refOUT{nOut+1} = gather(refOUT{nOut+1}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) ); + padCalc(1,2) + 1: end - padCalc(2,2), ... + padCalc(1,3) + 1: end - padCalc(2,3)) ); nOut = nOut + 2; refOUT{nOut} = refOUT{nOut} - mean(refOUT{nOut}(:)); @@ -639,15 +477,15 @@ refOUT{nOut+1} = refOUT{nOut+1} - mean(refOUT{nOut+1}(:)); refOUT{nOut+1} = refOUT{nOut+1} ./ rms(refOUT{nOut+1}(:)); end - - + + % Save a montage of the masked reference & shape masks if requested. - -% maskedOUTFILE = sprintf('%s_maskedRef-mont_%s.mrc',outputPrefix,halfSet); -% [ maskedReferences, ~ ] = BH_montage4d(refOUT, ''); -% SAVE_IMG(MRCImage(single(maskedReferences)), maskedOUTFILE); - + % maskedOUTFILE = sprintf('%s_maskedRef-mont_%s.mrc',outputPrefix,halfSet); + % [ maskedReferences, ~ ] = BH_montage4d(refOUT, ''); + % SAVE_IMG(MRCImage(single(maskedReferences)), maskedOUTFILE); + + end clear refIMG refWDG refOUT iRef @@ -655,64 +493,58 @@ %%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't %%%%%%%%%%%%%%%%%%%%% search at all in that dimension. -updateWeights = false; -gridSearch = ''; -if (use_new_grid_search) - gridSearch = eulerSearch(symmetry_op, angleSearch(1),... - angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, true); +gridSearch = ''; +if (emc.use_new_grid_search) + gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... + angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, true); nAngles = sum(gridSearch.number_of_angles_at_each_theta); - inPlaneSearch = gridSearch.parameter_map.psi + inPlaneSearch = gridSearch.parameter_map.psi -try - symmetry_constrained_search = pBH.('symmetry_constrained_search'); - fprintf('Using symmetry constrained search\n'); -catch - symmetry_constrained_search = false; -end - - if (symmetry_constrained_search) + % symmetry_constrained_search is now handled in BH_parseParameterFile + symmetry_constrained_search = emc.symmetry_constrained_search; + if (~strcmpi(symmetry_constrained_search, 'none')) + fprintf('Using symmetry constrained search\n'); + end + + if (~strcmpi(symmetry_constrained_search, 'none')) % symmetry expansion on in-plane search only - if (gridSearch.symmetry_symbol(1) ~= 'C') + if (symmetry_constrained_search(1) ~= 'C') error('symmetry constrained search only implemented for Cn symmetry'); else - symmetry_number = EMC_str2double(gridSearch.symmetry_symbol(2:end)); + symmetry_number = EMC_str2double(symmetry_constrained_search(2:end)) + orig_search = inPlaneSearch; for iSym = 1:symmetry_number-1 - inPlaneSearch = [inPlaneSearch,inPlaneSearch + iSym.*(360/symmetry_number)]; + inPlaneSearch = [inPlaneSearch,orig_search + iSym.*(360/symmetry_number)]; end end end - - flgRefine=false; - - for i = 1:length(gridSearch.parameter_map.phi) - if gridSearch.parameter_map.phi{i} > 0 - flgRefine=true; - break; - end - end - - - angleStep = []; + + flgRefine= gridSearch.number_of_out_of_plane_angles > 1; + % for i = 1:gridSearch.number_of_out_of_plane_angles + % if gridSearch.parameter_map.phi{i} > 0 + % flgRefine=true; + % break; + % end + % end + + + angleStep = []; else [ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch); + = BH_multi_gridSearchAngles(angleSearch); if any(angleStep(:,1)) flgRefine = true; else flgRefine = false; - end - - if sum(angleStep(:,2) > 0) - updateWeights = true; - else end + + end -% [masterTM] = BH_recordAngularSampling( masterTM, cycleNumber, angleStep, inPlaneSearch); - -nCount = 1; +% [subTomoMeta] = BH_recordAngularSampling( subTomoMeta, cycleNumber, angleStep, inPlaneSearch); +nCount = 1; firstLoop = true; @@ -723,1222 +555,1036 @@ -try - EMC_parpool(nParProcesses+1) -catch - delete(gcp('nocreate')) - EMC_parpool(nParProcesses+1) -end +% EMC_parpool now handles cleanup internally +EMC_parpool(nParProcesses+1); size(ref_FT2) system('mkdir -p alignResume'); system(sprintf('mkdir -p alignResume/%s',outputPrefix)); -softenWeight = 1/sqrt(samplingRate); -if ~(use_v2_SF3D) - for iParProc = 1:nParProcesses - - % Caclulating weights takes up a lot of memory, so do all that are necessary - % prior to the main loop -- CHANGE THE CHECK TO JUST READ THE HEADER NOT LOAD - % THE WEIGHT INTO GPU MEMORY - for iTomo = iterList{iParProc} - - BH_multi_loadOrCalcWeight(masterTM,ctfGroupList,tomoList{iTomo},samplingRate ,... - sizeCalc,geometry,flgPrecision,1); - - - end - end - - % Clear all of the GPUs prior to entering the main processing loop - for iGPU = 1:nGPUs - g = gpuDevice(iGPU); - fprintf('\n\nClear gpu %d mem prior to main loop, %3.3e available\n\n',iGPU,g.AvailableMemory); - clear g - end -end parVect = 1:nParProcesses; fprintf('Starting main loopwith N references %d\n', nReferences(1)); + +% This may be modified in the parfor loop (tho that prob isn't really necessary) +particle_symmetry = emc.symmetry; +if (emc.force_no_symmetry) + particle_symmetry = 'C1'; +end parfor iParProc = parVect - symmetry = symmetry_op; % Why TF would this be necessary? -% for iParProc = 1:nParProcesses -%profile on + % for iParProc = parVect + + % To avoid unitialized temporary warnings + iMaxWedgeIfft = []; + imgWdgInterpolator = ''; + refInterpolator = ''; + refWdgInterpolator = ''; + particleInterpolator = ''; + phiInc = []; + thetaInc = []; + psiInc = []; + estPeakCoords = []; + iTrimParticle = []; + iTrimInitial = []; + iWedgeInitial = []; + iWedgeMask = []; + refToAlign = ''; + iRotRef = []; + iRotWdg = []; + iRotWdgMask = []; + + symmetry = emc.symmetry; + bestAngles_tmp = struct(); geometry_tmp = geometry; - -% % % % Get the gpuIDX assigned to this process -% % % iGPUidx = gpuDevice(); -% % % iGPUidx = iGPUidx.Index; - gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; - iGPUidx = gpuIDXList(iParProc); - gpuDevice(iGPUidx); - fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); + + gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; + iGPUidx = gpuIDXList(iParProc); + gpuDevice(iGPUidx); + fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); + for iTomo = iterList{iParProc} - - - - nCtfGroups = ctfGroupList.(tomoList{iTomo})(1); - % Check for interupted alignment. - previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); - if exist(previousAlignment,'file') - % Sometimes when multiple nodes are used, an extra line is added. - % TODO fix this workaround - system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s_clean; mv %s_clean %s',... - previousAlignment,previousAlignment,previousAlignment,previousAlignment)); - bestAngles_tmp.(tomoList{iTomo}) = load(previousAlignment); - fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); - else - % There is some memory leak somewhere that I haven't been able to figure - % out. I am clearing all vars but output in the children functions ... this - % isn't ideal, but for now is an acceptable stop gap. - %D = gpuDevice(gpuList(iGPU)); - - % shake up the random number generator for phi and theta - rng('shuffle'); - - bandpassFilt_tmp = cell(nReferences(1),1); - bandpassFiltREF_tmp = cell(nReferences(1),1); - for iRef = 1:nReferences(1) - if flgMultiRefAlignment <= 2 - bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); - else - bandpassFilt_tmp{iRef} = (bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = (bandpassFiltREF{iRef}); - end - end - - - - ref_FT1_tmp = cell(2,1); - ref_FT2_tmp = cell(2,1); - ref_WGT_tmp = cell(2,1); - ref_WGT_rot = cell(2,1); - - - volMask_tmp = gpuArray(volMask); - volBinary_tmp = single(find( volMask_tmp > 0.01 )); - peakMaskInterpolator = ''; - peakMaskInterpolator = interpolator(gpuArray(peakMask),[0,0,0],[0,0,0], rotConvention , 'forward', 'C1', false); - - if (track_stats) - mip = struct(); - mip.('mask') = gpuArray(stat_mask); - end - - - wCCC_tmp = cell(length(wCCC)); - - - for iRef = 1:nReferences(1) - for iWccc = 1:length(wCCC{iRef}) - if (flgWeightCCC) - wCCC_tmp{iRef}{iWccc} = gpuArray(wCCC{iRef}{iWccc}); + % Check for interupted alignment. + previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); + if exist(previousAlignment,'file') + % Sometimes when multiple nodes are used, an extra line is added. + % TODO fix this workaround + system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s_clean; mv %s_clean %s',... + previousAlignment,previousAlignment,previousAlignment,previousAlignment)); + bestAngles_tmp.(tomoList{iTomo}) = importdata(previousAlignment); + fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); + else + % There is some memory leak somewhere that I haven't been able to figure + % out. I am clearing all vars but output in the children functions ... this + % isn't ideal, but for now is an acceptable stop gap. + %D = gpuDevice(gpuList(iGPU)); + + % shake up the random number generator for phi and theta + rng('shuffle'); + + bandpassFilt_tmp = cell(nReferences(1),1); + bandpassFiltREF_tmp = cell(nReferences(1),1); + for iRef = 1:nReferences(1) + if emc.multi_reference_alignment <= 2 + bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); + bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); else - % The check in xcf_rotational looks for a cell - wCCC_tmp{iRef} = 0; + bandpassFilt_tmp{iRef} = (bandpassFilt{iRef}); + bandpassFiltREF_tmp{iRef} = (bandpassFiltREF{iRef}); end end - end - - - for iGold = 1:2 - for iRef = 1:nReferences(iGold) - if flgMultiRefAlignment <= 2 - ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = gpuArray(refWgtROT{iGold}{iRef}); - else - % Temp workaround, six big ribo refs crashing - ref_FT1_tmp{iGold}{iRef} = (ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = (ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = (refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = (refWgtROT{iGold}{iRef}); + + + ref_FT1_tmp = cell(2,1); + ref_FT2_tmp = cell(2,1); + ref_WGT_tmp = cell(2,1); + ref_WGT_rot = cell(2,1); + + + volMask_tmp = gpuArray(volMask); + volBinary_tmp = single(find( volMask_tmp > 0.01 )); + peakMaskInterpolator = ''; + peakMaskInterpolator = interpolator(gpuArray(peakMask),[0,0,0],[0,0,0], rotConvention , 'forward', 'C1', false); + + mip = struct(); + if (emc.track_stats) + mip.('mask') = gpuArray(stat_mask); + else + % to avoid uninitialized temporaries warnings + mip.('mask') = []; + end + + + wCCC_tmp = cell(length(wCCC)); + + + + for iRef = 1:nReferences(1) + for iWccc = 1:length(wCCC{iRef}) + if (flgWeightCCC) + wCCC_tmp{iRef}{iWccc} = gpuArray(wCCC{iRef}{iWccc}); + else + % The check in xcf_rotational looks for a cell + wCCC_tmp{iRef} = 0; + end end end - end - - - sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) - tic; - - % Load the tomo into gpu - tomoName = tomoList{iTomo}; - %fprintf('gpu %d working on tomoName %s\n', iGPU, tomoName); - - tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); - % Load in the geometry for the tomogram, and get number of subTomos. - positionList = geometry_tmp.(tomoList{iTomo}); - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); - -% [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); - binShift = [0,0,0]; - nSubTomos = size(positionList,1); - - - - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; - if ~(use_v2_SF3D) - wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); -% wgtName = sprintf('cache/%s_bin%d.wgt', tomoList{iTomo},... -% samplingRate); - maxWedgeMask = BH_unStackMontage4d(1:nCtfGroups,wgtName,... - ceil(sqrt(nCtfGroups)).*[1,1],''); - maxWedgeIfft = maxWedgeMask; - - for iWdg = 1:length(maxWedgeMask) - if ~isempty(maxWedgeMask{iWdg}) - maxWedgeMask{iWdg} = (maxWedgeMask{iWdg} - min(maxWedgeMask{iWdg}(:))) + 1e-3; - maxWedgeMask{iWdg} = maxWedgeMask{iWdg}.^softenWeight; - maxWedgeIfft{iWdg} = ifftshift(maxWedgeMask{iWdg}); - + + + + for iGold = 1:2 + for iRef = 1:nReferences(iGold) + if emc.multi_reference_alignment <= 2 + ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); + ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); + ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); + ref_WGT_rot{iGold}{iRef} = gpuArray(refWgtROT{iGold}{iRef}); + else + % Temp workaround, six big ribo refs crashing + ref_FT1_tmp{iGold}{iRef} = (ref_FT1{iGold}{iRef}); + ref_FT2_tmp{iGold}{iRef} = (ref_FT2{iGold}{iRef}); + ref_WGT_tmp{iGold}{iRef} = (refWGT{iGold}{iRef}); + ref_WGT_rot{iGold}{iRef} = (refWgtROT{iGold}{iRef}); end end - fprintf('loaded %s.\n',wgtName); - end + + % sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) + tic; + + % Load the tomo into gpu + tomoName = tomoList{iTomo}; + + tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); + % Load in the geometry for the tomogram, and get number of subTomos. + positionList = geometry_tmp.(tomoList{iTomo}); + - - % Can't clear inside the parfor, but make sure we don't have two tomograms + binShift = [0,0,0]; + nSubTomos = size(positionList,1); + + + + iTiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoName).tiltName; + + + + % Can't clear inside the parfor, but make sure we don't have two tomograms % in memory at once. - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); - - if (flgCutOutVolumes) - volumeData = []; - else - [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,loadTomo); - if ( loadTomo ) - volHeader = struct(); - volHeader.('nX') = size(volumeData,1); - volHeader.('nY') = size(volumeData,2); - volHeader.('nZ') = size(volumeData,3); - else - volHeader = getHeader(volumeData); - end - end - - - % For now, set up for full grid-search only, as I intend to just do - % translational and in-plane searches for now anyhow. - - [~,iv1,iv2,iv3] = BH_resample3d(volMask_tmp,eye(3),[0,0,0],... - {'Bah',1,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv'); - inputVectors = {iv1,iv2,iv3}; - iv1 = []; iv2 = []; iv3 = []; - cccStorageBest = cell(nPeaks,1); - cccStorageRefine = cell(nPeaks,1); - for iPeak = 1:nPeaks - cccStorageBest{iPeak} = zeros(nSubTomos,10); - cccStorageRefine{iPeak}= zeros(nSubTomos,10); - end - % reset for each tomogram - wdgIDX = 0; - - for iSubTomo = 1:nSubTomos - - make_SF3D = true; - breakPeak = 0; % for try catch on cut out vols - if (wdgIDX ~= positionList(iSubTomo,9)) && ~(use_v2_SF3D) - % Geometry is sorted on this value so that tranfers are minimized, - % as these can take up a lot of mem. For 9 ctf Groups on an 80s - % ribo at 2 Ang/pix at full sampling ~ 2Gb eache. - - wdgIDX = positionList(iSubTomo,9); - fprintf('pulling the wedge %d onto the GPU\n',wdgIDX); - % Avoid temporar - - iMaxWedgeMask = []; iMaxWedgeIfft = []; - iMaxWedgeMask = gpuArray(maxWedgeMask{wdgIDX}); - iMaxWedgeIfft = gpuArray(maxWedgeIfft{wdgIDX}); - imgWdgInterpolator = ''; - [imgWdgInterpolator, ~] = interpolator(iMaxWedgeMask,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - - + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + reconCoords = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + + TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); + + if (emc.flgCutOutVolumes) + volumeData = []; + else + do_load = false; + [ volumeData ] = BH_multi_loadOrBuild(emc.alt_cache, ... + tomoList{iTomo}, ... + mapBackIter, ... + samplingRate,... + iGPUidx, ... + do_load); + volHeader = getHeader(volumeData); end - -% [~,iw1,iw2,iw3] = BH_resample3d(iMaxWedgeMask, eye(3), [0,0,0], ... -% {'Bah',1,'linear',1,wdgBinary_tmp}, ... -% 'GPU', 'inv'); -% inputWgtVectors = {iw1,iw2,iw3}; -% iw1 = []; iw2 = []; iw3 = []; - - - for iPeak = 1:nPeaks - - if (track_stats) - measure_noise = true; - mip.('x') = {}; - mip.('x2') = {}; - mip.('N') = 0; -% mip.('X') = zeros(1,3,'single','gpuArray'); -% mip.('X2') = zeros(3,3,'single','gpuArray'); - end - if (breakPeak) - continue; - end - getInitialCCC = 1; - cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); - cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); - powerOut = zeros(nAngles(1).*nReferences(1),1,'gpuArray'); + % For now, set up for full grid-search only, as I intend to just do + % translational and in-plane searches for now anyhow. - - - % Used in refinment loop - angCount = 1; + [~,iv1,iv2,iv3] = BH_resample3d(volMask_tmp,eye(3),[0,0,0],... + {'Bah',1,'linear',1,volBinary_tmp}, ... + 'GPU', 'inv'); + inputVectors = {iv1,iv2,iv3}; + iv1 = []; iv2 = []; iv3 = []; + cccStorageBest = cell(emc.nPeaks,1); + cccStorageRefine = cell(emc.nPeaks,1); + for iPeak = 1:emc.nPeaks + cccStorageBest{iPeak} = zeros(nSubTomos,10); + cccStorageRefine{iPeak}= zeros(nSubTomos,10); + end + % reset for each tomogram + wdgIDX = 0; - % Check that the given subTomo is not to be ignored - classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); - particleIDX = positionList(iSubTomo, 4); - half_set = positionList(iSubTomo, 7); - - - % if classVector{half_set}(1,:) == 0 - % classPosition = 1; - % flgAllClasses = true; - % else - % classPosition = find(classVector{half_set}(1,:) == classIDX); - % flgAllClasses = false; - % end - % Align all valid subtomos, even if the do not belong to the classes we've selected as references. - % To ignore particles, remove them with geometry RemoveClases.m - flgAllClasses = true; - - - - if (classIDX ~= -9999) && ... % All previously ignored particles - ( flgAllClasses || ismember(classIDX, classVector{half_set}(1,:)) ) - - - center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; - angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); + for iSubTomo = 1:nSubTomos - % Find range to extract, and check for domain error. - if (flgCutOutVolumes) - % Need some check that the windowsize has not changed! TODO TODO - - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(2*CUTPADDING+sizeWindow, ... - sizeWindow,maskRadius, center); - else - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... - sizeWindow,maskRadius, center); - end - - - - - if ischar(indVAL) - fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) - nIgnored = nIgnored + 1; - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - else + make_SF3D = true; + breakPeak = 0; % for try catch on cut out vols - if (flgCutOutVolumes) - % Test with some generic padding , only to be used on bin 1 at - % first!!! TODO add a flag to check this. - try - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iparticle = gpuArray(getVolume(MRCImage(particleOUT_name),[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - catch - fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - breakPeak = 1; - continue; - end - else - - if ( loadTomo ) - iparticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3))); - + for iPeak = 1:emc.nPeaks + + if (emc.track_stats) + measure_noise = true; + mip.('x') = {}; + mip.('x2') = {}; + mip.('N') = 0; else - iparticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); + measure_noise = false; end + if (breakPeak) + continue; + end + getInitialCCC = 1; + cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); + cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); - end - [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... - padVAL(2,1:3), 'GPU', 'singleTaper'); - - - if (make_SF3D) - make_SF3D = false; - if use_v2_SF3D - % For now excluding the soften weight. - [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, ... - center,reconGeometry); + % Used in refinment loop + angCount = 1; + + % Check that the given subTomo is not to be ignored + classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); + particleIDX = positionList(iSubTomo, 4); + half_set = positionList(iSubTomo, 7); + + + % if classVector{half_set}(1,:) == 0 + % classPosition = 1; + % flgAllClasses = true; + % else + % classPosition = find(classVector{half_set}(1,:) == classIDX); + % flgAllClasses = false; + % end + % Align all valid subtomos, even if the do not belong to the classes we've selected as references. + % To ignore particles, remove them with geometry RemoveClases.m + % FIXME: what was this for? + flgAllClasses = true; + + + + if (classIDX ~= -9999) && ... % All previously ignored particles + ( flgAllClasses || ismember(classIDX, classVector{half_set}(1,:)) ) + + + center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; + angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); + + % Find range to extract, and check for domain error. + if (emc.flgCutOutVolumes) + % Need some check that the windowsize has not changed! TODO TODO - imgWdgInterpolator = ''; - % The unshifted mask is kept in texture mem until no longer - % needed - [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); - - end - % Just use C1 to initialize, whether or not this is the final - refInterpolator = ''; - refWdgInterpolator= ''; - particleInterpolator= ''; - - [refInterpolator, ~] = interpolator(gpuArray(ref_FT2_tmp{1}{1}),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator = interpolator(gpuArray(ref_WGT_rot{half_set}{iRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); - particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); - end - - if (use_new_grid_search) - theta_search = 1:gridSearch.number_of_out_of_plane_angles; - else - theta_search = 1:size(angleStep,1); - end - - for iAngle = theta_search - - if (use_new_grid_search) - theta = gridSearch.parameter_map.theta(iAngle); - if length(gridSearch.parameter_map.phi{iAngle}) > 1 - phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); - else - phiInc = 0; - end - thetaInc = gridSearch.theta_step; - numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid(2*CUTPADDING+sizeWindow, ... + sizeWindow,maskRadius, center); else - theta = angleStep(iAngle,1); - phiInc = angleStep(iAngle,3); - thetaInc = angleStep(iAngle,4); - numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... + sizeWindow,maskRadius, center); end - % To prevent only searching the same increments each time in a limited - % grid search, radomly offset the azimuthal angle by a random number - % between 0 and 1/2 the azimuthal increment. - azimuthalRandomizer = (rand(1)-0.5)*phiInc; - - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. - + + if ischar(indVAL) + fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) + nIgnored = nIgnored + 1; + geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; + else + + + if (emc.flgCutOutVolumes) + % Test with some generic padding , only to be used on bin 1 at + % first!!! TODO add a flag to check this. + try + particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); + iparticle = gpuArray(OPEN_IMG('single',particleOUT_name,[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + catch + fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); + geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; + breakPeak = 1; + continue; + end + else - - if (use_new_grid_search) - % FIXME randomizer passed as bool to eulerSearch - phi_search = gridSearch.parameter_map.phi{iAngle}; + iparticle = gpuArray(OPEN_IMG('single', volumeData, [indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + end + [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... + padVAL(2,1:3), 'GPU', 'singleTaper'); + + + if (make_SF3D) + make_SF3D = false; + % For now excluding the soften weight. + [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center, reconCoords, emc.wiener_constant); + imgWdgInterpolator = ''; + % The unshifted mask is kept in texture mem until no longer + % needed + [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); + + % Just use C1 to initialize, whether or not this is the final + refInterpolator = ''; + refWdgInterpolator= ''; + particleInterpolator= ''; + + [refInterpolator, ~] = interpolator(gpuArray(ref_FT2_tmp{1}{1}),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator = interpolator(gpuArray(ref_WGT_rot{half_set}{iRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); + particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); + end + + if (emc.use_new_grid_search) + theta_search = 1:gridSearch.number_of_out_of_plane_angles; else - phi_search = 0:angleStep(iAngle,2); + theta_search = 1:size(angleStep,1); end - - for iAzimuth = phi_search - - if (use_new_grid_search) - phi = rem(iAzimuth + azimuthalRandomizer,360); - psiInc = gridSearch.psi_step; - else - phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); - psiInc = angleStep(iAngle,5); - - end - - - for iInPlane = inPlaneSearch - psi = iInPlane; - %[phi,theta,psi-phi]; - - - RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - cccStorageTrans= zeros(1.*nReferences(1),10,'gpuArray'); - - for alignLoop = 1:2 - - - switch alignLoop + for iAngle = theta_search - case 1 - % This takes care of non-inter shift in the origin that is - % ignored during the windowing of the particle. - estPeakCoord = shiftVAL; - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - - case 2 - + if (emc.use_new_grid_search) + theta = gridSearch.parameter_map.theta(iAngle); + if length(gridSearch.parameter_map.phi{iAngle}) > 1 + phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); + else + phiInc = 0; + end + thetaInc = gridSearch.theta_step; + numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); + else + theta = angleStep(iAngle,1); + phiInc = angleStep(iAngle,3); + thetaInc = angleStep(iAngle,4); + numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - bestOfRefs = sortrows(gather(cccStorageTrans), -6); - %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); - - estPeakCoord = bestOfRefs(1,8:10); - - - + end - -% fprintf('Symmetry confirmation %d\n',symmetry); -% [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... -% estPeakCoord,... -% {'Bah',symmetry,'linear',1,volBinary_tmp}, ... -% 'GPU', 'inv',inputVectors); - [ iTrimParticle ] = particleInterpolator.interp3d(... - RotMat,... - estPeakCoord,rotConvention ,... - 'inv',symmetry); - - - - - if (getInitialCCC) -% [ iTrimInitial ] = BH_resample3d(iparticle, ... -% reshape(angles,3,3),... -% shiftVAL,... -% {rotConvention ,symmetry,'linear',1,volBinary_tmp}, ... -% 'GPU', 'inv',inputVectors); - [ iTrimInitial ] = particleInterpolator.interp3d(... - reshape(angles,3,3),... - shiftVAL,rotConvention ,... - 'inv',symmetry); + % To prevent only searching the same increments each time in a limited + % grid search, radomly offset the azimuthal angle by a random number + % between 0 and 1/2 the azimuthal increment. + + azimuthalRandomizer = (rand(1)-0.5)*phiInc; + + + % Calculate the increment in phi so that the azimuthal sampling is + % consistent and equal to the out of plane increment. + + + + if (emc.use_new_grid_search) + % FIXME randomizer passed as bool to eulerSearch + phi_search = gridSearch.parameter_map.phi{iAngle}; + else + phi_search = 0:angleStep(iAngle,2); + end + + + for iAzimuth = phi_search + + if (emc.use_new_grid_search) + phi = rem(iAzimuth + azimuthalRandomizer,360); + psiInc = gridSearch.psi_step; + else + phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); + psiInc = angleStep(iAngle,5); + end + + + for iInPlane = inPlaneSearch + psi = iInPlane; + %[phi,theta,psi-phi]; -% % % powerInitial = sum(abs(iTrimInitial(volBinary_tmp))).^2; -% % % - -% iWedgeInitial = BH_resample3d(iMaxWedgeMask, reshape(angles,3,3), [0,0,0], ... -% {rotConvention ,symmetry,'linear',1,wdgBinary_tmp}, ... -% 'GPU', 'inv',inputWgtVectors); - [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... - reshape(angles,3,3),... - [0,0,0],rotConvention ,... - 'inv',symmetry); + RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; + + cccStorageTrans= zeros(1.*nReferences(1),10,'gpuArray'); + + for alignLoop = 1:2 - - end - -% iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... -% {rotConvention ,symmetry,'linear',1,wdgBinary_tmp}, ... -% 'GPU', 'inv',inputWgtVectors); + + switch alignLoop + + case 1 + % This takes care of non-inter shift in the origin that is + % ignored during the windowing of the particle. + estPeakCoord = shiftVAL; + % Estimate the peakshift by rotating the ref not the particle. + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + case 2 + + bestOfRefs = sortrows(gather(cccStorageTrans), -6); + %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); + + estPeakCoord = bestOfRefs(1,8:10); + + [ iTrimParticle ] = particleInterpolator.interp3d(... + RotMat,... + estPeakCoord,rotConvention ,... + 'inv',particle_symmetry); + + if (getInitialCCC) + [ iTrimInitial ] = particleInterpolator.interp3d(... + reshape(angles,3,3),... + shiftVAL,rotConvention ,... + 'inv',particle_symmetry); + + [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... + reshape(angles,3,3),... + [0,0,0],rotConvention ,... + 'inv',particle_symmetry); + end + + [ iWedgeMask ] = imgWdgInterpolator.interp3d(... + RotMat,... + [0,0,0],rotConvention ,... + 'inv',particle_symmetry); + end % switch on align loop + + + switch emc.multi_reference_alignment + case 0 + refToAlign = 1; + case 1 + % Align this particle against all possible refs + refToAlign = 1:max(nReferences(:)); + case 2 + ref_to_class_idx_value = find(ref_to_class_idx{iGold} == classIDX); + if isempty(ref_to_class_idx_value) + error("No reference class found for classIDX %d", classIDX); + else + if length(ref_to_class_idx_value) > 1 + error("Multiple reference classes found for classIDX %d", classIDX); + end + end + + refToAlign = ref_to_class_idx_value; + otherwise + error('emc.multi_reference_alignment is not 0,1,2') + end + + for iRef = refToAlign + + switch alignLoop + + case 1 + % use transpose of RotMat + + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + estPeakCoord,rotConvention ,... + 'forward','C1'); + + + + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + + + + % maybe I should be rotating peak mask here in case it hastrack_stats + % an odd shape, since we are leaving the proper frame + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{iRef} ,'',... + padCalc,flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{iRef} ,'',padCalc,flgPrecision); + + if (emc.track_stats && measure_noise) + + + [ ~, mip ] = BH_multi_xcf_Translational_2( ... + rotPart_FT, ... + conj(iRotRef),... + ifftshift(iRotWdg),... + iMaxWedgeIfft,... + iRotMask, peakCOM,... + mip); + + + end + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + cccStorageTrans(iRef,:) = [iRef, particleIDX, ... + phi, theta, psi - phi, ... + 0, 0, ... + peakCoord + estPeakCoord]; + case 2 + + % get starting point + if (getInitialCCC) + + initialRotPart_FT = BH_bandLimitCenterNormalize(... + iTrimInitial.*volMask_tmp,... + bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); + + + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( initialRotPart_FT, ... + ref_FT1_tmp{half_set}{iRef}, ... + ifftshift(iWedgeInitial),... + ref_WGT_tmp{half_set}{iRef}, ... + wCCC_tmp{iRef}); + + + + + cccInitial(iRef,:) = [iRef, particleIDX, ... + 0,0,0, ... + iCCC, 1, ... + shiftVAL]; + + + initialRotPart_FT = []; + + + end + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); + + + + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1_tmp{half_set}{iRef},... + ifftshift(iWedgeMask),... + ref_WGT_tmp{half_set}{iRef}, ... + wCCC_tmp{iRef}); + + + + + + + % Note that no new translational estimate is made, so no + % need to multiply by RotMat + cccStorage2(angCount,:) = ... + [iRef, particleIDX, ... + phi, theta, psi , ... + iCCC, 1, ... + estPeakCoord]; + + + angCount = angCount + 1; + end + + + end % loop over references. + + + + end + % This volume won't be needed until the next subTomo is considered, + % which is also where getInitialCCC Boolean is set to True again. + iTrimInitial = []; + getInitialCCC = 0; - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... - RotMat,... - [0,0,0],rotConvention ,... - 'inv',symmetry); - - - - - + end % in plane angles + end % azimuth + end % polar + + % % % fprintf('Power ratio is %3.3f\n',powerOut./powerInitial); + + cccPreRefineSort = sortrows(gather(cccStorage2),-6); + + if (length(refToAlign) > 1) + cccInitial = sortrows(gather(cccInitial), -6); + cccInitial = cccInitial(1,:); + else + cccInitial = gather(cccInitial(refToAlign,:)); - -% % % powerOut(angCount) = sum(abs(iTrimParticle(volBinary_tmp))).^2; - end - - - switch flgMultiRefAlignment - case 0 - refToAlign = 1; - case 1 - refToAlign = 1:max(nReferences(:)); - case 2 - refToAlign = classIDX; - otherwise - error('flgMultiRefAlignment is not 0,1,2') + + if cccInitial(1,6 ) > cccPreRefineSort(1,6) + cccPreRefineSort(1,:) = cccInitial(1,:); end - - for iRef = refToAlign % 1:max(nReferences(:)) - - switch alignLoop - - case 1 - - - % use transpose of RotMat - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - estPeakCoord,rotConvention ,... - 'forward','C1'); - - - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - - - - % maybe I should be rotating peak mask here in case it has - % an odd shape, since we are leaving the proper frame - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{iRef} ,'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{iRef} ,'',padCalc,flgPrecision); + + + + % This only seems to be a problem with cut out volumes. + % Normalization maybe? + if ~any(cccPreRefineSort(1,:)) + cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); + continue + end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + if (flgRefine) + + + + % Get the results from just this subTomo and sort on CCC + + rRef = cccPreRefineSort(1,1); + rPart = cccPreRefineSort(1,2); + rPhi = cccPreRefineSort(1,3); + rPhiInc = phiInc / 4; + rTheta= cccPreRefineSort(1,4); + rTheInc = thetaInc /2; + rPsi = cccPreRefineSort(1,5); + rPsiInc = psiInc /2; + % Confirm shiftVAL is doing what it should be + rXYZest = cccPreRefineSort(1,8:10); + + if (rTheInc) + % For a larger out of plane step, search a larger range in plane + psiRefineStep = floor(sqrt(rTheInc)); + else + psiRefineStep = 1; + end + + thetaRefineStep =1; + phiRefineStep=2; + totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; + totalRefineStep = prod((2.*totalRefineStep)+1); + + cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); + + if (rPsiInc == 0) + inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; + else + inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; + end + polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; + azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; + + searchList = zeros(totalRefineStep,3); + nSearch = 1; + for iPhi = azimuthalRefine + for iTheta = polarRefine + for iPsi = inPlaneRefine + % best iPsi is origin Psi - Phi, no need to subtract here. - if (track_stats && measure_noise) - - - [ ~, mip ] = BH_multi_xcf_Translational_2( ... - rotPart_FT, ... - conj(iRotRef),... - ifftshift(iRotWdg),... - iMaxWedgeIfft,... - iRotMask, peakCOM,... - mip); - - + searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; + + nSearch = nSearch + 1; + end + end + end % end of building angle list + + for iRefine = 1:nSearch-1 + for alignLoop = 1:2 + if alignLoop == 1 + rXYZ = rXYZest; + elseif alignLoop == 2 + rXYZ = cccStorage3(iRefine,8:10); end - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - - cccStorageTrans(iRef,:) = [iRef, particleIDX, ... - phi, theta, psi - phi, ... - 0, 0, ... - peakCoord + estPeakCoord]; - case 2 + RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; - % get starting point - if (getInitialCCC) - - initialRotPart_FT = BH_bandLimitCenterNormalize(... - iTrimInitial.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( initialRotPart_FT, ... - ref_FT1_tmp{half_set}{iRef}, ... - ifftshift(iWedgeInitial),... - ref_WGT_tmp{half_set}{iRef}, ... - wCCC_tmp{iRef}); - - - - - cccInitial(iRef,:) = [iRef, particleIDX, ... - 0,0,0, ... - iCCC, 1, ... - shiftVAL]; - - - initialRotPart_FT = []; - + switch alignLoop + % This keeps seperate shifts due to windowing and binning from + % shifts found in CCC + case 1 + + % Estimate the peakshift by rotating the ref not the particle. + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + case 2 + + [ iTrimParticle ] = particleInterpolator.interp3d(... + RotMat,... + rXYZ,rotConvention ,... + 'inv',particle_symmetry); + + + [ iWedgeMask ] = imgWdgInterpolator.interp3d(... + RotMat,... + [0,0,0],rotConvention ,... + 'inv',particle_symmetry); + end - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - - + + + if alignLoop == 1 + + + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + rXYZ,rotConvention ,... + 'forward','C1'); + + + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{rRef},'',... + padCalc,flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{rRef} ,'',padCalc,flgPrecision); + + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + % 2016-11-11 also took out (+ rXYZ) + cccStorage3(iRefine,:) = [rRef, rPart, ... + searchList(iRefine,:), ... + 1, 1, ... + peakCoord+rXYZ]; + else + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{rRef},volBinary_tmp,... + padCalc,flgPrecision); [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{half_set}{iRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{half_set}{iRef}, ... - wCCC_tmp{iRef}); - - - - - - - % Note that no new translational estimate is made, so no - % need to multiply by RotMat - cccStorage2(angCount,:) = ... - [iRef, particleIDX, ... - phi, theta, psi , ... - iCCC, 1, ... - estPeakCoord]; - - - angCount = angCount + 1; + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1_tmp{half_set}{rRef},... + ifftshift(iWedgeMask),... + ref_WGT_tmp{half_set}{rRef}, ... + wCCC_tmp{iRef}); + + + cccStorage3(iRefine,:) = [rRef, rPart, ... + searchList(iRefine,:), ... + iCCC, 1, ... + rXYZ] ; + end + end + end - - - end % loop over references. + + sortRef = sortrows(gather(cccStorage3),-6); + cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); + + end % end of refinement loop - - - end - % This volume won't be needed until the next subTomo is considered, - % which is also where getInitialCCC Boolean is set to True again. - iTrimInitial = []; - getInitialCCC = 0; - - end % in plane angles - end % azimuth - end % polar - -% % % fprintf('Power ratio is %3.3f\n',powerOut./powerInitial); - - cccPreRefineSort = sortrows(gather(cccStorage2),-6); - - if (length(refToAlign) > 1) - cccInitial = sortrows(gather(cccInitial), -6); - cccInitial = cccInitial(1,:); - else - cccInitial = gather(cccInitial(refToAlign,:)); - - end - - if cccInitial(1,6 ) > cccPreRefineSort(1,6) - cccPreRefineSort(1,:) = cccInitial(1,:); - end - - - - % This only seems to be a problem with cut out volumes. - % Normalization maybe? - if ~any(cccPreRefineSort(1,:)) - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); - continue - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - if (flgRefine) - - - - % Get the results from just this subTomo and sort on CCC - - rRef = cccPreRefineSort(1,1); - rPart = cccPreRefineSort(1,2); - rPhi = cccPreRefineSort(1,3); - rPhiInc = phiInc / 4; - rTheta= cccPreRefineSort(1,4); - rTheInc = thetaInc /2; - rPsi = cccPreRefineSort(1,5); - rPsiInc = psiInc /2; - % Confirm shiftVAL is doing what it should be - rXYZest = cccPreRefineSort(1,8:10); - - if (rTheInc) - % For a larger out of plane step, search a larger range in plane - psiRefineStep = floor(sqrt(rTheInc)); - else - psiRefineStep = 1; - end - - thetaRefineStep =1; - phiRefineStep=2; - totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; - totalRefineStep = prod((2.*totalRefineStep)+1); - - cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); - - if (rPsiInc == 0) - inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; - else - inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; - end - polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; - azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; - - searchList = zeros(totalRefineStep,3); - nSearch = 1; - for iPhi = azimuthalRefine - for iTheta = polarRefine - for iPsi = inPlaneRefine - % best iPsi is origin Psi - Phi, no need to subtract here. - - searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; - - nSearch = nSearch + 1; - end - end - end % end of building angle list - - for iRefine = 1:nSearch-1 - for alignLoop = 1:2 - if alignLoop == 1 - rXYZ = rXYZest; - elseif alignLoop == 2 - rXYZ = cccStorage3(iRefine,8:10); + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Get the final translational shift for the best scoring angular + % match. + try + if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) + bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); + else + bestRotPeak = cccPreRefineSort(1,:); + bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); + end + catch + fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); + cccStorageRefine{iPeak}(iSubTomo,:) + cccPreRefineSort(1,:) end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); + finalRef = bestRotPeak(1,1); + finalPart = bestRotPeak(1,2); + finalPhi = bestRotPeak(1,3); + finalTheta= bestRotPeak(1,4); + finalPsi = bestRotPeak(1,5); + % Confirm shiftVAL is doing what it should be + finalrXYZest = bestRotPeak(1,8:10); + + RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); RotMat = reshape(angles,3,3) * RotMat; - - - - - switch alignLoop - % This keeps seperate shifts due to windowing and binning from - % shifts found in CCC - case 1 - - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - case 2 - - % Assuming if class specific symmetry, then some not just 1 - if (force_no_symmetry) - symmetry = 'C1'; - end - -% [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... -% rXYZ,... -% {rotConvention ,symmetry,'linear',1,volBinary_tmp}, ... -% 'GPU', 'inv',inputVectors); - [ iTrimParticle ] = particleInterpolator.interp3d(... - RotMat,... - rXYZ,rotConvention ,... - 'inv',symmetry); - - -% iTrimParticle = iTrimParticle(... -% padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); -% -% iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... -% {rotConvention ,symmetry,'linear',1,wdgBinary_tmp},... -% 'GPU', 'inv',inputWgtVectors); - - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... - RotMat,... - [0,0,0],rotConvention ,... - 'inv',symmetry); - - - - - - - - end - - - - if alignLoop == 1 - - % use transpose of RotMat -% try -% iRotRef = BH_resample3d(ref_FT2_tmp{half_set}{rRef}, RotMat', ... -% rXYZ, {rotConvention ,1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); -% catch -% cccPreRefineSort(1,1) -% end -% iRotWdg = BH_resample3d(ref_WGT_rot{half_set}{rRef}, RotMat', ... -% [0,0,0], {rotConvention ,1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); -% - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - rXYZ,rotConvention ,... - 'forward','C1'); - - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{rRef},'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{rRef} ,'',padCalc,flgPrecision); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - - % 2016-11-11 also took out (+ rXYZ) - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - 1, 1, ... - peakCoord+rXYZ]; - else - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{rRef},volBinary_tmp,... - padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{half_set}{rRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{half_set}{rRef}, ... - wCCC_tmp{iRef}); - - - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - iCCC, 1, ... - rXYZ] ; - end - end - - end - - sortRef = sortrows(gather(cccStorage3),-6); - cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); - - end % end of refinement loop - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % Get the final translational shift for the best scoring angular - % match. - try - if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) - bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); -% % % % Get the negative slope of the top ten CCC scores. -% % % topTen = fit([.1:.1:1]',sortRef(1:10,6),'linear'); -% % % bestRotPeak(1,7) = topTen(100)-topTen(101); - - else - bestRotPeak = cccPreRefineSort(1,:); - bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); -% % % rowNum = min(size(cccPreRefineSort,1),10*nPeaks); -% % % topX = 1- 0.1.*(10-rowNum); -% % % % Get the negative slope of the top ten CCC scores. -% % % topTen = fit([.1:.1:topX]',cccPreRefineSort(1:rowNum,6),'linear'); -% % % bestRotPeak(1,7) = topTen(100)-topTen(101); - - end - catch - fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); - cccStorageRefine{iPeak}(iSubTomo,:) - cccPreRefineSort(1,:) -% % % rowNum = min(size(cccPreRefineSort,1),10*nPeaks) -% % % topX = 1- 0.1.*(10-rowNum) -% % % fprintf('\nNow check the fits, first and second clause\n'); -% % % topTen = fit([.1:.1:1]',sortRef(1:10,6),'linear') -% % % fprintf('\nSecond\n'); -% % % topTen = fit([.1:.1:topX]',cccPreRefineSort(1:rowNum,6),'linear') -% % % error('Error in sorting the best peak in alignRaw'); - end - - finalRef = bestRotPeak(1,1); - finalPart = bestRotPeak(1,2); - finalPhi = bestRotPeak(1,3); - finalTheta= bestRotPeak(1,4); - finalPsi = bestRotPeak(1,5); - % Confirm shiftVAL is doing what it should be - finalrXYZest = bestRotPeak(1,8:10); - - RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - - - - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - + + + + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + % use transpose of RotMat %%% 2016-11-11 estPeakCoord should have been finalrXYZest in %%% the last writing, but now switching to zeros try -% iRotRef = BH_resample3d(ref_FT2_tmp{half_set}{finalRef}, RotMat', ... -% finalrXYZest, {rotConvention ,1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); -% iRotWdg = BH_resample3d(ref_WGT_rot{half_set}{finalRef}, RotMat', ... -% [0,0,0], {rotConvention ,1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - finalrXYZest,rotConvention ,... - 'forward','C1'); - - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + finalrXYZest,rotConvention ,... + 'forward','C1'); + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); catch fprintf('\n\nFinal ref,part,phi,theta,psi %f %f %f %f %f\n\n',... bestRotPeak(:,1:5)); - bestRotPeak(1,1:5) - fprintf('BreakPeak %d\n',breakPeak); + bestRotPeak(1,1:5) + fprintf('BreakPeak %d\n',breakPeak); error('errrorsoedfsdf') end - - -% iRotRef = ... -% iRotRef(padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{finalRef} ,'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{finalRef} ,'',padCalc,flgPrecision ); - - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - - - -% % % end - - - - - % Subtract shiftVAL since this is due to windowing, not the actual - % position. - cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... - peakCoord + finalrXYZest - shiftVAL]) ; - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - % It is probably more useful see the shifts in the particle - % reference frame vs. the avg which was the original - if (bh_global_print_shifts_in_particle_basis) - printShifts = zeros(3,3); - printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); - printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); - printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); - else - printShifts = [cccInitial(1,end-2:end); ... - cccPreRefineSort(1,end-2:end);... - cccStorageBest{iPeak}(iSubTomo,end-2:end)]; - end - - % Print out in Angstrom - printShifts = printShifts .* pixelSize; - - - deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); - if (deltaCCC < 0) && (abs(deltaCCC) > 0.15*cccInitial(1,6)) - fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); - fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{finalRef} ,'',... + padCalc,flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{finalRef} ,'',padCalc,flgPrecision ); + + + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + % Subtract shiftVAL since this is due to windowing, not the actual + % position. + cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... + peakCoord + finalrXYZest - shiftVAL]) ; + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + % It is probably more useful see the shifts in the particle + % reference frame vs. the avg which was the original + % Note: the final translation is with the ref rotated to the particles orientation, so the + % shift is in the tomogram (lab) reference frame. + if (emc.printShiftsInParticleBasis) + printShifts = zeros(3,3); + printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); + printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); + printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); + else + printShifts = [cccInitial(1,end-2:end); ... + cccPreRefineSort(1,end-2:end);... + cccStorageBest{iPeak}(iSubTomo,end-2:end)]; + end + + % Print out in Angstrom + printShifts = printShifts .* emc.pixel_size_angstroms; + + + deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); + if (emc.print_alignment_stats && deltaCCC < 0 && abs(deltaCCC) > 0.15*cccInitial(1,6)) + fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); + fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + + end + + if (emc.track_stats) + + if thetaInc > 0 + cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); + else + cccStorageBest{iPeak}(iSubTomo,end-3) = 0; + end + + % % I'm not sold on what do do with this. The distribution over the + % % shift parameters doesn't really seem to make sense to me. There + % % are too many factors that can lead to large shifts (e.g. + % % tomoCPR) If we were searching the full angular space each + % % iteration, then this would make sense. + % mip_mean = mip.X./mip.N; + % mip_covar = mip.X2./mip.N - transpose(mip_mean)*(mip_mean); + % mip_covar_inv = mip_covar\eye(3); + % gauss_norm = ((2.*pi).^(3/2).*abs(mip_covar)).^-1; + % gauss_exp = exp(-0.5.*(printShifts(2,:)-mip_mean)*mip_covar_inv*transpose(printShifts(2,:)-mip_mean)); + + end + + + cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); + cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); + if (emc.print_alignment_stats && flgRefine) + cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:), ... + 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... + cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... + 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + + else + if (emc.print_alignment_stats) + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + end + end + + + end % if condition on newly ignored particles + + end - end - - if (track_stats) - - if thetaInc > 0 - cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); - else - cccStorageBest{iPeak}(iSubTomo,end-3) = 0; + if ~(rem(iSubTomo,100)) + timeClass = toc; + fprintf('Refining %d/%d subTomo from %s...%fs\n', iSubTomo, nSubTomos, tomoName, timeClass); + tic; end -% % I'm not sold on what do do with this. The distribution over the -% % shift parameters doesn't really seem to make sense to me. There -% % are too many factors that can lead to large shifts (e.g. -% % tomoCPR) If we were searching the full angular space each -% % iteration, then this would make sense. -% mip_mean = mip.X./mip.N; -% mip_covar = mip.X2./mip.N - transpose(mip_mean)*(mip_mean); -% mip_covar_inv = mip_covar\eye(3); -% gauss_norm = ((2.*pi).^(3/2).*abs(mip_covar)).^-1; -% gauss_exp = exp(-0.5.*(printShifts(2,:)-mip_mean)*mip_covar_inv*transpose(printShifts(2,:)-mip_mean)); - - end + + iParticle = []; + iSymParti = []; + iTrimParticle = []; + iAsym = []; + iTrimAsym = []; + iWedgeMask = []; + rotPart_FT = []; + rotParticle = []; + end % end loop over possible peaks + iMaxWedgeIfft = []; + end % loop over subTomos + + + for iPeak = 1:emc.nPeaks - cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); - cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); - if (flgRefine) - cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:), ... - 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... - cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... - 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - + % Get rid of any zero entries left over from pre-initialization + if iPeak == 1 + nonZeroInits = ( cccStorageBest{iPeak}(:,2) ~= 0 ); + cccStorageBest{1}=cccStorageBest{1}(nonZeroInits,:); + sortCCC = zeros(size(cccStorageBest{1},1),10*emc.nPeaks); else - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - + cccStorageBest{iPeak}=cccStorageBest{iPeak}(nonZeroInits,:); end - - - end % if condition on newly ignored particles - - end - - if ~(rem(iSubTomo,100)) - timeClass = toc; - fprintf('\nworking on %d/%d subTomo from %s...%fs\n',... - iSubTomo,nSubTomos,tomoName,timeClass); - tic; - end - - - iParticle = []; - iSymParti = []; - iTrimParticle = []; - iAsym = []; - iTrimAsym = []; - iWedgeMask = []; - rotPart_FT = []; - rotParticle = []; - end % end loop over possible peaks - if use_v2_SF3D - iMaxWedgeIfft = []; + sortCCC(:,1+10*(iPeak-1):10+10*(iPeak-1)) = cccStorageBest{iPeak}; + end + + % % % % I think this is redundant now, but leaving until I double check. + % % % save('sortCCC.mat','sortCCC'); + [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); + + cccSortedandUnique = sortCCC(a,:); + bestAngles_tmp.(tomoList{iTomo}) = gather(cccSortedandUnique); + + % save doesn't work in a parfor, so write out the results for each tomogram so that a + % run may be resumed if cancelled. + angOut = fopen(sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}),'w'); + + for iRow = 1:size( bestAngles_tmp.(tomoList{iTomo}),1) + for iPeak = 1:emc.nPeaks + fprintf(angOut,'%d %d %6.3f %6.3f %6.3f %6.6f %6.6f %6.3f %6.3f %6.3f ', ... + bestAngles_tmp.(tomoList{iTomo})(iRow,1+10*(iPeak-1):10+10*(iPeak-1))); end - end % loop over subTomos - - - for iPeak = 1:nPeaks - - % Get rid of any zero entries left over from pre-initialization - if iPeak == 1 - nonZeroInits = ( cccStorageBest{iPeak}(:,2) ~= 0 ); - cccStorageBest{1}=cccStorageBest{1}(nonZeroInits,:); - sortCCC = zeros(size(cccStorageBest{1},1),10*nPeaks); - else - cccStorageBest{iPeak}=cccStorageBest{iPeak}(nonZeroInits,:); + fprintf(angOut,'\n'); end - - sortCCC(:,1+10*(iPeak-1):10+10*(iPeak-1)) = cccStorageBest{iPeak}; - end - -% % % % I think this is redundant now, but leaving until I double check. -% % % save('sortCCC.mat','sortCCC'); - [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); - - cccSortedandUnique = sortCCC(a,:); -% % % save('cccSortedandUnique.mat','cccSortedandUnique'); -% % % g = gather(geometry); -% % % save('TBL_geom.mat','g'); - - bestAngles_tmp.(tomoList{iTomo}) = gather(cccSortedandUnique); - - % save doesn't work in a parfor, so write out the results for each tomogram so that a - % run may be resumed if cancelled. - angOut = fopen(sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}),'w'); - - for iRow = 1:size( bestAngles_tmp.(tomoList{iTomo}),1) - for iPeak = 1:nPeaks - fprintf(angOut,'%d %d %6.3f %6.3f %6.3f %6.6f %6.6f %6.3f %6.3f %6.3f ', ... - bestAngles_tmp.(tomoList{iTomo})(iRow,1+10*(iPeak-1):10+10*(iPeak-1))); - end - fprintf(angOut,'\n'); - end - fclose(angOut); - - end % if clause to check for previous alignment + fclose(angOut); + + end % if clause to check for previous alignment end % loop over tomos bestAnglesResults{iParProc} = bestAngles_tmp; geometryResults{iParProc} = geometry_tmp; -%profile off -%profsave + %profile off + %profsave end % parfor if ( flgReverseOrder || flgStartThird ) - fprintf('This reverse run will not write the metaData\n'); + fprintf('This multi-node run will not write the metaData\n'); else - save('bestAnglesResults.mat', 'bestAnglesResults'); bestAngles = struct(); - for iParProc = 1:nParProcesses + for iParProc = 1:nParProcesses for iTomo = iterList{iParProc} geometry.(tomoList{iTomo}) = geometryResults{iParProc}.(tomoList{iTomo}); bestAngles.(tomoList{iTomo}) = bestAnglesResults{iParProc}.(tomoList{iTomo}); end end -% save('bestAnglesTemp.mat', 'bestAngles'); - save('bestAngles.mat', 'bestAngles'); - - [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, nPeaks, rotConvention, updateWeights, updateClassByBestReferenceScore); - masterTM.(cycleNumber).('RawAlign') = rawAlign; - masterTM.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); - masterTM.('updatedWeights') = true; - - clear bestAngles rawAlign - subTomoMeta = masterTM; - save(pBH.('subTomoMeta'), 'subTomoMeta'); - - + [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, emc.nPeaks, rotConvention, emc.update_class_by_ccc); + subTomoMeta.(cycleNumber).('RawAlign') = rawAlign; + subTomoMeta.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); + + clear bestAngles rawAlign + % Save using wrapper + BH_saveSubTomoMeta(emc.('subTomoMeta'), subTomoMeta); + end delete(gcp('nocreate')) diff --git a/alignment/BH_alignReferences3d.m b/alignment/BH_alignReferences3d.m deleted file mode 100755 index bdfb9287..00000000 --- a/alignment/BH_alignReferences3d.m +++ /dev/null @@ -1,759 +0,0 @@ -efunction [ ] = BH_alignReferences3d( PARAMETER_FILE, CYCLE) -%Align an symmetrize references -% - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -if (nargin ~= 2) - error('args = PARAMETER_FILE, CYCLE') -end -startTime = clock; -CYCLE = EMC_str2double(CYCLE); - -% not for normal use, but under certain circumstances allow override of 0.5 cutoff for -% ref and allow to 0.143 -if (CYCLE < 0) - CYCLE = abs(CYCLE); - flgAlignCutoff = 0; -else - flgAlignCutoff = 1; -end - -cycleNumber = sprintf('cycle%0.3u', CYCLE); - -pBH = BH_parseParameterFile(PARAMETER_FILE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - -maxGoldStandard = subTomoMeta.('maxGoldStandard'); -flgClassify = pBH.('flgClassify'); -try - flgMultiRefAlignment = pBH.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end - - -memoryLevel = pBH.('avgMemory'); - -% specify an angular shift to apply to the first reference, "" translational, -% identify the ref to align all other refs to, and whether to use bandpass from -% param (based on est from FSC on RawAlign average, or re-extract and calc FSC -% to design filter. The second option will only work in later iterations as the -% alignment of the raw images gets better, as symmetry will magnify small -% geometrical errors.) -bFactor = pBH.('Fsc_bfactor'); -samplingRate = pBH.('Ali_samplingRate'); -pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; -if pBH.('SuperResolution') - pixelSize = pixelSize * 2; -end -try - scaleCalcSize = pBH.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end -angleSearch = pBH.('Ref_angleSearch'); - -refName = pBH.('Ref_className'); - -peakSearch = floor(pBH.('particleRadius')./pixelSize); -peakCOM = [1,1,1].*peakCOM; -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); - - - -flgAngleShift{1}= pBH.('ref_AngleShift_odd'); -flgTransShift{1}= pBH.('ref_TransShift_odd'); -flgRefRef{1} = pBH.('ref_Ref_odd'); -refVectorFull{1}= pBH.('Ref_references_odd'); -features{1} = pBH.('Pca_coeffs_odd'); -% % % vol_geometry{1} = subTomoMeta.(cycleNumber).('ClusterResults').( ... -% % % sprintf('%s_%d_%d_nClass_%d_ODD', ... -% % % outputPrefix,features{1}(1,1), ... -% % % features{1}(1,end), refName)); - -flgAngleShift{2}= pBH.('ref_AngleShift_eve'); -flgTransShift{2}= pBH.('ref_TransShift_eve'); -flgRefRef{2} = pBH.('ref_Ref_eve'); -refVectorFull{2}= pBH.('Ref_references_eve'); -features{2} = pBH.('Pca_coeffs_eve'); -% % % vol_geometry{2} = subTomoMeta.(cycleNumber).('ClusterResults').( ... -% % % sprintf('%s_%d_%d_nClass_%d_EVE', ... -% % % outputPrefix,features{2}(1,1), ... -% % % features{2}(1,end), refName)); -% Merge alignments that could come from using different feature vectors in the -% classification, back into one metadata. -% % % vol_geometry = BH_mergeClassGeometry(vol_geometry{1},vol_geometry{2}); - % Copy to reset after initial extraction of primary references. -vol_geometry = subTomoMeta.(cycleNumber).('ClusterRefGeom'); - -vol_geometry_clean = vol_geometry; - - - - - - - - - -masterTM = subTomoMeta; clear subTomoMeta - - - -refVector = cell(2,1); -refGroup = cell(2,1); -refSym = cell(2,1); - -for iGold = 1:2 - refVectorFull{iGold} - % Sort low to high, because order is rearranged as such unstack - refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; - % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) - % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) - % axial symmetry to apply, negative value indicates creating a mirrored ref - % accros the corresponding axis - refSym{iGold} = refVectorFull{iGold}(2,:) -end - -% make sure the number of references match the unique groups in the classVector -% and also that the class/group pairs match the class/ref pairs. -nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]; - -uniqueSym = cell(2,1); -for iGold = 1:2 - [~,uniqueGroup,~] = unique(refGroup{iGold}); - uniqueSym{iGold} = refSym{iGold}(uniqueGroup) -end - - - - - -[ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Cls', samplingRate); - -[ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea(maskSize, maskRadius, scaleCalcSize) -padREF = [0,0,0;0,0,0]; - - -if any(peakSearch > maskRadius) - fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') - - peakSearch( (peakSearch > maskRadius) ) = ... - maskRadius( (peakSearch > maskRadius) ); -end - - - -% To properly average averages without re-extracting, the original number -% in each average must be taken in to account. - - -% Read in the references. -refIMG = cell(2,1); -imgCounts = cell(2,1); -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - - % To allow more flexibility ref/ cls field prefixes are used, and rather than - % making something special for the naming of cls/ref locations which SHOULD be - % interchangeable, just try one then the other since this impacts nothing - % else. Use the unweighted average for the first pass - - try - imgNAME = sprintf('class_%d_Locations_Ref_%s', refName, halfSet); - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - [ refIMG{iGold} ] = BH_unStackMontage4d(refVector{iGold}, ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - catch - imgNAME = sprintf('class_%d_Locations_Cls_%s', refName, halfSet); - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - [ refIMG{iGold} ] = BH_unStackMontage4d(refVector{iGold}, ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - end - - occCell = BH_multi_isCell( refIMG{iGold} ); - nRefs = length(occCell); - tIMG = cell(nRefs,1); - for iRef = 1:nRefs - tIMG{iRef} = refIMG{iGold}{occCell(iRef)}; - end - refIMG{iGold} = tIMG; clear tIMG - - sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}; - sizeREF = sizeREF(2:2:6)' - -% % % % clear out empty cell contents, and return -% % % n = 1 ; tIMG = cell(nReferences(iGold)); -% % % for iP = 1:length(refTMP) -% % % if ~isempty(refTMP{iP}) -% % % tIMG{n} = refTMP{iP}; -% % % -% % % n = n + 1; -% % % end -% % % end -% % % -% % % refIMG{iGold} = tIMG ; clear tIMG refTMP -% % % - - -end - -% % Prevent divergent orientations -% [ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, 40 ); - -% % % [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc, padREF ] = ... -% % % BH_multi_validArea( maskRadius, sizeREF ) - - -% optimize the fft for the given size. Padding to the next power of 2 is usually -% slower given the dimensionality of the volume data. -fftPlanner = rand(sizeCalc); -fftw('planner', 'exhaustive'); -fftn(fftPlanner); -clear fftPlanner - - -% Make a mask, and apply to the average motif && save a masked, binned copy of -% the average for inspection. -mask = struct(); - - - [ volMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); - mask.('volMask') = gather(volMask); - - [ peakMask ] = gpuArray(BH_mask3d(maskType, sizeMask, peakSearch, maskCenter)); - mask.('peakMask') = gather(peakMask); - - peakBinary = (peakMask >= 0.01); - -% The bandpass here is from the full average - assuming each class is a -% substantial portion of the total population. - - bandpassFilt = cell(1,1); - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'cpu', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - for iRef = 1:1 - fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); - - % The class averages have roughly the same SNR as the references so apply any - % bFactor to them as well. - [ bandpassFilt{iRef}, ~ ] = BH_multi_cRef( fscINFO, radialGrid , bFactor,flgAlignCutoff); - end - clear radialGrid - mask.('bandpassFilt') = (bandpassFilt); - bandpassFilt{1} = gpuArray(bandpassFilt{1}); -% [maskMontage,~] = BH_montage4d({gather(volMask), gather(peakMask)},''); -% SAVE_IMG(MRCImage(maskMontage), sprintf('%s_maskMontage.mrc', outputPrefix)); - -%Improve the estimated center for the reference used to align any other -% optional references or sub references, applying any symmetry as well. Then -% re-extract these refs with the new transformation. -estShifts = cell(2,1); - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - halfNUM = iGold; - - % The first class in the refRef vector is used to align the primary reference - % for each reference group, to which the members of each group are - % subsequently aligned. - refRefIDX = find(refVector{iGold} == flgRefRef{iGold}(1)); - refRef = refIMG{iGold}{refRefIDX}; - % Only apply trans shifts from estimate, then later use angles as well. - refRef = BH_axialSymmetry(refRef,1, 0,'GPU',flgTransShift{iGold}(1,:)); - - refRef = refRef(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3) ); - - refAsym= refIMG{iGold}{refRefIDX}(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3) ); - - - - refAsym= gpuArray(refAsym); - - - - - refAsym_FT = BH_bandLimitCenterNormalize(refAsym.*peakMask, bandpassFilt{1}, ... - peakBinary,padCalc,'double'); - refRef_FT =conj(BH_bandLimitCenterNormalize(refRef.*peakMask, bandpassFilt{1},... - peakBinary,padCalc,'double')); - - - [ estPeakCoord ] = BH_multi_xcf_Translational( refAsym_FT, refRef_FT, ... - peakMask, peakCOM); - - estShifts{iGold} = gather(estPeakCoord); - fprintf('estPeak at %f %f %f\n estShifts now %f %f %f \n', estPeakCoord', estShifts{iGold}'); - clear refRef refAsym refRef_FT refAsym_FT - - - refRef = BH_axialSymmetry(refIMG{iGold}{refRefIDX}, refSym{iGold}(refRefIDX),... - flgAngleShift{iGold}(1), ... - 'GPU',estShifts{iGold}); - refRefRotAvg = BH_axialSymmetry(refIMG{iGold}{refRefIDX}, 120,... - flgAngleShift{iGold}(1), ... - 'GPU',estShifts{iGold}); - - if isa(refRef,'cell') - refRef = refRef{2}; - end - - SAVE_IMG(MRCImage(gather(refRef)),sprintf('initialAxialOffsetCheck_%s.mrc',halfSet)); - refWDG = NaN; - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % % - % To allow more flexibility ref/ cls field prefixes are used, and rather than - % making something special for the naming of cls/ref locations which SHOULD be - % interchangeable, just try one then the other since this impacts nothing - % else. - - try - imgClassNAME = sprintf('class_%d_Locations_%s_%s', refName, 'Ref',halfSet); - - - [ classIMG ] = BH_unStackMontage4d(1:refName, ... - masterTM.(cycleNumber).(imgClassNAME){1},... - masterTM.(cycleNumber).(imgClassNAME){2},... - sizeWindow); - catch - imgClassNAME = sprintf('class_%d_Locations_%s_%s', refName, 'Cls',halfSet); - - - [ classIMG ] = BH_unStackMontage4d(1:refName, ... - masterTM.(cycleNumber).(imgClassNAME){1},... - masterTM.(cycleNumber).(imgClassNAME){2},... - sizeWindow); - end - - - - - %%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't - %%%%%%%%%%%%%%%%%%%%% search at all in that dimension. - - [ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch(1,:)) - - - - % First align the reference reference to any sub-references in its group. After - % finding the best alignment extract from the tomograms, with any symmetry. - - % Second align the first member of each ref group to the new reference, and use - % alignment to resample the head of each ref group. - - % Third align each sub-ref to the head of each ref group, and finally extract - % these. - - - -% % Store the cross correlation score, peak location, and wedge weight - bestAnglesTotal = zeros(nReferences(iGold),10); - nCount = 1; - - for iClass = flgRefRef{iGold}(1,:) - tic; - % Load the class onto gpu - iClassImg = gpuArray(classIMG{iClass}); - - iClassWdg = NaN; - - - - [ alignOUT ] = run_alignment(angleStep, inPlaneSearch, ... - iClassImg, iClassWdg, ... - refRef, refWDG, refRefRotAvg, ... - volMask, bandpassFilt, padCalc, padWindow, ... - peakMask, peakCOM, iClass, ... - uniqueSym{iGold}); - - - bestAnglesTotal(nCount,:) = [gather(alignOUT(1,1:10))]; - - timeClass = toc; - fprintf('finished working on %d/%d classes...%fs\n',nCount, ... - length(flgRefRef{iGold}),timeClass); - nCount = nCount + 1; - - end % loop over (classes) - - bestAnglesTotal = gather(bestAnglesTotal); - - - [ vol_geometry ] = BH_refAlignmentsApply( vol_geometry, bestAnglesTotal,... - samplingRate, ... - flgRefRef{iGold}(1,:), flgRefRef{iGold}(2,:),halfNUM ); - - - - -end - -masterTM.(cycleNumber).('RefAlignment') = vol_geometry; -subTomoMeta = masterTM; -save(pBH.('subTomoMeta'), 'subTomoMeta'); - - -gpuDevice(1); -BH_average3d(PARAMETER_FILE,num2str(CYCLE),'RefAlignment'); - -% average3d puts info about the ref locations in the montage, so reload the -% metadata - -load(pBH.('subTomoMeta')); -masterTM = subTomoMeta; - -% reload masks onto gpu -volMask = gpuArray(mask.('volMask')); -peakMask = gpuArray(mask.('peakMask')); -for iRef = 1:length(mask.('bandpassFilt')) - bandpassFilt{iRef} = gpuArray(mask.('bandpassFilt'){iRef}); -end - -clear refRef refRefRotAvg initPeakMask wdgIMG refIMG - - -% Reset the geometry so that the original classes persist. -vol_geometry = vol_geometry_clean; -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Now we have two montages of references (if gold standard) which later I will -% add in an fsc comparision, which will be used to derive an ideal bandpass -% filter. For now, continue on to use these to further align "sub references" -% using the supplied bandpass filter. -refIMG = cell(2,1); -imgCounts = cell(2,1); - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - halfNUM = iGold; - - % Read in the new refs we just created. - newRefNAME = sprintf('class_%d_Locations_%s_%s', refName, 'REF',halfSet); - - fprintf('reading in new ref %s.\n',halfSet); - [ refIMG{iGold} ] = BH_unStackMontage4d(1:nRefs, ... - masterTM.(cycleNumber).(newRefNAME){1},... - masterTM.(cycleNumber).(newRefNAME){2},... - sizeWindow); - - imgCounts{iGold} = masterTM.(cycleNumber).(newRefNAME){3}; - - occCell = BH_multi_isCell( refIMG{iGold} ); - nRefs = length(occCell); - tIMG = cell(nRefs,1); - for iRef = 1:nRefs - tIMG{iRef} = refIMG{iGold}{occCell(iRef)}; - end - refIMG{iGold} = tIMG; clear tIMG - - % Save the intermediate "primaryRef" since this will make trouble shooting - % errors in symmetry/grouping until explicit error checks can be added to - % BH_parseParameterFile - CMD = sprintf('mv %s_filtered%d_REF_%s.mrc %s_primaryRef_%d_%s.mrc ', ... - outputPrefix, refName, halfSet,outputPrefix, refName, halfSet); - system(CMD, '-echo'); -end - - -[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, maxGoldStandard ); - -[ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch(2,:)) - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - halfNUM = iGold; - - refImg = zeros([size(refIMG{iGold}{1}),nRefs],'single','gpuArray'); - refRotAvg = zeros([size(refIMG{iGold}{1}),nRefs], 'single','gpuArray'); - %refWdg = zeros([size(refIMG{iGold}{1}),nRefs],'single','gpuArray'); - refWdg = NaN - for iRef = 1:nRefs; - refRotAvg(:,:,:,iRef) = BH_axialSymmetry(refIMG{iGold}{iRef},120,0,'GPU',[0,0,0]); - refImg(:,:,:,iRef) = refIMG{iGold}{iRef}; - end - - - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - - % Store the cross correlation score, peak location, and wedge weight - bestAnglesTotal = zeros(nReferences(iGold),10); - nCount = 1; - - - imgClassNAME = sprintf('class_%d_Locations_%s_%s', refName, 'Ref',halfSet) - refName - - [ classIMG ] = BH_unStackMontage4d(1:refName, ... - masterTM.(cycleNumber).(imgClassNAME){1},... - masterTM.(cycleNumber).(imgClassNAME){2},... - sizeWindow); - - - - - refVector{iGold}(1,:) - for iClass = refVector{iGold}(1,:) - tic; - iClass - % Load the class onto gpu - iClassImg = gpuArray(classIMG{iClass}); - - iClassWdg = NaN; - - - - % only use the relevant class - iGroup = refGroup{iGold}(ismember(refVector{iGold},iClass)) - - - [ alignOUT ] = run_alignment(angleStep, inPlaneSearch, ... - iClassImg, iClassWdg, ... - refImg(:,:,:,iGroup), refWdg, ... - refRotAvg(:,:,:,iGroup), ... - volMask, bandpassFilt, padCalc, padWindow, ... - peakMask, peakCOM, iClass,uniqueSym{iGold}); - - - bestAnglesTotal(nCount,:) = [gather(iGroup),gather(alignOUT(1,2:10))]; - - timeClass = toc; - fprintf('finished working on %d/%d classes %s...%fs\n',nCount, ... - length(refVector{iGold}),halfSet,timeClass); - nCount = nCount + 1; - - end % loop over (classes) - - bestAnglesTotal = gather(bestAnglesTotal) - - - [ vol_geometry ] = BH_refAlignmentsApply( vol_geometry, bestAnglesTotal,... - samplingRate, ... - refVector{iGold}(1,:), refGroup{iGold}(1,:),halfNUM ); - - -end - - masterTM.(cycleNumber).('RefAlignment') = vol_geometry; - subTomoMeta = masterTM; - save(pBH.('subTomoMeta'), 'subTomoMeta'); - - BH_average3d(PARAMETER_FILE,num2str(CYCLE),'RefAlignment'); - - -fprintf('total execution time : %f seconds\n', etime(clock, startTime)); - - - -function [ alignOUT ] = run_alignment(angleStep, inPlaneSearch, ... - iClassImg, iClassWdg, ... - refRef, ref_WDG, refRefRotAvg, ... - volMask, bandpassFilt, padCalc, padWindow, ... - peakMask, peakCOM, iClass,uniqueSym) - - clear cccStorage1 cccStorage2 cccStorage3 cccStorage4 - - % Out of plane search - if any(angleStep(2:4)) - flgSearchDepth = 3; - - elseif any(angleStep(5)) - % in plane - flgSearchDepth = 2; - peakListTop10(angleStep(5).*nReferences,6) = gpuArray(0); - nPeak = 1; - for iRef = 1 - for iPsi = inPlaneSearch - peakListTop10(nPeak,1) = iRef; - nPeak = nPeak + 1; - end - end - else - error('specify at least an in plane search') - end - - if (flgSearchDepth == 3) - - [ cccStorage1 ] = BH_multi_angularSearch( angleStep, 0, 0, ... - iClassImg, iClassWdg, ... - refRefRotAvg, ref_WDG, ... - refRefRotAvg, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM, iClass, ... - uniqueSym); - - - cccStorage1(1:10,:) - - - % Second loop over top 10 peaks now using the non-rotationally averaged - % reference and including out of plane angles. - - % peakList is # rows = top peaks - % reference, phi, theta - % put zero peak at top of list - zeroPeak = sortrows(cccStorage1,[3, 4, 5]); - zeroPeak = [zeroPeak(1,:) ; cccStorage1 ]; - % if zero peak was already there, remove it so no duplicate - zeroPeak = unique(zeroPeak, 'stable', 'rows'); - peakListTop10 = [zeroPeak(1:10,1),zeroPeak(1:10,3:4),zeroPeak(1:10,8:10)] - end - - - [ cccStorage2 ] = BH_multi_angularSearch( angleStep, peakListTop10, ... - inPlaneSearch, ... - iClassImg, iClassWdg, ... - refRef, ref_WDG, ... - refRefRotAvg, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM, iClass, ... - uniqueSym); - - - cccStorage2 = unique(cccStorage2((cccStorage2(:,6) ~= 0),:), 'stable','rows'); - if size(cccStorage2, 1) > 9 - cccStorage2(1:10,:) - else - cccStorage2 - end - - % This is to save time assuming that we can get a good estimate of the - % particles shift by taking the average of the higher ranking alignments. In - % testing this was always within ~ half a pixel. If CCC scores are strangely - % low, suspect this as a break point. - - - % Get the top three peaks with unique phi, and theta - % Return [ref,phi,theta,psi,phistep,thetastep,psistep] - % Stable prevents any sorting - if (flgSearchDepth== 3) - [~,ia,~] = unique(cccStorage2(:,3:4),'stable' ,'rows'); - peakListTop3 = zeros(3,10); - else - [~,ia,~] = unique(cccStorage2(:,3:5),'stable' ,'rows'); - peakListTop3 = zeros(3,10); - end - - if numel(ia) >= 10 - TOP = 10; - else - TOP = numel(ia); - end - - - for top3 = 1:TOP - outOfPlaneAngle = cccStorage2(ia(top3),4); - angleIndex = find(angleStep(:,1)==outOfPlaneAngle,1,'first'); - if (flgSearchDepth == 3 ) - % Search around top 3 +/- 0.5 the original out of plane angular increment - peakListTop3(top3,:) = [cccStorage2(ia(top3),1), ... - cccStorage2(ia(top3),3:5),... - angleStep(angleIndex,3)./4,... - angleStep(angleIndex,4)./2,... - angleStep(angleIndex,5)./2, cccStorage2(ia(top3),8:10)]; - else - % Search around top 3 +/- 0.5 the original out of plane angular increment - peakListTop3(top3,:) = [cccStorage2(ia(top3),1), ... - cccStorage2(ia(top3),3:5),... - 0,... - 0,... - angleStep(1,5)./2, cccStorage2(ia(top3),8:10)]; - end - end - peakListTop3 - [ cccStorage3 ] = BH_multi_angularSearch( angleStep, peakListTop3, ... - 0, ... - iClassImg, iClassWdg, ... - refRef, ref_WDG, ... - refRefRotAvg, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM,iClass, ... - uniqueSym); - - - cccStorage3 = unique(cccStorage3((cccStorage3(:,6) ~= 0),:), 'stable','rows'); - if size(cccStorage3, 1) > 9 - cccStorage3(1:10,:) - else - cccStorage3 - end - - - if (flgSearchDepth == 3 ) - % Use previous increments/2 - - % Search around the top peak +/- 0.25 the orginal angular increment - topPeak = [cccStorage3(1,1), ... - cccStorage3(1,3:5), ... - cccStorage3(1,11:13)./2, ... - cccStorage3(1,8:10)] - else - topPeak = [cccStorage3(1,1), ... - cccStorage3(1,3:5), ... - 0, ... - 0,... - angleStep(1,5)./3, cccStorage3(1,8:10)] - end - - [ cccStorage4 ] = BH_multi_angularSearch( angleStep, topPeak, ... - 0, ... - iClassImg, iClassWdg, ... - refRef, ref_WDG, ... - refRefRotAvg, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM, iClass, ... - uniqueSym); - - alignOUT = cccStorage4(1,:) - - - -end - diff --git a/alignment/BH_fitBeads.m b/alignment/BH_fitBeads.m index 4d7e3057..62528ee8 100644 --- a/alignment/BH_fitBeads.m +++ b/alignment/BH_fitBeads.m @@ -10,7 +10,7 @@ g1 = 2.5; % number of stdDev the central gaussian should fall by the bead radius defined below %s1 = pixelRadius. Set in loop so fractions of pixel radius can be searched b/c beads are not uniform diameter (but right now only looking at 1) s2 = 1.25; % radius for the edge gaussian, I suspect this should be closer to 1 or even 0.5? -g2 = s2./2; % center the edge gaussian at pixel_radius + g2 +g2 = s2./2; % center the edge gaussian at pixel_radius + g2 g3 = 6; % give the central gaussian a more flat top to match what a bead looks like. Abs is taken so odd values okay show_ref_profile = false; @@ -20,8 +20,8 @@ bgVal = 0; edgeVal = 4; beadVal = -4; - -input_ts = gpuArray(getVolume(MRCImage(img_name))); + +input_ts = gpuArray(OPEN_IMG('single', img_name)); system(sprintf('model2point -contour %s %s.txt',input_name,input_name)); input_pos= gpuArray(load(sprintf('%s.txt',input_name))); output_pos = zeros(size(input_pos),'single'); @@ -32,22 +32,22 @@ firstLoop = false; KERNEL = EMC_gaussianKernel([1,3], 0.5, 'gpu', {}); - + for iPrj = 1:nPrjs img = input_ts(:,:,iPrj); - + lCoord = input_pos(:,4) == iPrj -1; - + [w] = BH_multi_gridCoordinates([d1,d2],'Cartesian','GPU',{'none'},1,0,1); img_derivative = real(ifftn(fftn(img).*w)); -% img_derivative = EMC_convn(single(img_derivative), KERNEL); - + % img_derivative = EMC_convn(single(img_derivative), KERNEL); + img_derivative = img_derivative - mean(img_derivative(:)); img_derivative = img_derivative ./ rms(img_derivative(:)); - + r = ceil(pixel_radius.*3); avg_size = 2.*r+1 .*[1,1]; x = input_pos(lCoord,2) - 0.5; % in imod, model point 7.5 is in the "middle" of pixel 7 in emClarity, a pixel is 6.5 - 7.5 with 7 in the middle @@ -56,8 +56,8 @@ yi = floor(y); xf = x - xi; yf = y - yi; - - + + % Note if you put this into parallel, this will not work. if (firstLoop) % Get an avgerage bead for this projection to determine the intensities in @@ -66,9 +66,9 @@ avg_bead = zeros(avg_size,'single','gpuArray'); n = 0; for i = 1:length(x) - % add edge checking - avg_bead = avg_bead + img_derivative(xi(i)-r:xi(i)+r,yi(i)-r:yi(i)+r); - n = n + 1; + % add edge checking + avg_bead = avg_bead + img_derivative(xi(i)-r:xi(i)+r,yi(i)-r:yi(i)+r); + n = n + 1; end avg_bead = avg_bead ./ n ; % Get the mean values for the three main pixel values @@ -76,157 +76,157 @@ meanVect = gFit.mu; [beadVal,beadCoord] = min(meanVect); [edgeVal,edgeCoord] = max(meanVect); - + switch (beadCoord+edgeCoord) case 3 bgVal = meanVect(3); case 4 bgVal = meanVect(2); - case 5 + case 5 bgVal = meanVect(1); otherwise error('failed to find the correct index for the background mean'); end firstLoop = false; end - - - + + + % The edge radius should be around 1 pix unless you blur the derivative % first. if (show_ref_profile) v = -6:0.01:6; figure, plot(v, beadVal.* exp(-abs((g1.*v./pixel_radius).^g3)) + .... - edgeVal.*(exp(-(g1.*(v-g2-pixel_radius)./s2).^2) + exp(-(g1.*(v+g2 + pixel_radius)./s2).^2)) + ... - bgVal); + edgeVal.*(exp(-(g1.*(v-g2-pixel_radius)./s2).^2) + exp(-(g1.*(v+g2 + pixel_radius)./s2).^2)) + ... + bgVal); end - - + + [t] = BH_multi_gridCoordinates(avg_size,'Cartesian','GPU',{'none'},0,1,1); - - - - - + + + + + nBeads = length(x); - nSkipped = 0; + nSkipped = 0; s1 = pixel_radius; ref = beadVal.* exp(-abs((g1.*t./s1).^g3)) + .... - edgeVal.*(exp(-(g1.*(t-g2-s1)./s2).^2)) + ... - bgVal; + edgeVal.*(exp(-(g1.*(t-g2-s1)./s2).^2)) + ... + bgVal; ref = gpuArray(ref); imgFT = fftn(img_derivative); global_ref = zeros([d1,d2],'single','gpuArray'); - - for i = 1:nBeads - xl = xi(i)-r; - xh = xi(i)+r; - yl = yi(i)-r; - yh = yi(i)+r; - - if (xl < 1 || yl < 1 || xh > d1 || yh > d2) + + for i = 1:nBeads + xl = xi(i)-r; + xh = xi(i)+r; + yl = yi(i)-r; + yh = yi(i)+r; + + if (xl < 1 || yl < 1 || xh > d1 || yh > d2) continue; - else - global_ref(xl:xh,yl:yh,1) = ref; - end + else + global_ref(xl:xh,yl:yh,1) = ref; + end end - + global_ref = fftshift(real(ifftn(conj(fftn(global_ref)).*imgFT))); - max_global_shift = 14; - global_tile = max_global_shift.*[2,2]+1; - pad_GF = BH_multi_padVal([d1,d2],global_tile); - global_ref = BH_padZeros3d(global_ref,'fwd',pad_GF,'GPU','single'); - - [~,maxCoord] = max(global_ref(:)); - [mi,mj] = ind2sub(global_tile,maxCoord); - global_shifts = [mi,mj] - (max_global_shift+1); - xi = xi + global_shifts(1); - yi = yi + global_shifts(2); - x = x + global_shifts(1); - y = y + global_shifts(2); + max_global_shift = 14; + global_tile = max_global_shift.*[2,2]+1; + pad_GF = BH_multi_padVal([d1,d2],global_tile); + global_ref = BH_padZeros3d(global_ref,'fwd',pad_GF,'GPU','single'); + + [~,maxCoord] = max(global_ref(:)); + [mi,mj] = ind2sub(global_tile,maxCoord); + global_shifts = [mi,mj] - (max_global_shift+1); + xi = xi + global_shifts(1); + yi = yi + global_shifts(2); + x = x + global_shifts(1); + y = y + global_shifts(2); mip = zeros([d1,d2],'single','gpuArray'); padVal = BH_multi_padVal(size(t),[d1,d2]); - + % Loop over references of different fractions of the particle radius. For % now, just using 1. for iRef = [0.80:0.05:1.2] s1 = pixel_radius * iRef; - + ref = beadVal.* exp(-abs((g1.*t./s1).^g3)) + .... - edgeVal.*(exp(-(g1.*(t-g2-s1)./s2).^2)) + ... - bgVal; + edgeVal.*(exp(-(g1.*(t-g2-s1)./s2).^2)) + ... + bgVal; ref = BH_padZeros3d(ref,'fwd',padVal,'GPU','single'); - + ccf = real(fftshift(ifftn(conj(fftn(ref)).*imgFT))); lMip = ccf > mip; mip(lMip) = ccf(lMip); end - + if (show_mip) figure, imshow3D(gather(real(mip))); end % Get the updated peak positions, with optional over-sampling of the bead % an option to pad zeros in half transforms would be nice here. Just use % native matlab FFT for now. - + r = ceil(pixel_radius.*1.0); rp = padBy.*(2.*r+1).*[1,1]; padVal = BH_multi_padVal( (2.*r+1).*[1,1] , rp ); - maskRadius = 0.5.*(1- 2.5./sampling).*r.*[1,1]; + maskRadius = 0.5.*(1- 2.5./sampling).*r.*[1,1]; peakMask = BH_mask3d('sphere',padBy.*(2.*r+1).*[1,1],maskRadius,[0,0],'2d'); xo = x; yo = y; [bx,by] = ndgrid(gpuArray(-r:r),gpuArray(-r:r)); - - ro = floor(rp/2) + 1; + + ro = floor(rp/2) + 1; for i = 1:nBeads - % add edge checking - - xl = xi(i)-r; - xh = xi(i)+r; - yl = yi(i)-r; - yh = yi(i)+r; - - if (xl < 1 || yl < 1 || xh > d1 || yh > d2) - fprintf('skipping bead %d/%d\n', i,nBeads); - nSkipped = nSkipped + 1; - xo(i) = x(i); - yo(i) = y(i); - else - ccf = mip(xl:xh,yl:yh); - ccf = fftshift(fftn(ccf)); - ccf = BH_padZeros3d(ccf,'fwd',padVal,'GPU','single'); - ccf = real(ifftn(ifftshift(ccf))).*peakMask; - - [~,maxCoord] = max(ccf(:)); - [mi,mj] = ind2sub(rp,maxCoord); - mi = (mi- ro(1))./ padBy; - mj = (mj- ro(2))./ padBy; -% ccf = log(ccf+1); -% % % % % comX = sum(sum(bx.*ccf))./sum(ccf(:)); -% % % % % comY = sum(sum(by.*ccf))./sum(ccf(:)); -% % % % % xo(i) = (comX + x(i)); -% % % % % yo(i) = (comY + y(i)); - - xo(i) = (mi - xf(i) + x(i)); - yo(i) = (mj - yf(i) + y(i)); - end + % add edge checking + + xl = xi(i)-r; + xh = xi(i)+r; + yl = yi(i)-r; + yh = yi(i)+r; + + if (xl < 1 || yl < 1 || xh > d1 || yh > d2) + fprintf('skipping bead %d/%d\n', i,nBeads); + nSkipped = nSkipped + 1; + xo(i) = x(i); + yo(i) = y(i); + else + ccf = mip(xl:xh,yl:yh); + ccf = fftshift(fftn(ccf)); + ccf = BH_padZeros3d(ccf,'fwd',padVal,'GPU','single'); + ccf = real(ifftn(ifftshift(ccf))).*peakMask; + + [~,maxCoord] = max(ccf(:)); + [mi,mj] = ind2sub(rp,maxCoord); + mi = (mi- ro(1))./ padBy; + mj = (mj- ro(2))./ padBy; + % ccf = log(ccf+1); + % % % % % comX = sum(sum(bx.*ccf))./sum(ccf(:)); + % % % % % comY = sum(sum(by.*ccf))./sum(ccf(:)); + % % % % % xo(i) = (comX + x(i)); + % % % % % yo(i) = (comY + y(i)); + + xo(i) = (mi - xf(i) + x(i)); + yo(i) = (mj - yf(i) + y(i)); + end end - + fprintf('Updated the fit for %d/%d beads\n', nBeads - nSkipped, nBeads); if (show_results) figure, imshow3D(gather(img_derivative)); hold on - plot(y,x,'ro','MarkerSize',7); + plot(y,x,'ro','MarkerSize',7); plot(yo,xo, 'b+','MarkerSize', 7); end % We need to add back the 0.5 for imod model coords output_pos(lCoord,2) = gather(xo + 0.5); output_pos(lCoord,3) = gather(yo + 0.5); output_pos(lCoord,[1,4]) = gather(input_pos(lCoord,[1,4])); - + end f = fopen(sprintf('%s.txt',output_name),'w'); diff --git a/alignment/BH_refine_on_beads.m b/alignment/BH_refine_on_beads.m index 4993deb4..ddbe70f0 100644 --- a/alignment/BH_refine_on_beads.m +++ b/alignment/BH_refine_on_beads.m @@ -34,51 +34,51 @@ % Step down in samping rate sampling_step=2; -for imageBinning = [15:-sampling_step:min_sampling_rate] +for imageBinning = [15:-sampling_step:min_sampling_rate] - - if first_run + + if first_run input_name=baseName; - output_name=sprintf('%s_%d',baseName,imageBinning); + output_name=sprintf('%s_%d',baseName,imageBinning); last_binning=imageBinning; first_run=false; xTiltOption = '-XAXISTILT 0.0 '; z_factor_file = ''; local_file = sprintf('-LOCALFILE %s.local ',input_name); else - input_name=sprintf('%s_%d',baseName,last_binning); - output_name=sprintf('%s_%d',baseName,imageBinning); + input_name=sprintf('%s_%d',baseName,last_binning); + output_name=sprintf('%s_%d',baseName,imageBinning); last_binning=imageBinning; xTiltOption = sprintf('-XTILTFILE %s.Xtlt ',input_name); z_factor_file = sprintf('-ZFACTORFILE %s.Zfactor ', input_name); if isempty(doLocal) local_file = ''; else - local_file = sprintf(' -LOCALFILE %s.local ' , input_name); + local_file = sprintf(' -LOCALFILE %s.local ' , input_name); end end - + % Create a new stack at each binning [ fail ] = system(sprintf( ... - ['newstack ', ... + ['newstack ', ... '-InputFile %s.fixed ', ... % Always the same input (the preprocessed stack) - '-OutputFile %s_3dfind.ali ', ... + '-OutputFile %s_3dfind.ali ', ... '-TransformFile %s.xf ', ... % Coming from the last iteration '-ImagesAreBinned 1.0 ', ... % Input image binning is always 1 - '-BinByFactor %d'],baseName,output_name,input_name,imageBinning)); - - if (fail) - error('failed in newstack') + '-BinByFactor %d'],baseName,output_name,input_name,imageBinning)); + + if (fail) + error('failed in newstack') end - + if ( n_findBeads3d > 0) n_findBeads3d = n_findBeads3d - 1; - + % The defaults are taken from etomo on whatever tilt I had developed % this on - may not be optimal. Log option can produce problems if not % using preprocessed stack. [ fail ] = system(sprintf([... - 'tilt ' ... + 'tilt ' ... '-InputProjections %s_3dfind.ali ' ... '-OutputFile %s_3dfind.rec ' ... '-TILTFILE %s.tlt ' ... @@ -94,15 +94,15 @@ '-SHIFT 0.0 0.0 ' ... '-PERPENDICULAR ' ... '-FULLIMAGE %d,%d %s %s'],output_name,output_name,... - input_name,imageBinning,thickness,... - xTiltOption,NX,NY,local_file,z_factor_file)); - - if (fail) - error('failed in creating a 3d to look for beads') + input_name,imageBinning,thickness,... + xTiltOption,NX,NY,local_file,z_factor_file)); + + if (fail) + error('failed in creating a 3d to look for beads') end - + [ fail ] = system(sprintf([... - 'findbeads3d ' ... + 'findbeads3d ' ... '-InputFile %s_3dfind.rec ' ... '-OutputFile %s_3dfind.mod ' ... '-BeadSize %f ' ... @@ -114,123 +114,123 @@ '-BinningOfVolume %d'], ... output_name,output_name,bead_size,imageBinning ... )); - - if (fail) - error('failed in findbeads3d') - end - % The output is a 3d model, with YZ flipped. Project it perpendicular - % into natural orientation. - [ fail ] = system(sprintf([... - 'tilt ' ... - '-InputProjections %s_3dfind.ali ' ... - '-OutputFile %s.erase ' ... - '-IMAGEBINNED %d ' ... - '-TILTFILE %s.tlt ' ... - '-THICKNESS %d ' ...ls - '-RADIAL 0.35,0.035 ' ... - '-FalloffIsTrueSigma 1 ' ... - '%s ' ... - '-UseGPU 0 ' ... - '-ActionIfGPUFails 1,2 ' ... - '-OFFSET 0.0 ' ... - '-SHIFT 0.0,0.0 ' ... - '-ProjectModel %s_3dfind.mod ' ... - '-FULLIMAGE %d,%d ' ... - '-PERPENDICULAR ' ... - '-MODE 2 %s %s'], output_name,output_name,imageBinning,... - input_name,thickness,... - xTiltOption,output_name,NX,NY,local_file,z_factor_file)); - - if (fail) - error('failed in tilt') - end - - % Fix anything missed in findbeads3d - [ fail ] = system(sprintf([... + if (fail) + error('failed in findbeads3d') + end + + % The output is a 3d model, with YZ flipped. Project it perpendicular + % into natural orientation. + [ fail ] = system(sprintf([... + 'tilt ' ... + '-InputProjections %s_3dfind.ali ' ... + '-OutputFile %s.erase ' ... + '-IMAGEBINNED %d ' ... + '-TILTFILE %s.tlt ' ... + '-THICKNESS %d ' ...ls + '-RADIAL 0.35,0.035 ' ... + '-FalloffIsTrueSigma 1 ' ... + '%s ' ... + '-UseGPU 0 ' ... + '-ActionIfGPUFails 1,2 ' ... + '-OFFSET 0.0 ' ... + '-SHIFT 0.0,0.0 ' ... + '-ProjectModel %s_3dfind.mod ' ... + '-FULLIMAGE %d,%d ' ... + '-PERPENDICULAR ' ... + '-MODE 2 %s %s'], output_name,output_name,imageBinning,... + input_name,thickness,... + xTiltOption,output_name,NX,NY,local_file,z_factor_file)); + + if (fail) + error('failed in tilt') + end + + % Fix anything missed in findbeads3d + [ fail ] = system(sprintf([... 'beadtrack ' ... - '-InputSeedModel %s.erase ' ... - '-OutputModel %s_beadtrack.fid ' ... - '-ImageFile %s_3dfind.ali ' ... - '-ImagesAreBinned %d ' ... - '-PixelSize %f ' ... - '-BeadDiameter %f ' ... - '-RoundsOfTracking 3 ' ... - '-BoxSizeXandY %d,%d ' ... - '-MinBeadsInArea 3 ' ... - '-MinOverlapBeads 1 ' ... - '-UnsplitFirstRound ' ... - '-LocalAreaTracking ' ... - '-LocalAreaTargetSize %d ' ... - '-RotationAngle 0.0 ' ... - '-TiltFile %s.tlt ' ... - '-TiltDefaultGrouping 5 ' ... - '-MagDefaultGrouping 1 ' ... - '-RotDefaultGrouping 1 ' ... - '-LowPassCutoffInverseNm 0.71 '], output_name,output_name,output_name,imageBinning,... - pixelSizeInNanometers,bead_size,ptSizeX,ptSizeY,floor(512/imageBinning),... - input_name ... - )); - - if (fail) - error('Failed in beadtrack') - end + '-InputSeedModel %s.erase ' ... + '-OutputModel %s_beadtrack.fid ' ... + '-ImageFile %s_3dfind.ali ' ... + '-ImagesAreBinned %d ' ... + '-PixelSize %f ' ... + '-BeadDiameter %f ' ... + '-RoundsOfTracking 3 ' ... + '-BoxSizeXandY %d,%d ' ... + '-MinBeadsInArea 3 ' ... + '-MinOverlapBeads 1 ' ... + '-UnsplitFirstRound ' ... + '-LocalAreaTracking ' ... + '-LocalAreaTargetSize %d ' ... + '-RotationAngle 0.0 ' ... + '-TiltFile %s.tlt ' ... + '-TiltDefaultGrouping 5 ' ... + '-MagDefaultGrouping 1 ' ... + '-RotDefaultGrouping 1 ' ... + '-LowPassCutoffInverseNm 0.71 '], output_name,output_name,output_name,imageBinning,... + pixelSizeInNanometers,bead_size,ptSizeX,ptSizeY,floor(512/imageBinning),... + input_name ... + )); + + if (fail) + error('Failed in beadtrack') + end else [fail] = system(sprintf('imodtrans -i %s_3dfind.ali -2 %s.tltxf_Scaled %s_fitbyResid_%d.fid %s_beadtrack.fid', ... - output_name, input_name, input_name, n_ali_loops, output_name)); - - if (fail) - error('failed in projected model from previous tiltalign') + output_name, input_name, input_name, n_ali_loops, output_name)); + + if (fail) + error('failed in projected model from previous tiltalign') end - - end - + end + + for aliLoop = 1:n_ali_loops if aliLoop > 1 sprintf('MOVE BY RESIDUAL\n'); % use the residual from the previous, which has a header line -% cmd = sprintf('awk ''FNR==NR{if(FNR>1) {a[FNR-1]=$1-$4 FS $2-$5};next}{ print $1, a[FNR+1],$4}'' %s.resid_%d %s_beadtrack.fid.txt > %s_byResid_%d.txt',output_name,aliLoop-1,output_name,output_name,aliLoop); + % cmd = sprintf('awk ''FNR==NR{if(FNR>1) {a[FNR-1]=$1-$4 FS $2-$5};next}{ print $1, a[FNR+1],$4}'' %s.resid_%d %s_beadtrack.fid.txt > %s_byResid_%d.txt',output_name,aliLoop-1,output_name,output_name,aliLoop); cmd = sprintf('awk ''FNR==NR{a[FNR]=$1;next}{ if(FNR>1) print a[FNR-1],$1+$4 FS $2+$5 FS $3}'' %s_fit.fid.txt %s.resid_%d > %s_byResid_%d.txt',output_name,output_name,aliLoop-1,output_name,aliLoop); - + system(cmd); system(sprintf('point2model -circle %d %s_byResid_%d.txt %s_byResid_%d.fid',floor(bead_size/imageBinning),output_name,aliLoop,output_name,aliLoop)); model_to_align = sprintf('%s_fitbyResid_%d.fid',output_name,aliLoop); [~, nBeads] = BH_fitBeads(pixelSize,bead_diameter,imageBinning,... - sprintf('%s_3dfind.ali',output_name),... - sprintf('%s_byResid_%d.fid',output_name,aliLoop),... - model_to_align... - ); + sprintf('%s_3dfind.ali',output_name),... + sprintf('%s_byResid_%d.fid',output_name,aliLoop),... + model_to_align... + ); else model_to_align = sprintf('%s_fit.fid',output_name); [~, nBeads] = BH_fitBeads(pixelSize,bead_diameter,imageBinning,... - sprintf('%s_3dfind.ali',output_name),... - sprintf('%s_beadtrack.fid',output_name),... - model_to_align... - ); + sprintf('%s_3dfind.ali',output_name),... + sprintf('%s_beadtrack.fid',output_name),... + model_to_align... + ); end - + if (nBeads < 5) to_few_beads = true; return; end - + if (nBeads < 11) doLocal = ''; else doLocal = '-LocalAlignments '; end - - + + % No need to run tiltalign on the last iteration, b/c we use the 2d % model file for the input to the next iter. if (aliLoop < n_ali_loops) - + [ fail ] = system(sprintf([... - 'tiltalign ' ... + 'tiltalign ' ... '-ModelFile %s ' ... '-ImageFile %s_3dfind.ali ' ... '-ImagesAreBinned %d ' ... @@ -281,32 +281,32 @@ output_name,output_name,aliLoop ,output_name,output_name,output_name,output_name,... output_name,input_name,x_tilt_option(1:2),x_stretch_option(1:2),output_name,ptSizeX,ptSizeY,... x_tilt_option(3:4), x_stretch_option(3:4),doLocal,output_name)); - - if (fail) - error('failed in tiltalign') + + if (fail) + error('failed in tiltalign') end end % If cond on tiltAlign - + end % Transforms to full sampling (bin 1) [ fail ] = system(sprintf('xfproduct -scale 1,%d %s.xf %s.tltxf_nonScaled %s.xf', imageBinning,input_name,output_name,output_name)); - if (fail) - error('failed in xfproduct to full sampling') + if (fail) + error('failed in xfproduct to full sampling') end % imodtrans first scales by relative image size, then applies a % transformation as is, so we need to have the incremental transform at % the sampling rate of the next iteration as well. system(sprintf('awk ''{print 1.0, 0.0, 0.0, 1.0, 0.0, 0.0}'' %s.tltxf_nonScaled > dummy.xf',output_name)); [ fail ] = system(sprintf('xfproduct -scale 1,%f dummy.xf %s.tltxf_nonScaled %s.tltxf_Scaled', imageBinning/(imageBinning - sampling_step),output_name,output_name)); - - if (fail) - error('failed in xfproduct to incremental sampling') - end - - - + if (fail) + error('failed in xfproduct to incremental sampling') + end + - end + + + +end end diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index 5577d546..c42d714c 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -3,104 +3,104 @@ % sadf % TODO add options for experimenting. -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); skip_tilts = 0; if nargin > 6 - skip_tilts = EMC_str2double(varargin{1}); + skip_tilts = EMC_str2double(varargin{1}); end -pixelSize = pBH.('PIXEL_SIZE').*10^10; imgRotation = EMC_str2double(imgRotation); -try - RESOLUTION_CUTOFF = pBH.('autoAli_max_resolution'); +try + RESOLUTION_CUTOFF = emc.('autoAli_max_resolution'); catch RESOLUTION_CUTOFF=18; end % Min and max sampling rate in Ang/Pix (for patch tracking) try - MIN_SAMPLING_RATE = pBH.('autoAli_min_sampling_rate'); + MIN_SAMPLING_RATE = emc.('autoAli_min_sampling_rate'); catch MIN_SAMPLING_RATE = 10.0; end try - MAX_SAMPLING_RATE = pBH.('autoAli_max_sampling_rate'); + MAX_SAMPLING_RATE = emc.('autoAli_max_sampling_rate'); catch MAX_SAMPLING_RATE = 4.0; end try - PATCH_SIZE_FACTOR = pBH.('autoAli_patch_size_factor'); + PATCH_SIZE_FACTOR = emc.('autoAli_patch_size_factor'); catch PATCH_SIZE_FACTOR = 4; end % Check this first to allow only patch tracking even if there are beads try - REFINE_ON_BEADS = pBH.('autoAli_refine_on_beads'); + REFINE_ON_BEADS = emc.('autoAli_refine_on_beads'); catch REFINE_ON_BEADS = false; end % Check this first to allow only patch tracking even if there are beads try - BORDER_SIZE_PIXELS = pBH.('autoAli_patch_tracking_border'); + BORDER_SIZE_PIXELS = emc.('autoAli_patch_tracking_border'); catch BORDER_SIZE_PIXELS = 64; end % Check this first to allow only patch tracking even if there are beads try - N_ITERS_NO_ROT = pBH.('autoAli_n_iters_no_rotation'); + N_ITERS_NO_ROT = emc.('autoAli_n_iters_no_rotation'); catch N_ITERS_NO_ROT = 3; end try - PATCH_OVERLAP = pBH.('autoAli_patch_overlap'); + PATCH_OVERLAP = emc.('autoAli_patch_overlap'); catch PATCH_OVERLAP = 0.5; end try - ITERATIONS_PER_BIN = pBH.('autoAli_iterations_per_bin'); + ITERATIONS_PER_BIN = emc.('autoAli_iterations_per_bin'); catch ITERATIONS_PER_BIN = 3; end % FIXME this should probably be specified in Ang try - FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(pBH.('autoAli_max_shift_in_angstroms')./pixelSize); + FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(emc.('autoAli_max_shift_in_angstroms')./emc.pixel_size_angstroms); catch - FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(40 ./ pixelSize); + FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(40 ./ emc.pixel_size_angstroms); end try - DIVIDE_SHIFT_LIMIT_BY = pBH.('autoAli_max_shift_factor'); + DIVIDE_SHIFT_LIMIT_BY = emc.('autoAli_max_shift_factor'); catch DIVIDE_SHIFT_LIMIT_BY = 1; % int(max_shift / (iter^DIVI...)) + 1 end - + % Now get the bead diameter, if it is zeros override the default to refine % on beads after patch tracking. -beadDiameter = pBH.('beadDiameter') * 10^10; +beadDiameter = emc.('beadDiameter') * 10^10; if beadDiameter == 0 REFINE_ON_BEADS = false; end - LOW_RES_CUTOFF=800; - CLEAN_UP_RESULTS=false; - MAG_OPTION=5; - tiltAngleOffset=0.0; - TILT_OPTION = 0; +LOW_RES_CUTOFF=800; +CLEAN_UP_RESULTS=false; +MAG_OPTION=5; +tiltAngleOffset=0.0; +TILT_OPTION = 0; +fprintf("Stack in is %s\n",stackIN); inputMRC = MRCImage(stackIN,0); -inputStack = single(getVolume(inputMRC)); +inputStack = OPEN_IMG('single', inputMRC); skip_tilts_logical = []; if (skip_tilts) @@ -126,28 +126,28 @@ % the rotate to avoid information loss branch iHeader = getHeader(inputMRC); iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... - iHeader.cellDimensionY/iHeader.nY, ... - iHeader.cellDimensionZ/iHeader.nZ]; + iHeader.cellDimensionY/iHeader.nY, ... + iHeader.cellDimensionZ/iHeader.nZ]; iOriginHeader= [iHeader.xOrigin , ... - iHeader.yOrigin , ... - iHeader.zOrigin ]; - + iHeader.yOrigin , ... + iHeader.zOrigin ]; + f = load(sprintf('../%s',tiltAngles)); f = f(skip_tilts_logical); fout = fopen(sprintf('%s.rawtlt',baseName),'w'); fprintf(fout,'%f\n',f'); -fclose(fout); +fclose(fout); clear f cd('../'); -binHigh=ceil(MIN_SAMPLING_RATE ./ pixelSize); +binHigh=ceil(MIN_SAMPLING_RATE ./ emc.pixel_size_angstroms); if MAX_SAMPLING_RATE > 4 binLow = MAX_SAMPLING_RATE; else - binLow = ceil(MAX_SAMPLING_RATE ./ pixelSize); + binLow = ceil(MAX_SAMPLING_RATE ./ emc.pixel_size_angstroms); end binInc = -1*ceil((binHigh- binLow)./3); @@ -163,34 +163,39 @@ % switch_axes = false; % abs(abs(imgRotation) - 180) -a = ones(nX,nY,'single','gpuArray'); -p = BH_multi_padVal([nX,nY],max([nX,nY]).*[2,2]); -pad = BH_padZeros3d(a,'fwd',p,'GPU','single'); +if (emc.autoAli_switchAxes) + a = ones(nX,nY,'single','gpuArray'); + p = BH_multi_padVal([nX,nY],max([nX,nY]).*[2,2]); + pad = BH_padZeros3d(a,'fwd',p,'GPU','single'); -b = BH_resample2d(pad,[imgRotation,0,0],[0,0],'Bah','GPU','inv',1,size(pad)); -s = pad+b; -score_1 = sum(sum(s==2))./sum(b(:)); + b = BH_resample2d(pad,[imgRotation,0,0],[0,0],'Bah','GPU','inv',1,size(pad)); + s = pad+b; + score_1 = sum(sum(s==2))./sum(b(:)); -pad = rot90(pad); -b = BH_resample2d(pad,[90-imgRotation,0,0],[0,0],'Bah','GPU','forward',1,size(pad)); -s = pad+b; -score_2 = sum(sum(s==2))./sum(b(:)); + pad = rot90(pad); + b = BH_resample2d(pad,[90-imgRotation,0,0],[0,0],'Bah','GPU','forward',1,size(pad)); + s = pad+b; + score_2 = sum(sum(s==2))./sum(b(:)); -if score_2 > score_1 - switch_axes = true; + if score_2 > score_1 + switch_axes = true; + else + switch_axes = false; + end + clear a b s p pad else switch_axes = false; end if (switch_axes) - -% if ( abs(abs(imgRotation) - 180) > maxAngle ) + + % if ( abs(abs(imgRotation) - 180) > maxAngle ) fprintf('Your image rotation will result in a loss of data. Switching X/Y axes\n') -% switch_axes = true; + % switch_axes = true; rotStack = zeros(nY,nX,nZ,'single'); -% + % ny = nY; nY = nX; nX = ny; @@ -199,17 +204,17 @@ % Once we've done this, we want to work as if this is how the stack % came off the scope. system(sprintf('newstack -fromone -secs %d -rotate 90 fixedStacks/%s.fixed %s >/dev/null',iPrj,baseName,tmpFile)); - rotStack(:,:,iPrj) = getVolume(MRCImage(sprintf('%s',tmpFile))); - + rotStack(:,:,iPrj) = OPEN_IMG('single', sprintf('%s',tmpFile)); + system(sprintf('rm %s',tmpFile)); end inputStack = rotStack; clear rotStack - imgRotation = imgRotation + 90; - SAVE_IMG(inputStack,sprintf('fixedStacks/%s.fixed',baseName),iPixelHeader,iOriginHeader); + imgRotation = imgRotation + 90; + SAVE_IMG(inputStack,{sprintf('fixedStacks/%s.fixed',baseName),'half'},iPixelHeader,iOriginHeader); elseif ( skip_tilts) % Originally saved in the skip_tilts block, but that is redundant if we % save in the switch_axes block in the new implementation. - SAVE_IMG(inputStack,sprintf('fixedStacks/%s.fixed',baseName),iPixelHeader,iOriginHeader); + SAVE_IMG(inputStack,{sprintf('fixedStacks/%s.fixed',baseName),'half'},iPixelHeader,iOriginHeader); else % No modifications, so just link to the original stack cd('fixedStacks'); @@ -222,28 +227,29 @@ fprintf('Preprocessing tilt-series\n'); -%gradientAliasFilter = BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',pixelSize); - gradientAliasFilter = {BH_bandpass3d(1.*[nX,nY,1],0,0,0,'GPU','nyquistHigh'),... - BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',pixelSize)}; -if pixelSize < 2 +%gradientAliasFilter = BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',emc.pixel_size_angstroms); +gradientAliasFilter = {BH_bandpass3d(1.*[nX,nY,1],0,0,0,'GPU','nyquistHigh'),... + BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',emc.pixel_size_angstroms)}; +if emc.pixel_size_angstroms < 2 medianFilter = 5; else medianFilter = 3; end for iPrj = 1:nZ -% tmpPrj = BH_preProcessStack(gpuArray(inputStack(:,:,iPrj)),gradientAliasFilter,medianFilter); + % tmpPrj = BH_preProcessStack(gpuArray(inputStack(:,:,iPrj)),gradientAliasFilter,medianFilter); tmpPrj = real(ifftn(fftn(gpuArray(inputStack(:,:,iPrj))).*gradientAliasFilter{1})); tmpPrj = medfilt2(tmpPrj,medianFilter.*[1,1]); tmpPrj = real(ifftn(fftn(tmpPrj).*gradientAliasFilter{2})); -% tmpPrj = BH_resample2d(tmpPrj,rotMat,[0,0],'Bah','GPU','inv',1,size(tmpPrj)); + % tmpPrj = BH_resample2d(tmpPrj,rotMat,[0,0],'Bah','GPU','inv',1,size(tmpPrj)); inputStack(:,:,iPrj) = gather(tmpPrj); end -SAVE_IMG(inputStack,fixedName,pixelSize); + +SAVE_IMG(inputStack,{fixedName,'half'},emc.pixel_size_angstroms); fprintf('finished preprocessing tilt-series\n'); -clear tmpPrj inputStack +clear tmpPrj inputStack gradientAliasFilter cd(wrkDir) @@ -253,30 +259,30 @@ fclose(rotFile); - - + + system('pwd') fprintf('Running %s\n',runPath); system(sprintf('%s %s %f %f %d %d %d %d %d %d %s %d %d %d %f %f %f %d %d %d > ./emC_autoAliLog_%s.txt',... - runPath, ... - baseName, ... - pixelSize, ... - imgRotation, ... - binHigh, ... - binLow, ... - binInc,... - nX,nY,nZ,ext,... - PATCH_SIZE_FACTOR,... - N_ITERS_NO_ROT,... - BORDER_SIZE_PIXELS,... - PATCH_OVERLAP,... - RESOLUTION_CUTOFF,... - LOW_RES_CUTOFF,... - ITERATIONS_PER_BIN,... - FIRST_ITER_SHIFT_LIMIT_PIXELS,... - DIVIDE_SHIFT_LIMIT_BY,... - baseName)); - + runPath, ... + baseName, ... + emc.pixel_size_angstroms, ... + imgRotation, ... + binHigh, ... + binLow, ... + binInc,... + nX,nY,nZ,ext,... + PATCH_SIZE_FACTOR,... + N_ITERS_NO_ROT,... + BORDER_SIZE_PIXELS,... + PATCH_OVERLAP,... + RESOLUTION_CUTOFF,... + LOW_RES_CUTOFF,... + ITERATIONS_PER_BIN,... + FIRST_ITER_SHIFT_LIMIT_PIXELS,... + DIVIDE_SHIFT_LIMIT_BY,... + baseName)); + cd(sprintf('%s',startDir)); % 2021-May-08 BAH, not needed b/c fixed/name.fixed should remain rotated by @@ -290,7 +296,7 @@ if strcmpi(TILT_OPTION,'0') % If not fitting tilt angles we need a copy of them with .tlt - system(sprintf('cp fixedStacks/%s.rawtlt fixedStacks/%s.tlt',baseName,baseName)); + system(sprintf('cp fixedStacks/%s.rawtlt fixedStacks/%s.tlt',baseName,baseName)); end to_few_beads = false; @@ -300,32 +306,32 @@ extList = {'tlt','xf','local'}; % stack is skipped in second round. leave as number 1 for iExt = 1:length(extList) system(sprintf('ln -sf ../fixedStacks/%s.%s %s.%s', ... - baseName,extList{iExt},baseName,extList{iExt})); - end - + baseName,extList{iExt},baseName,extList{iExt})); + end + system(sprintf('ln -sf ../fixedStacks/%s.%s.preprocessed %s.%s', ... - baseName,'fixed',baseName,'fixed')); - + baseName,'fixed',baseName,'fixed')); + % Stopping for now at a bin5, this should be dynamic along with a handful % of other options. min_sampling_rate = 5; - [ to_few_beads ] = BH_refine_on_beads(baseName,nX,nY,3000,pixelSize,1.05.*100, min_sampling_rate); + [ to_few_beads ] = BH_refine_on_beads(baseName,nX,nY,3000,emc.pixel_size_angstroms,1.05.*100, min_sampling_rate); if (to_few_beads) fprintf('\nWARNING: to few beads found. Using iterative patch tracking results\n'); else for iExt = 1:length(extList) system(sprintf('mv ../fixedStacks/%s.%s ../fixedStacks/%s.%s_patchTracking', ... - baseName,extList{iExt},baseName,extList{iExt})); + baseName,extList{iExt},baseName,extList{iExt})); system(sprintf('cp %s_%d.%s ../fixedStacks/%s.%s', ... - baseName,min_sampling_rate,extList{iExt},baseName,extList{iExt})); - end - - + baseName,min_sampling_rate,extList{iExt},baseName,extList{iExt})); + end + + system(sprintf('imodtrans -i ../fixedStacks/%s.fixed %s_%d_fit.fid ../fixedStacks/%s.erase',... - baseName,baseName,min_sampling_rate,baseName)); - + baseName,baseName,min_sampling_rate,baseName)); + system(sprintf('newstack -xf ../fixedStacks/%s.xf -bin 12 ../fixedStacks/%s.fixed ../fixedStacks/%s_bin12.ali',baseName,baseName,baseName)); end @@ -335,8 +341,8 @@ if (to_few_beads || ~REFINE_ON_BEADS) cd fixedStacks system(sprintf('%s %s %d %d %d %d', findBeadsPath, baseName,... - nX,nY,3000,... - ceil(1.05*100/pixelSize))); + nX,nY,3000,... + ceil(1.05*100/emc.pixel_size_angstroms))); cd .. end diff --git a/alignment/BH_templateSearch3d.m b/alignment/BH_templateSearch3d.m deleted file mode 100755 index e64d1cda..00000000 --- a/alignment/BH_templateSearch3d.m +++ /dev/null @@ -1,1307 +0,0 @@ -function [hAvg, hRms, avgRange, rmsRange] = BH_templateSearch3d( PARAMETER_FILE,... - tomoName,tomoNumber,TEMPLATE, ... - SYMMETRY, wedgeType, varargin) - - -%3d template matching - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -precision = 'single'; -precisionTaper = 'singleTaper'; - -if length(varargin) == 1 - % Allow for an override of the max number, useful when only a few tomos - % have a strong feature like carbon that is hard to avoid. - cmdLineThresh = 0; - gpuIDX = EMC_str2double(varargin{1}); -elseif length(varargin) == 2 - cmdLineThresh = EMC_str2double(varargin{1}); - gpuIDX = EMC_str2double(varargin{2}); -end - tomoNumber = EMC_str2double(tomoNumber); - - - [ useGPU ] = BH_multi_checkGPU( gpuIDX ) - - - - -gpuDevice(useGPU); - -SYMMETRY = EMC_str2double(SYMMETRY); -startTime = clock ; - -pBH = BH_parseParameterFile(PARAMETER_FILE); -try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR -% clear subTomoMeta - % Make sure we get a CTF corrected stack - shouldBeCTF = 1 -catch - mapBackIter = 0; - shouldBeCTF = -1 -end -samplingRate = pBH.('Tmp_samplingRate'); - -try - tmpDecoy = pBH.('templateDecoy') -catch - tmpDecoy = 0 -end - -try - scale_mip = pBH.('scale_mip'); -catch - scale_mip = false; -end - -try - max_tries = pBH.('max_peaks'); -catch - max_tries = 10000; -end - -if ( cmdLineThresh ) - peakThreshold = cmdLineThresh; - fprintf('\nOverride peakThreshold from paramfile (%d) with cmd line arg (%d)\n\n',... - cmdLineThresh, pBH.('Tmp_threshold')); -else - peakThreshold = pBH.('Tmp_threshold'); -end - -latticeRadius = pBH.('particleRadius'); -try - targetSize = pBH.('Tmp_targetSize') -catch - targetSize = [512,512,512]; -end -angleSearch = pBH.('Tmp_angleSearch'); - -statsRadius = 1; - -convTMPNAME = sprintf('convmap_wedgeType_%d_bin%d',wedgeType,samplingRate) - -try - eraseMaskType = pBH.('Peak_mType'); -catch - eraseMaskType = 'sphere'; -end -try - eraseMaskRadius = pBH.('Peak_mRadius'); -catch - eraseMaskRadius = 0.75.*latticeRadius; -end - - -nPreviousSubTomos = 0; - -reconScaling = 1; -try - nPeaks = pBH.('nPeaks'); -catch - nPeaks = 1; -end - -pixelSizeFULL = pBH.('PIXEL_SIZE').*10^10; -if pBH.('SuperResolution') - pixelSizeFULL = pixelSizeFULL * 2; -end - -pixelSize = pixelSizeFULL.*samplingRate; - - - - -try - wantedCut = pBH.('lowResCut'); -catch - wantedCut = 28; -end - -TLT = load(sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tomoName,mapBackIter+1)); -def = mean(-1.*TLT(:,15))*10^6; %TODO if you switch to POSITIVEDEFOCUS this will be wrong -firstZero = -0.2*def^2 +5.2*def +11; - -% Take the lower of firstZero lowResCut or Nyquist -lowResCut = max(wantedCut, firstZero); - -if pixelSize*2 > lowResCut - fprintf('\nLimiting to Nyquist (%f) instead of user requested lowResCut %f Angstrom\n',pixelSize*2,lowResCut); - lowResCut = pixelSize*2; -else - fprintf('\nUsing max (%f) of specified resolution cutoff of %f and first ctf zero %f Angstrom\n',lowResCut, wantedCut, firstZero); -end - - - -mapPath = './cache'; -mapName = sprintf('%s_%d_bin%d',tomoName,tomoNumber,samplingRate); -mapExt = '.rec'; - -sprintf('recon/%s_recon.coords',tomoName) -[ recGeom, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName) ); - -reconCoords = recGeom(tomoNumber,:); -clear recGeom - - -[ tomogram ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... - reconCoords, mapBackIter, samplingRate,... - shouldBeCTF*gpuIDX, reconScaling,1); - - -% We'll handle image statistics locally, but first place the global environment -% into a predictible range - - - -[template, tempPath, tempName, tempExt] = ... - BH_multi_loadOrBin( TEMPLATE, 1, 3 ); - - - - - -% The template will be padded later, trim for now to minimum so excess -% iterations can be avoided. -fprintf('size of provided template %d %d %d\n',size(template)); -trimTemp = BH_multi_padVal(size(template),ceil(2.*max(pBH.('Ali_mRadius')./pixelSizeFULL))); -template = BH_padZeros3d(template, trimTemp(1,:),trimTemp(2,:),'cpu','singleTaper'); -SAVE_IMG(MRCImage(template),'template_trimmed.mrc'); -clear trimTemp -fprintf('size after trim to sqrt(2)*max(lattice radius) %d %d %d\n',size(template)); - -if isempty(mapPath) ; mapPath = '.' ; end -if isempty(tempPath) ; tempPath = '.' ; end -% Check to see if only tilt angles are supplied, implying a y-axis tilt scheme, -% or otherwise, assume a general geometry as in protomo. -% % % tiltGeometry = load(RAWTLT); -RAWTLT = sprintf('fixedStacks/ctf/%s_ali1_ctf.tlt',tomoName); -tiltGeometry = load(RAWTLT); -% subTomoMeta.('tiltGeometry').(mapName) = tiltGeometry; - -% Make sure the template and is an even sized image -template = padarray(template, mod(size(template),2),0, 'post'); -template = template - mean(template(:)); - -templateBIN = BH_reScale3d(template,'',sprintf('%f',1/samplingRate),'cpu'); -templateBIN = templateBIN - mean(templateBIN(:)); -templateBIN = templateBIN ./rms(templateBIN(:)); - - -sizeTemp = size(template) -sizeTempBIN = size(templateBIN) - - - -statsRadiusAng = statsRadius.*[2,2,2].*max(latticeRadius); -statsRadius = ceil(statsRadiusAng./pixelSize); -latticeRadius = (0.75 .* latticeRadius) ./ (pixelSize); -latticeRadius = floor(latticeRadius); -latticeRadius = latticeRadius + mod(latticeRadius, 2); - -eraseMaskRadius = floor((eraseMaskRadius) ./ (pixelSize)); -eraseMaskRadius = eraseMaskRadius + mod(eraseMaskRadius,2); - -fprintf('\ntomograms normalized in %f Angstrom cubic window\n',statsRadiusAng(1)); - -fprintf('\nlatticeRadius = %dx%dx%d pixels\n\n', latticeRadius); -fprintf('\neraseMaskType %s, eraseMaskRadius %dx%dx%d pixels\n',eraseMaskType,eraseMaskRadius); - % For wedgeMask -particleThickness = latticeRadius(3); - -gpuDevice(useGPU); - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -% Initialize a whole mess of control variables and storage volumes. % -%Out of plane range inc (starts from 1.* inc) -rotConvention = 'Bah'; -% Check and override the rotational convention to get helical averaging. -% Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = pBH.('doHelical'); -catch - doHelical = 0; -end -if ( doHelical ) - rotConvention = 'Helical' -end - -rotConvention - -[ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch) - - - -[ OUTPUT ] = BH_multi_iterator( [targetSize; ... - size(tomogram);... - sizeTempBIN; ... - 2.*latticeRadius], 'convolution' ); - - - -tomoPre = OUTPUT(1,:); -tomoPost = OUTPUT(2,:); -sizeChunk = OUTPUT(3,:); -validArea = OUTPUT(4,:); -validCalc = OUTPUT(5,:); -nIters = OUTPUT(6,:); - -%[ padVal ] = BH_multi_padVal( sizeTemp, sizeChunk ); -%tempPre = padVal(1,:); -%tempPost = padVal(2,:); - -[ padBIN ] = BH_multi_padVal( sizeTempBIN, sizeChunk ); -[ trimValid ] = BH_multi_padVal(sizeChunk, validArea); - -if ( tmpDecoy ) - % This is probably sample dependent. should search a small range and find - % the maximum rate of change in the ccc - - % the -1 searches for the next smallest fast fourier size - templateBIN = gpuArray(templateBIN); - - - - decoyTest = BH_reScale3d(templateBIN,'',tmpDecoy,'GPU'); - decoyTrim = BH_multi_padVal(size(decoyTest),size(templateBIN)); - decoyTest = fftn(BH_padZeros3d(decoyTest,decoyTrim(1,:),decoyTrim(2,:),'GPU','single')); - decoyShift = -1.*gather(BH_multi_xcf_Translational(decoyTest,conj(fftn(templateBIN)),'',[3,3,3])); - decoyNorm = gather(sum(abs(decoyTest(:)))./sum(abs(fftn(templateBIN(:))))); - padDecoy = BH_multi_padVal(size(decoyTest),sizeChunk) + decoyTrim; - clear decoyTest - templateBIN = gather(templateBIN); - fprintf('tmpDecoy %f normFactor %f and shift by %2.2f %2.2f %2.2f\n',tmpDecoy,decoyNorm,decoyShift); - -end - - -fprintf('\n-----\nProcessing in chunks\n\n'); -fprintf('tomo prepadding %d %d %d\n', tomoPre); -fprintf('tomo postpadding %d %d %d\n', tomoPost); -fprintf('size to process %d %d %d\n', sizeChunk); -fprintf('valid Area %d %d %d\n', validArea); -fprintf('valid Calc %d %d %d\n', validCalc); -fprintf('# of iterations %d %d %d\n', nIters); -fprintf('-----\n'); - -size(tomogram) - -% [ tomogram ] = BH_padZeros3d(tomogram, tomoPre, tomoPost, ... -% 'cpu', 'singleTaper'); -tomogram = padarray(tomogram,tomoPre,'symmetric','pre'); -tomogram = padarray(tomogram,tomoPost,'symmetric','post'); -sizeTomo = size(tomogram); - - -[ validAreaMask ] = gather(BH_mask3d('rectangle',sizeChunk,validCalc./2,[0,0,0])); -[ vA ] = BH_multi_padVal( validArea, sizeChunk ); -% This would need to be changed to take a mask size and not just a radius. -% Currently, this would not produce the correct results for odd size area -% % % fftMask = BH_fftShift(validArea,sizeChunk,0); - -% Array for storing chunk results -RESULTS_peak = zeros(sizeTomo, 'single'); -RESULTS_angle= zeros(sizeTomo, 'single'); - -if (scale_mip) - RESULTS_sum = zeros(sizeTomo,'single'); - RESULTS_sum_sq = zeros(sizeTomo,'single'); -end - -if ( tmpDecoy ) - RESULTS_decoy = RESULTS_peak; -end -% Loop over tomogram -% Set this up second - - -% optimize fft incase a power of two is not used, this will make things run ok. -opt = zeros(sizeChunk, precision,'gpuArray'); -fftw('planner','patient'); -fftn(opt); -clear opt ans - -% Temp while testing new dose weighting -TLT = tiltGeometry; -nPrjs = size(TLT,1); - - -kVal = 0; - -% % [ OUTPUT ] = BH_multi_iterator( [sizeTempBIN;kVal.*[1,1,1]], 'extrapolate' ); -[ OUTPUT ] = BH_multi_iterator( [sizeChunk;kVal.*[1,1,1]], 'extrapolate' ); - - - -switch wedgeType - case 1 - % make a binary wedge - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'binaryWedgeGPU',particleThickness,... - 1, 1, samplingRate); - case 2 - % make a non-CTF wedge - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 2, 1, samplingRate); - case 3 - % make a CTF without exposure weight - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 3, 1, samplingRate); - case 4 - % make a wedge with full-ctf - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 4, 1, samplingRate); - otherwise - error('wedgeType must be 1-4'); -end - -wedgeMask = (ifftshift(wedgeMask)); -% -% % Now just using the mask to calculate the power remaining in the template, -% % without actually applying. -% wedgeMask = gather(find(ifftshift(wedgeMask))); - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Preprocess the tomogram - -tomoIDX = 1; -nTomograms = prod(nIters); - - -tomoStack = zeros([sizeChunk,nTomograms], 'single'); - -% backgroundVol = zeros(sizeChunk,'single'); -tomoCoords= zeros(nTomograms, 3, 'uint16'); - -% % % [ tomoBandpass ] = BH_bandpass3d(sizeChunk, 0,maxSizeForHighPass, ... -% % % lowResCut,'cpu', pixelSize ); -% In switching to the full 3D-sampling function the high pass is -% already incorporated in the CTF. Still include one for very low -% resolution to deal with gradients in the tomos. -[ tomoBandpass ] = BH_bandpass3d(sizeChunk, 10e-4,800, ... - lowResCut,'cpu', pixelSize ); - -% if ~(shouldBeCTF) -% tomoBandpass = wedgeMask .* tomoBandpass; -% end - -clear wedgeMask - -try - doMedFilt = pBH.('Tmp_medianFilter'); - if ~ismember(doMedFilt,[3,5,7]) - error('Tmp_medianFilter can only be 3,5, or 7'); - else - fprintf('Using median filter, size %d',doMedFilt); - end -catch - doMedFilt =0 -end - -calcStats = 0; -if calcStats - maskStack = false([sizeChunk,nTomograms]); - calcMask = 0; -else - calcMask = 1; -end -firstStats = 1; -flgOOM = 0; - -fullX = 0; -fullX2 = 0; -fullnX = 0; - -for iX = 1:nIters(1) - cutX = 1 + (iX-1).*validArea(1); - for iY = 1:nIters(2) - cutY = 1 + (iY-1).*validArea(2); - for iZ = 1:nIters(3) - cutZ = 1 + (iZ-1).*validArea(3); - - fprintf('preprocessing tomo_chunk %d/%d col %d/%d row %d/%d plane idx%d\n' , ... - iY,nIters(2),iX,nIters(1),iZ,nIters(3),tomoIDX) - - - % Cut out chunk and zero pad - double would be more accurate, but for - % template matching which is fairly crude anyhow, this should be okay, and - % allows much larger chunks to be processed. - - - tomoChunk = tomogram(cutX:cutX+sizeChunk(1)-1,... - cutY:cutY+sizeChunk(2)-1,... - cutZ:cutZ+sizeChunk(3)-1); - - - tomoChunk = real(ifftn(fftn(tomoChunk).*tomoBandpass)); - - - - if doMedFilt - if ( flgOOM ) - tomoChunk = (medfilt3(tomoChunk,doMedFilt.*[1,1,1])); - else - tomoChunk = gpuArray(medfilt3(tomoChunk,doMedFilt.*[1,1,1])); - end - else - if ( flgOOM ) - % Leave on CPU - else - tomoChunk = gpuArray(tomoChunk); - statsRadius = gather(statsRadius); - end - end - - % Handle all mean centering and rms normalization in local window - - [ averageMask, flgOOM ] = BH_movingAverage(tomoChunk, statsRadius); - - if isa(tomoChunk(1),'gpuArray') && flgOOM - tomoChunk = gather(tomoChunk); - end - - - - tomoChunk= tomoChunk - averageMask; clear averageMask - - [ rmsMask ] = BH_movingRMS(tomoChunk, statsRadius); - - - - if ( shouldBeCTF == 1 ) - tomoStack(:,:,:,tomoIDX) = gather((tomoChunk ./ rmsMask).*validAreaMask); - else - % Using the non-ctf corrected stack since we limit toA all practical - % defocus (<8um) should be entirely negative, so just flip in real - % space - - tomoStack(:,:,:,tomoIDX) = gather(-1.*(tomoChunk ./ rmsMask).*validAreaMask); -% backgroundVol = backgroundVol + gather(tomoChunk.*maskStack(:,:,:,tomoIDX)); - end - - clear rmsMask - - - fullX = fullX + sum(sum(sum(tomoStack(:,:,:,tomoIDX)))); - fullX2 = fullX2 + sum(sum(sum(tomoStack(:,:,:,tomoIDX).^2))); - fullnX = fullnX + prod(sizeChunk); - - tomoCoords(tomoIDX,:) = [cutX,cutY,cutZ]; - tomoIDX = tomoIDX + 1; - - - end % end of loop over Z chunks - end % end of loop over Y chunks -end % end of loop over X chunks - -% Normalize the global variance -globalVariance = (fullX2 - fullX)/fullnX; -fprintf('After local normalization, scaling also the global variance\n'); - -for iChunk = 1:tomoIDX-1 - tomoStack(:,:,:,iChunk) = tomoStack(:,:,:,iChunk) ./ globalVariance; -end - - - -clear tomoWedgeMask averagingMask rmsMask bandpassFilter statBinary validAreaMask tomoChunk - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -kVal = 0; - -[ OUTPUT ] = BH_multi_iterator( [sizeTempBIN;kVal.*[1,1,1]], 'extrapolate' ); - - -switch wedgeType - case 1 - % make a binary wedge - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'binaryWedgeGPU',particleThickness,... - 1, 1, samplingRate); - case 2 - % make a non-CTF wedge - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 2, 1, samplingRate); - case 3 - % make a CTF without exposure weight - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 3, 1, samplingRate); - case 4 - % make a wedge with full-ctf - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 4, 1, samplingRate); - otherwise - error('wedgeType must be 1-4'); -end - - - - -% -% % Now just using the mask to calculate the power remaining in the template, -% % without actually applying. -% wedgeMask = gather(find(ifftshift(wedgeMask))); - - - - - - - -currentGlobalAngle = 1; -ANGLE_LIST = zeros(nAngles(1),3, 'single'); - -highThr=sqrt(2).*erfcinv(ceil(peakThreshold.*0.025).*2./(numel(tomogram).*nAngles(1))) - -nComplete = 0; -totalTime = 0; -firstLoopOverTomo = true; -for iAngle = 1:size(angleStep,1) - - theta = angleStep(iAngle,1); - - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. - if (doHelical) - phi_step = 360; - else - phiStep = angleStep(iAngle,3); - end - phiStep = angleStep(iAngle,3); - - gpuDevice(useGPU); - - %numRefIter = nAngles(1); - numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - tempImg = gpuArray(templateBIN); %%%%% NEW switch to bin -% tempWdg = gpuArray(wedgeMask); -% tempBnd = gpuArray(tempBandpass); - tempBandpass = gpuArray(tomoBandpass); - - interpolationNormFactor = sum(abs(tempImg(:)).^2); - - - - clear referenceStack tempFilter - % Calculate all references for each out of plane tilt only once - referenceStack = zeros([sizeTempBIN,numRefIter], 'single', 'gpuArray'); - - - - tomoIDX = 1; - firstLoopOverAngle = true; - % Iterate over the tomogram pulling each chunk one at a time. - for iTomo = 1:nTomograms - tic; - iCut = tomoCoords(tomoIDX,:); - % reset the angle count and value at the begining of loop - % inside, while each new outer loop changes the start values. - -% nAngle = angleIncStart; - intraLoopAngle = 1; - - % Truth value to initialize temp results matrix each new tomo - % chunk. - firstLoopOverChunk = true; - - fprintf('working on tilt(%d/%d) tomoChunk(idx%d/%d)\t' ... - ,iAngle,size(angleStep,1), tomoIDX,nTomograms); - - - % fftn(double(gpuArray))) ~ 2.5x faster than transfering a double - % complex - if strcmpi(precision, 'double') - tomoFou = fftn(double(gpuArray(tomoStack(:,:,:,tomoIDX)))); - else - tomoFou = fftn(gpuArray(tomoStack(:,:,:,tomoIDX))); - end - for iAzimuth = 0:angleStep(iAngle,2) - - if ( doHelical ) - phi = 90 ; - else - phi = phiStep * iAzimuth; - end - - for iInPlane = inPlaneSearch - psi = iInPlane; - - %calc references only on first chunk - if (firstLoopOverAngle) - - ANGLE_LIST(currentGlobalAngle,:) = [phi, theta, psi - phi]; - [phi, theta, psi - phi]; - % Rotate the reference, lowpass and wedge mask, send to gpu - % Inverse rotation(i.e. rotate particle, angles saved - % are to rotate frame to particle for extraction.) -% % % % % tempRot = BH_resample3d(tempImg, [phi, theta, psi - phi], [1,1,1], ... -% % % % % {'Bah', 1,'linear',1,interpMaskGPU},... -% % % % % 'GPU','forward'); - - - tempRot = BH_resample3d(tempImg, [phi, theta, psi - phi], [1,1,1], ... - {'Bah', 1,'linear',1},... - 'GPU','forward'); - - - - %%%%%tempFou = BH_bandLimitCenterNormalize(tempRot,tempWedgeMask,'',[tempPre;tempPost],precisionTaper); - - %%%%%tempRot = BH_padZeros3d(real(ifftn(tempFou)),-1.*tempPre,-1.*tempPost,'GPU',precision); - - %%%%%tempRot = gather(BH_reScale3d(tempRot,'',sprintf('%f',1/samplingRate),'GPU')); - - % if (firstLoopOverTomo) - % SAVE_IMG(MRCImage(tempRot), sprintf('temp_%s.mrc',convTMPNAME),pixelSize); - % end - - % First correct for any change in power due to - % rotation/interpolation - tempRot = tempRot .* (interpolationNormFactor./sum(abs(tempRot(:)).^2)); - % Then correct for any change in power due to the wedge. These can be combined -% normFT = abs(fftn(tempRot).*tempBnd).^2; -% % -% % -% normScore = sum(normFT(:)) ./ sum(normFT(:).*tempWdg(:)); -% clear normFT; -% tempRot = tempRot .* normScore; - %clear normScore - - referenceStack(:,:,:,intraLoopAngle) = tempRot; - -% % % % % tempFou = fftn(BH_padZeros3d(tempRot,padBIN(1,:),padBIN(2,:),'GPU',precision)); - - tempFou = BH_bandLimitCenterNormalize( tempRot, tempBandpass, '', ... - padBIN, 'single' ); - - - else - -% % % % % tempFou = (fftn(BH_padZeros3d( ... -% % % % % referenceStack(:,:,:,intraLoopAngle), ... -% % % % % padBIN(1,:), padBIN(2,:),'GPU', precision))); - - - tempFou = BH_bandLimitCenterNormalize( referenceStack(:,:,:,intraLoopAngle), tempBandpass, '', ... - padBIN, 'single' ); - - end - - -% % % ccfmapFull = fftshift(real(single(ifftn(tomoFou.*conj(tempFou))))); -% % % -% % % -% % % ccfmap = ccfmapFull(vA(1,1) + 1:end - vA(2,1), ... -% % % vA(1,2) + 1:end - vA(2,2), ... -% % % vA(1,3) + 1:end - vA(2,3)); - - % Even with local normalization, test with all padding and - % goodness. -% tomoNorm = ((sqrt(sum(sum(sum(abs(tomoFou).^2)))) ./ numel(tomoFou))); -% tempNorm = ((sqrt(sum(sum(sum(abs(tempFou).^2)))) ./ numel(tempFou))); - -% ./(sum(sum(sum((abs(tomoFou).*abs(tempFou)).^2)))) - ccfmap = BH_padZeros3d(fftshift(real(single(... - ifftn(tomoFou.*conj(tempFou) )))),...%./(tomoNorm.*tempNorm))))),... - trimValid(1,:),trimValid(2,:),'GPU',precision); -% -% Since the vast majority of ccf values are known to not be due to -% the target, they are noise. Try just normalizing the StdDev of -% everything assuming this is the noise. - ccfmap = ccfmap ./ std(ccfmap(:)); - - - - if ( tmpDecoy > 0 ) - tempFou = []; - if (firstLoopOverAngle) - - decoy = BH_padZeros3d(BH_reScale3d(tempRot./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - padDecoy(1,:),padDecoy(2,:),'GPU','single'); - else - % Probably just make a second decoy stack to avoid - % re-interpolating. If it works, then do this. - decoy = BH_padZeros3d(BH_reScale3d(referenceStack(:,:,:,intraLoopAngle)./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - padDecoy(1,:),padDecoy(2,:),'GPU','single'); - end - - - - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*conj(fftn(decoy)))))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU',precision); - - - elseif ( tmpDecoy < 0 ) - - % Just use the mirror image of the template, i.e. take the conj - % (of the conj) so just the padded FFT of the ref. - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*tempFou)))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU',precision); - tempFou = []; - end - clear tempRot - % If first loop over tomo, initialize the storage volumes, if - % first loop over the chunk but not over the tomo, pull storage - % chunks from storage volume. - if (firstLoopOverTomo && firstLoopOverChunk) - %store ccfmap as complex with phase = angle of reference - magTmp = ccfmap; - if ( tmpDecoy ) - decoyTmp = decoy; - end - angTmp = zeros(size(magTmp), 'single','gpuArray'); - angTmp = angTmp + 1; - - if (scale_mip) - sumTmp = ccfmap; - sumSqTmp = ccfmap.^2; - end - firstLoopOverTomo = false; - firstLoopOverChunk = false; - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - - elseif (firstLoopOverChunk) - % These double cuts are old, and don't really make sense. Make - % this more consistant with current operations when there is - % time. - magTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - if ( tmpDecoy ) - decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyTmp = gpuArray(decoyTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - - magTmp = gpuArray(magTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - - firstLoopOverChunk = false; - - replaceTmp = ( magTmp < ccfmap ); - - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - - if (scale_mip) - - sumTmp = RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sumSqTmp = RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sumTmp = gpuArray(sumTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - sumSqTmp = gpuArray(sumSqTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - end - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp - - else - % update higher values of ccfmap with new reference if applicable. - - if (scale_mip) - sumTmp = sumTmp + ccfmap; - sumSqTmp = sumSqTmp + ccfmap.^2; - end - - replaceTmp = ( magTmp < ccfmap ); - - - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - if ( tmpDecoy ) - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp - end - nComplete = nComplete + 1; - - end - end - - % After searching all angles on this chunk, but out meaningful - % portion for storage. - - magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - - - magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); - angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); - - - RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; - - clear magStoreTmp - - RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; - clear angStoreTmp - - if (scale_mip) - sumStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sumSqStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - - - sumStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sumTmp); - sumSqStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sumSqTmp); - - - RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sumStoreTmp; - - clear sumStoreTmp - - RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sumSqStoreTmp; - clear sumSqStoreTmp - end - - if ( tmpDecoy ) - decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(decoyTmp); - RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = decoyStoreTmp; - - end - tomoTime = toc; - totalTime = totalTime + toc; timeEstimate = totalTime * (nTomograms*nAngles(1)./(nComplete-1)); - fprintf('elapsed time = %f s est remain %f s\n', tomoTime, timeEstimate); - tomoIDX = tomoIDX + 1; - firstLoopOverAngle = false; - currentGlobalAngle = currentGlobalAngle - intraLoopAngle + 1; - end - - currentGlobalAngle = currentGlobalAngle + intraLoopAngle - 1; -end -%save('angle_list.txt','angle_list','-ascii'); -clear tomoStack -% Cut out the post padding used to iterate over the tomogram -RESULTS_peak = RESULTS_peak(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); -%RESULTS_peak(RESULTS_peak < 0) = 0; -RESULTS_angle = RESULTS_angle(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); - -if (scale_mip) - RESULTS_sum = RESULTS_sum(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)) ./ nComplete; - - RESULTS_sum_sq = RESULTS_sum_sq(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)) ./ nComplete; -end - - -if ( tmpDecoy ) - RESULTS_decoy = RESULTS_decoy(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); -% RESULTS_decoy = RESULTS_decoy ./ std(RESULTS_decoy(:)); - RESULTS_decoy(RESULTS_decoy < 1) = 1; - -end -gpuDevice(useGPU); - - -% scale the magnitude of the results to be 0 : 1 -szK = latticeRadius;%floor(0.8.*szM); -rmDim = max(max(eraseMaskRadius),max(szK)).*[1,1,1]; -mag = RESULTS_peak; clear RESULTS_peak -% Normalize so the difference if using a decoy makes sense. The input decoy -% should have the same power, so I'm not sure why this is needed, but it is -% an easy fix and a problem for future Ben to figure out. -% mag = mag ./ std(mag(:)); - -system(sprintf('mkdir -p %s',convTMPNAME)); -system(sprintf('mv temp_%s.mrc %s',convTMPNAME,convTMPNAME)); - -resultsOUT = sprintf('./%s/%s_convmap.mrc',convTMPNAME,mapName); -anglesOUT = sprintf('./%s/%s_angles.mrc',convTMPNAME,mapName); -angleListOUT = sprintf('./%s/%s_angles.list',convTMPNAME,mapName); -SAVE_IMG(MRCImage(mag),resultsOUT); -SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); - -if (scale_mip) - scaledMipOUT = sprintf('./%s/%s_convmap_scaled.mrc',convTMPNAME,mapName); - VarEst = RESULTS_sum_sq - RESULTS_sum.^2; - nonZero = abs(VarEst(:)) > 1e-3; - SAVE_IMG(MRCImage(RESULTS_sum),sprintf('./%s/%s_convmap_sum.mrc',convTMPNAME,mapName)); - SAVE_IMG(MRCImage(RESULTS_sum_sq),sprintf('./%s/%s_convmap_sumSq.mrc',convTMPNAME,mapName)); - SAVE_IMG(MRCImage(VarEst),sprintf('./%s/%s_convmap_varEst.mrc',convTMPNAME,mapName)); - - VarEst(nonZero) = ( mag(nonZero) - RESULTS_sum(nonZero) ) ./ VarEst(nonZero); - SAVE_IMG(MRCImage(VarEst),scaledMipOUT); -end - -if ( tmpDecoy ) - decoyOUT = sprintf('./%s/%s_decoy.mrc',convTMPNAME,mapName); - SAVE_IMG(MRCImage((RESULTS_decoy)),decoyOUT); - diffOUT = sprintf('./%s/%s_convmap-decoy.mrc',convTMPNAME,mapName); - decoyLogical = mag < RESULTS_decoy; - mag(decoyLogical) = 0; - mag(~decoyLogical) = mag(~decoyLogical) - RESULTS_decoy(~decoyLogical); clear RESULTS_decoy - SAVE_IMG(MRCImage((mag)),diffOUT); -end -angleFILE = fopen(angleListOUT,'w'); -fprintf(angleFILE,'%2.2f\t%2.2f\t%2.2f\n', ANGLE_LIST'); -fclose(angleFILE); - - -% mag = mag - min(mag(:)); mag = mag ./ max(mag(:)); - -% Zero out one lattice width from the edges to reduce edge effect (but cutting -% out and padding back in.) Also pad by size of removal mask (subtract this from -% coordinates) -mag = mag(szK(1)+1:end - szK(1), ... - szK(2)+1:end - szK(2), ... - szK(3)+1:end - szK(3)); -mag = BH_padZeros3d(mag,szK+rmDim,szK+rmDim, 'cpu', 'single'); -%dev.FreeMemory; -%%%Ang = angle(RESULTS_peak); %clear Results -% negative phase angles mapped back to 0-->pi -%Ang(sign(Ang) < 0) = Ang(sign(Ang)<0) + pi; serotonin_ali1_75_1.mod -%Ang = BH_padZeros3d(round(Ang./angleIncrement),szK,szK,'cpu','single'); -Ang = BH_padZeros3d(RESULTS_angle,rmDim,rmDim,'cpu','single'); - - -%mag = mag - min(mag(:)); mag = mag ./ max(mag(:)); - -Tmean = mean(mag(( mag ~= 0 ))); -Tstd = std(mag(( mag~=0 ))); -threshold = Tmean + peakThreshold*Tstd; -mag((Ang < 0)) = 0; - -mag = gpuArray(mag); -sizeTomo = size(mag); - - -[MAX, coord] = max(mag(:)); - -peakMat = zeros(peakThreshold,10*nPeaks); - -n = 1; - -fprintf('rmDim %f szK %f\n', rmDim,szK); -removalMask = BH_mask3d(eraseMaskType,[2,2,2].*rmDim+1,eraseMaskRadius,[0,0,0]); - -maskCutOff = 0.999; -nIncluded = gather(sum(sum(sum(removalMask > maskCutOff)))); -nTries = 0; -if strcmpi(eraseMaskType,'rectangle') - areaPreFactor = 0; -else - areaPreFactor = (4/3*pi); -end - -while nIncluded < areaPreFactor*prod(eraseMaskRadius) - maskCutOff = 0.99*maskCutOff; - nIncluded = gather(sum(sum(sum(removalMask > maskCutOff)))); - nTries = nTries + 1; - if (nTries > 1000) - error('Did not find an appropriate erase mask'); - end - -end -this_try = 0; -while n <= peakThreshold && (this_try < max_tries) -this_try = this_try + 1; - -% -% Some indicies come back as an error, even when they seem like the -% should be fine. I'm not sure why, and I should think about this -% more, but for now, just set that one index to zero (instead of a -% whole box) and move on with life. It looks like the index that is -% kicking out the error is equal to -1*numberofreferences, which -% might be an issue because that corresonds to the positive upper -% limit of the reference index. Ignoring it still seems to be okay -% but it bothers me not to know. - - -[i,j,k] = ind2sub(sizeTomo,coord); -try - c = gather([i,j,k]); -catch - print('Ran into some trouble gathering the i,j,k. Breaking out\n'); - break -end - - if Ang(gather(coord)) > 0 - - % box for removal and center of mass calc, use a larger box if multiple - % peaks are being saved. - bDist = 1+round(log(nPeaks)); - clI = c(1) - bDist; - chI = c(1) + bDist; - clJ = c(2) - bDist; - chJ = c(2) + bDist; - clK = c(3) - bDist; - chK = c(3) + bDist; - - magBox = mag(clI:chI,clJ:chJ,clK:chK); - - angBox = Ang(clI:chI,clJ:chJ,clK:chK); - - [cmX, cmY, cmZ] = ndgrid(-1*bDist:1*bDist, ... - -1*bDist:1*bDist, ... - -1*bDist:1*bDist ); - - cMass = [ sum(sum(sum(magBox.*cmX))) ; ... - sum(sum(sum(magBox.*cmY))) ; ... - sum(sum(sum(magBox.*cmZ))) ] ./ sum(magBox(:)); - - - -% cenP = [ (c(1)+cMass(1)-1) - sizeTomo(1)./2 ,... -% (c(2)+cMass(2)-1) - sizeTomo(2)./2 ,... -% (c(3)+cMass(3)-1) - sizeTomo(3)./2 ]; - - % Switching from centered to lower left coordinates and subtracting the - % padding - - cenP = c + cMass' - rmDim; - - - - % If the most frequent peak is unique use it; - [peakM, ~, peakC] = mode(angBox(:)); - if length(peakC) == 1 && peakM - % Need to ensure the mode is none zero which is possible. - peakMat(n,4:6) = ANGLE_LIST(peakM,:); - topPeak = peakM; - else - % Otherwise use the value at the max for the peak val; - peakMat(n,4:6) = ANGLE_LIST(Ang(coord),:); - topPeak = Ang(coord); - end - peakMat(n,1:3) = gather(samplingRate.*cenP); - - if nPeaks > 1 - oldPeaks = ( angBox == topPeak ); - - for iPeak = 2:nPeaks - [peakM, ~, ~] = mode(angBox(~oldPeaks)); - % There could be redundancy, as given by peakC, but just take the - % first value given by peak M. - peakMat(n,[1:3]+10*(iPeak-1)) = gather(samplingRate.*cenP); - peakMat(n,[4:6]+10*(iPeak-1)) = ANGLE_LIST(peakM,:); - - oldPeaks = ( angBox == peakM | oldPeaks ); - - end - - end - - - - - rmMask = BH_resample3d(removalMask,peakMat(n,4:6),[0,0,0],'Bah','GPU','forward'); - % Invert after resampling so that zeros introduced by not extrapolating - % the corners are swapped to ones, i.e. not removed. -% rmMask = (1-rmMask); - - mag(c(1)-rmDim:c(1)+rmDim,... - c(2)-rmDim:c(2)+rmDim,... - c(3)-rmDim:c(3)+rmDim) = ... - mag(c(1)-rmDim:c(1)+rmDim,... - c(2)-rmDim:c(2)+rmDim,... - c(3)-rmDim:c(3)+rmDim) .* (rmMask< maskCutOff); - - peakMat(n,10) = (gather(MAX) - Tmean)./Tstd; % record stds above mean - n = n + 1; - - if ~mod(n,100) - n - end - - else - Ang(gather(coord)); - mag(coord) = 0; - end - - -[MAX, coord] = max(mag(:)); - -end - -peakMat = peakMat( ( peakMat(:,1)>0 ),:); - -%save('peakMat_post.mat', 'peakMat'); - -% A temp test, not the correct output just score x y z dx dy dz e1 e2 e3 - -csv_out = sprintf('./%s/%s.csv',convTMPNAME,mapName); -pos_out = sprintf('./%s/%s.pos',convTMPNAME,mapName); -%fieldOUT = zeros(length(peakMat(:,1)),26); -fileID = fopen(csv_out,'w'); -fileID2 = fopen(pos_out,'w'); -errID = fopen(sprintf('./%s/%s.errID',convTMPNAME,mapName)); - - -if SYMMETRY > 1 - symmetry = 0:360/SYMMETRY:359; - symCell = cell(length(symmetry),1); - for iSym = 1:length(symmetry) - symCell{iSym} = BH_defineMatrix([symmetry(iSym),0,0], 'Bah', 'inv'); - end - -end -n=1 -for i = 1:length(peakMat(:,1)) - if all(peakMat(i,1:3)) - - if SYMMETRY > 1 - % Generate a uniform distribution over the in-plane - % randomizations - iSym = rem( n + SYMMETRY, SYMMETRY)+1; - r = reshape(BH_defineMatrix(peakMat(i,4:6), 'Bah', 'inv')*... - symCell{iSym},1,9); - else - r = reshape(BH_defineMatrix(peakMat(i,4:6), 'Bah', 'inv'),1,9); - end - fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... - peakMat(i,4:6),r,1); - - if nPeaks > 1 - - for iPeak = 2:nPeaks - if SYMMETRY > 1 - % Generate a uniform distribution over the in-plane - % randomizations - iSym = rem( n + SYMMETRY, SYMMETRY)+1; - r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), 'Bah', 'inv')*... - symCell{iSym},1,9); - else - r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), 'Bah', 'inv'),1,9); - end - fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,[1:3]+10*(iPeak-1)), ... - peakMat(i,[4:6]+10*(iPeak-1)),r,1); - end - - - end - - fprintf(fileID,'\n'); - - - - - fprintf(fileID2,'%f %f %f\n',peakMat(i,1:3)./samplingRate); - - - n = n +1; - end -end - -%lastIndex = find(fieldOUT(:,4),1,'last'); - -fclose(fileID); -fclose(fileID2); - -system(sprintf('point2model -number 1 -sphere 3 -scat ./%s/%s.pos ./%s/%s.mod', convTMPNAME,mapName,convTMPNAME, mapName)); - -fileID = fopen(sprintf('./%s/%s.path',convTMPNAME,mapName),'w'); -fprintf(fileID,'%s,%s,%s,%s',mapName,mapPath,mapExt,RAWTLT); -fclose(fileID); -%subTomoMeta.('cycle000').('geometry').(mapName) = fieldOUT; -% subTomoMeta.('mapPath').(mapName) = mapPath; -% subTomoMeta.('mapExt').(mapName) = mapExt; - -% if any(ismember(fieldnames(subTomoMeta), 'nSubTomoTotal')) -% subTomoMeta.('nSubTomoTotal') = subTomoMeta.('nSubTomoTotal') + lastIndex; -% else -% subTomoMeta.('nSubTomoTotal') = lastIndex; -% end - -% preFscSplit = gather(subTomoMeta); -% -% % Randomly divide the data into half sets. -% [ subTomoMeta ] = BH_fscSplit( preFscSplit ); -% subTomoMeta.('currentCycle') = 0; - -% save(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); -% save(sprintf('./convmap/%s.mat~', pBH.('subTomoMeta')), 'subTomoMeta'); -%save('test.pos','a','-ascii'); - - -fprintf('Total execution time : %f seconds\n', etime(clock, startTime)); - - - -end % end of templateSearch3d function - diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index bef2d17d..c39335df 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -1,8 +1,8 @@ function [] = BH_templateSearch3d_2( PARAMETER_FILE,... - tomoName,tomoNumber,TEMPLATE, ... - SYMMETRY, wedgeType, varargin) - - + tomoName,tomoIdx,TEMPLATE, ... + SYMMETRY, wedgeType, varargin) + + %3d template matching %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -10,19 +10,26 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - ctf3dNoSubTomoMeta = true; -if length(varargin) == 1 +if length(varargin) > 0 % Allow for an override of the max number, useful when only a few tomos % have a strong feature like carbon that is hard to avoid. gpuIDX = EMC_str2double(varargin{1}); -elseif length(varargin) > 1 - error('emClarity templateSearch paramN.m tiltN regionN referenceName symmetry(C1) '); +else + gpuIDX = 1; +end +if length(varargin) == 2 + mapBackIter = EMC_str2double(varargin{2}); +else + mapBackIter = 0; +end +if length(varargin) > 2 + error('emClarity templateSearch paramN.m tiltN regionN referenceName symmetry(C1) '); end -tomoNumber = EMC_str2double(tomoNumber); +tomoIdx = EMC_str2double(tomoIdx); -[ useGPU ] = BH_multi_checkGPU( gpuIDX ) +[ useGPU ] = BH_multi_checkGPU( gpuIDX ); @@ -33,85 +40,42 @@ % SYMMETRY = EMC_str2double(SYMMETRY); SYMMETRY=1; -startTime = clock ; +startTime = datetime("now") ; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); -if ctf3dNoSubTomoMeta - mapBackIter = 0; - shouldBeCTF = 1; -else - try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR - % clear subTomoMeta - % Make sure we get a CTF corrected stack - shouldBeCTF = 1 - catch - mapBackIter = 0; - shouldBeCTF = -1; - end -end - samplingRate = pBH.('Tmp_samplingRate'); +% Currently hardcoded to always expect a tomogram constructed with ctf correction +% using emClarity ctf3d paramN.m templateSearch +use_ctf3d_templateSearch=true; + +samplingRate = emc.('Tmp_samplingRate'); + +test_half = emc.('Tmp_half_precision'); -try - tmpDecoy = pBH.('templateDecoy') -catch - tmpDecoy = 0 -end -try - super_sample = pBH.('super_sample'); - if (super_sample > 0) - [~,v] = system('cat $IMOD_DIR/VERSION'); - v = split(v,'.'); - if (EMC_str2double(v{1}) < 4 || (EMC_str2double(v{2}) <= 10 && EMC_str2double(v{3}) < 42)) - fprintf('Warning: imod version is too old for supersampling\n'); - super_sample = ''; - else - super_sample = sprintf(' -SuperSampleFactor %d',super_sample); - end - else - super_sample = ''; - end -catch - super_sample = ''; - expand_lines = ''; -end - peakThreshold = pBH.('Tmp_threshold'); +peakThreshold = emc.('Tmp_threshold'); -latticeRadius = pBH.('particleRadius'); +latticeRadius = emc.('particleRadius'); try - targetSize = pBH.('Tmp_targetSize') + targetSize = emc.('Tmp_targetSize') catch targetSize = [512,512,512]; end -angleSearch = pBH.('Tmp_angleSearch'); +angleSearch = emc.('Tmp_angleSearch'); convTMPNAME = sprintf('convmap_wedgeType_%d_bin%d',wedgeType,samplingRate) -try - use_new_grid_search = pBH.('use_new_grid_search'); -catch - use_new_grid_search = true; -end try - symmetry = pBH.('symmetry'); -catch - error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); -end - -try - eraseMaskType = pBH.('Peak_mType'); + eraseMaskType = emc.('Peak_mType'); catch eraseMaskType = 'sphere'; end try - eraseMaskRadius = pBH.('Peak_mRadius'); + eraseMaskRadius = emc.('Peak_mRadius'); catch eraseMaskRadius = 1.0.*latticeRadius; end @@ -120,84 +84,50 @@ nPreviousSubTomos = 0; reconScaling = 1; -try - nPeaks = pBH.('nPeaks'); -catch - nPeaks = 1; -end + ignore_threshold = false; try - max_tries = pBH.('max_peaks'); + max_tries = emc.('max_peaks'); catch max_tries = 10000; end try - over_ride = pBH.('Override_threshold_and_return_N_peaks') + over_ride = emc.('Override_threshold_and_return_N_peaks') ignore_threshold = true; fprintf('Override_threshold_and_return_N_peaks set to true, returning exactly %d peaks\n', over_ride); peakThreshold = over_ride; end -pixelSizeFULL = pBH.('PIXEL_SIZE').*10^10; -if pBH.('SuperResolution') - pixelSizeFULL = pixelSizeFULL * 2; -end +pixelSizeFULL = emc.pixel_size_angstroms; -pixelSize = pixelSizeFULL.*samplingRate; +pixelSize = emc.pixel_size_angstroms .* samplingRate; % For testing print_warning=false; -try - wantedCut = pBH.('lowResCut'); - fprintf('lowResCut is deprecated and will be removed in future versions.\n') - fprintf('please switch to Tmp_bandpass\n\n'); - bp_vals = [1e-3,600,wantedCut]; - print_warning = true; -catch - bp_vals = [1e-3,600,28]; -end -try - bp_vals = pBH.('Tmp_bandpass'); - if numel(bp_vals) ~= 3 - error('Tmp_bandpass is [filter at zero freq, res high-pass cutoff, res low-pass cutoff]'); - end - if print_warning - fprintf('WARNING, you specified lowResCut (deprecated) and Tmp_bandpass!\n'); - end - fprintf('You specified a bandpass with values [%2.2e,%3.2f,%3.2f]\n',bp_vals); -catch - bp_vals = [1e-3,600,28]; - fprintf('Using default bandpass with values [%2.2e,%3.2f,%3.2f]\n',bp_vals); -end + +bp_vals = emc.('Tmp_bandpass'); + try - stats_diameter_fraction = pBH.('diameter_fraction_for_local_stats') + stats_diameter_fraction = emc.('diameter_fraction_for_local_stats') catch - stats_diameter_fraction = 1 + stats_diameter_fraction = 1 end -sum_of_x = []; -sum_of_x2 = []; + +mean_r2 = 0; +mean_r_mask = 0; +reference_mask = []; + try - rescale_mip = pBH.('rescale_mip'); + measure_noise_variance = emc.('measure_noise_variance'); catch - rescale_mip = false; + measure_noise_variance = false; end -% Limit to the first zero if we are NOT using the CTF rec -if (shouldBeCTF ~= 1) -TLT = load(sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tomoName,mapBackIter+1)); - def = mean(-1.*TLT(:,15))*10^6; %TODO if you switch to POSITIVEDEFOCUS this will be wrong - firstZero = -0.2*def^2 +5.2*def +11; - - % Take the lower of firstZero lowResCut or Nyquist - bp_vals(3) = max(bp_vals(3), firstZero); - fprintf('\nUsing max (%f) of specified resolution cutoff of %f and first ctf zero %f Angstrom\n',bp_vals(3), wantedCut, firstZero); - -end if pixelSize*2 > bp_vals(3) fprintf('\nLimiting to Nyquist (%f) instead of user requested low pass cutoff %f Angstrom\n',pixelSize*2,bp_vals(3)); @@ -206,17 +136,13 @@ mapPath = './cache'; -mapName = sprintf('%s_%d_bin%d',tomoName,tomoNumber,samplingRate); +mapName = sprintf('%s_%d_bin%d',tomoName,tomoIdx,samplingRate); mapExt = '.rec'; -sprintf('recon/%s_recon.coords',tomoName) -[ recGeom, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName) ); +% [ recGeom, ~, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName), mapBackIter); -reconCoords = recGeom(tomoNumber,:); -clear recGeom - -bp_vals(2) = 2.*max(latticeRadius); +% bp_vals(2) = 2.*max(latticeRadius); statsRadiusAng = stats_diameter_fraction.*[2,2,2].*max(latticeRadius); statsRadius = ceil(statsRadiusAng./pixelSize); % Convert to binned pixels maskRadius = ceil(0.5.*[1,1,1].*max(latticeRadius)./pixelSize); @@ -227,48 +153,49 @@ eraseMaskRadius = floor((eraseMaskRadius) ./ (pixelSize)); eraseMaskRadius = eraseMaskRadius + mod(eraseMaskRadius,2); -fprintf('EXPERIMENTAL setting the highpass to match the max particle diameter. %3.3f Ang\n\n', bp_vals(2)); +% fprintf('EXPERIMENTAL setting the highpass to match the max particle diameter. %3.3f Ang\n\n', bp_vals(2)); fprintf('\ntomograms normalized in %f Angstrom cubic window\n',statsRadiusAng(1)); fprintf('\nlatticeRadius = %dx%dx%d pixels\n\n', latticeRadius); fprintf('\neraseMaskType %s, eraseMaskRadius %dx%dx%d pixels\n',eraseMaskType,eraseMaskRadius); - % For wedgeMask +% For wedgeMask particleThickness = latticeRadius(3); -% [ tomogram ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... -% reconCoords, mapBackIter, samplingRate,... -% shouldBeCTF*gpuIDX, reconScaling,1,'','ctf'); -[ tomogram ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... - reconCoords, mapBackIter, samplingRate,... - shouldBeCTF*gpuIDX, reconScaling,1,'',super_sample); - +do_load = true; +[ tomogram ] = BH_multi_loadOrBuild(emc.alt_cache, ... + sprintf('%s_%d',tomoName,tomoIdx), ... + mapBackIter, ... + samplingRate,... + gpuIDX, ... + do_load, ... + ''); + % We'll handle image statistics locally, but first place the global environment % into a predictible range - -[template, tempPath, tempName, tempExt] = ... - BH_multi_loadOrBin( TEMPLATE, 1, 3 ); - + +[template, tempPath, tempName, tempExt] = BH_multi_loadOrBin( TEMPLATE, 1, 3, true ); + % Bandpass the template so it is properly normalized bp_vals temp_bp = BH_bandpass3d(size(template),bp_vals(1),0.3.*bp_vals(2),bp_vals(3),'GPU',pixelSizeFULL); template = real(ifftn(fftn(gpuArray(template)).*temp_bp.^2)); clear temp_bp - + % The template will be padded later, trim for now to minimum so excess % iterations can be avoided. fprintf('size of provided template %d %d %d\n',size(template)); -trimTemp = BH_multi_padVal(size(template),ceil(2.0.*max(pBH.('Ali_mRadius')./pixelSizeFULL))); +trimTemp = BH_multi_padVal(size(template),ceil(2.0.*max(emc.('Ali_mRadius')./pixelSizeFULL))); % template = BH_padZeros3d(template, trimTemp(1,:),trimTemp(2,:),'cpu','singleTaper'); % SAVE_IMG(MRCImage(template),'template_trimmed.mrc'); clear trimTemp fprintf('size after trim to sqrt(2)*max(lattice radius) %d %d %d\n',size(template)); - + if isempty(mapPath) ; mapPath = '.' ; end if isempty(tempPath) ; tempPath = '.' ; end % Check to see if only tilt angles are supplied, implying a y-axis tilt scheme, @@ -288,7 +215,11 @@ templateBIN = templateBIN - mean(templateBIN(:)); templateBIN = templateBIN ./rms(templateBIN(:)); -[templateMask] = gather(EMC_maskReference(gpuArray(templateBIN),pixelSize,{'fsc', true})); +[templateMask] = (EMC_maskReference(gpuArray(templateBIN),pixelSize,{'fsc', true})); +templateMask = gather(templateMask); + +% templateMask = gather(EMC_maskShape('sphere', size(templateBIN), [3,3,3].*2, 'gpu', {'shift', [0,0,0];'kernel',false})); + sizeTemp = size(template); @@ -305,44 +236,34 @@ % Initialize a whole mess of control variables and storage volumes. % %Out of plane range inc (starts from 1.* inc) rotConvention = 'Bah'; -% Check and override the rotational convention to get helical averaging. -% Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = pBH.('doHelical'); -catch - doHelical = 0; -end -if ( doHelical ) - rotConvention = 'Helical' -end -rotConvention -if (use_new_grid_search) - gridSearch = eulerSearch(symmetry, angleSearch(1),... - angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, false); +if (emc.use_new_grid_search) + gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... + angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, false); + gridSearch.HelicalRestriction(emc.helical_search_theta_constraint); nAngles = sum(gridSearch.number_of_angles_at_each_theta); inPlaneSearch = gridSearch.parameter_map.psi; - - else - + if (emc.helical_search_theta_constraint ~= 0) + error('Helical search theta constraint not implemented for old grid search'); + end [ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch) + = BH_multi_gridSearchAngles(angleSearch) end - highThr=sqrt(2).*erfcinv(ceil(peakThreshold.*0.10).*2./(prod(size(tomogram)).*nAngles(1))) + [ OUTPUT ] = BH_multi_iterator( [targetSize; ... - size(tomogram);... - sizeTempBIN; ... - 2.*latticeRadius], 'convolution' ); + size(tomogram);... + sizeTempBIN; ... + 2.*latticeRadius], 'convolution' ); + - tomoPre = OUTPUT(1,:); tomoPost = OUTPUT(2,:); sizeChunk = OUTPUT(3,:); @@ -350,35 +271,17 @@ validCalc = OUTPUT(5,:); nIters = OUTPUT(6,:); + %[ padVal ] = BH_multi_padVal( sizeTemp, sizeChunk ); %tempPre = padVal(1,:); %tempPost = padVal(2,:); - [ padBIN ] = BH_multi_padVal( sizeTempBIN, sizeChunk ); [ trimValid ] = BH_multi_padVal(sizeChunk, validArea); -RMSFACTOR = sqrt(prod(sizeTempBIN) / prod(sizeChunk)); - -if ( tmpDecoy ) - % This is probably sample dependent. should search a small range and find - % the maximum rate of change in the ccc - - % the -1 searches for the next smallest fast fourier size - templateBIN = gpuArray(templateBIN); +RMSFACTOR = sqrt(prod(sizeTempBIN) / prod(sizeChunk)); - - decoyTest = BH_reScale3d(templateBIN,'',tmpDecoy,'GPU'); - decoyTrim = BH_multi_padVal(size(decoyTest),size(templateBIN)); - decoyTest = fftn(BH_padZeros3d(decoyTest,decoyTrim(1,:),decoyTrim(2,:),'GPU','single')); - decoyShift = -1.*gather(BH_multi_xcf_Translational(decoyTest,conj(fftn(templateBIN)),'',[3,3,3])); - decoyNorm = gather(sum(abs(decoyTest(:)))./sum(abs(fftn(templateBIN(:))))); - padDecoy = BH_multi_padVal(size(decoyTest),sizeChunk) + decoyTrim; - clear decoyTest - templateBIN = gather(templateBIN); - fprintf('tmpDecoy %f normFactor %f and shift by %2.2f %2.2f %2.2f\n',tmpDecoy,decoyNorm,decoyShift); -end fprintf('\n-----\nProcessing in chunks\n\n'); @@ -390,6 +293,7 @@ fprintf('# of iterations %d %d %d\n', nIters); fprintf('-----\n'); +valid_ratio = prod(sizeChunk) ./ prod(validCalc); size(tomogram) % [ tomogram ] = BH_padZeros3d(tomogram, tomoPre, tomoPost, ... @@ -399,24 +303,25 @@ sizeTomo = size(tomogram); -[ validAreaMask ] = gather(BH_mask3d('rectangle',sizeChunk,validCalc./2,[0,0,0])); +[ validCalcMask ] = BH_mask3d('rectangle',sizeChunk,validCalc./2,[0,0,0]); + + [ vA ] = BH_multi_padVal( validArea, sizeChunk ); % This would need to be changed to take a mask size and not just a radius. % Currently, this would not produce the correct results for odd size area % % % fftMask = BH_fftShift(validArea,sizeChunk,0); -% Array for storing chunk results -RESULTS_peak = zeros(sizeTomo, 'single'); +% Array for storing chunk results these could probably be half-precision +RESULTS_peak = zeros(sizeTomo, 'single'); RESULTS_angle= zeros(sizeTomo, 'single'); -if ( tmpDecoy ) - RESULTS_decoy = RESULTS_peak; -end -if (rescale_mip) - sum_of_x = zeros(sizeTomo, 'single'); - sum_of_x2 = zeros(sizeTomo, 'single'); + +RESULTS_sum = []; +RESULTS_sum_sq = []; +if (measure_noise_variance) + RESULTS_sum = zeros(sizeTomo, 'single'); + RESULTS_sum_sq = zeros(sizeTomo, 'single'); end -% Loop over tomogram -% Set this up second + % % % % optimize fft incase a power of two is not used, this will make things run ok. @@ -426,6 +331,7 @@ % % % clear opt ans [ bhF ] = fourierTransformer(randn(sizeChunk, 'single','gpuArray')); + sum_template = mean(templateBIN(:)); sum_templateMask = mean(templateMask(:)); sum_imgMask = prod(sizeChunk);% bhF.halfDimSize * sizeChunk(2) * sizeChunk(3); @@ -442,46 +348,17 @@ [ OUTPUT ] = BH_multi_iterator( [sizeChunk;kVal.*[1,1,1]], 'extrapolate' ); -% -% switch wedgeType -% case 1 -% % make a binary wedge -% [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... -% 'binaryWedgeGPU',particleThickness,... -% 1, 1, samplingRate); -% case 2 -% % make a non-CTF wedge -% [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... -% 'applyMask',particleThickness,... -% 2, 1, samplingRate); -% case 3 -% % make a CTF without exposure weight -% [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... -% 'applyMask',particleThickness,... -% 3, 1, samplingRate); -% case 4 -% % make a wedge with full-ctf -% [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... -% 'applyMask',particleThickness,... -% 4, 1, samplingRate); -% otherwise -% error('wedgeType must be 1-4'); -% end -% -% wedgeMask = (ifftshift(wedgeMask)); -% -% % Now just using the mask to calculate the power remaining in the template, -% % without actually applying. -% wedgeMask = gather(find(ifftshift(wedgeMask))); - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Preprocess the tomogram tomoIDX = 1; nTomograms = prod(nIters); -tomoStack = zeros([sizeChunk,nTomograms], 'single'); +wanted_storage_precision = 'single'; +if (test_half) + wanted_storage_precision = 'uint16'; +end + +tomoStack = zeros([sizeChunk,nTomograms], wanted_storage_precision); % tomoNonZero = zeros(nTomograms,6,'uint64'); % backgroundVol = zeros(sizeChunk,'single'); @@ -489,7 +366,7 @@ try - doMedFilt = pBH.('Tmp_medianFilter'); + doMedFilt = emc.('Tmp_medianFilter'); if ~ismember(doMedFilt,[3,5,7]) error('Tmp_medianFilter can only be 3,5, or 7'); else @@ -519,516 +396,322 @@ cutY = 1 + (iY-1).*validArea(2); for iZ = 1:nIters(3) cutZ = 1 + (iZ-1).*validArea(3); - - fprintf('preprocessing tomo_chunk %d/%d col %d/%d row %d/%d plane idx%d\n' , ... - iY,nIters(2),iX,nIters(1),iZ,nIters(3),tomoIDX) - - - - tomoChunk = gpuArray(tomogram(cutX:cutX+sizeChunk(1)-1,... - cutY:cutY+sizeChunk(2)-1,... - cutZ:cutZ+sizeChunk(3)-1)); - - % Make a list of the padded regions of the tomogram to exclude from - % statistical calculations - - tomoChunk = tomoChunk - mean(tomoChunk(:)); - tomoChunk = tomoChunk ./ rms(tomoChunk(:)); - -% tomoChunk = real(ifftn(fftn(tomoChunk).*tomoBandpass)); - - tomoChunk = bhF.invFFT(bhF.fwdFFT(tomoChunk,0,0,[bp_vals, pixelSize]),2); - - - if doMedFilt - if ( flgOOM ) - tomoChunk = (medfilt3(tomoChunk,doMedFilt.*[1,1,1])); + + fprintf('preprocessing tomo_chunk %d/%d col %d/%d row %d/%d plane idx%d\n' , ... + iY,nIters(2),iX,nIters(1),iZ,nIters(3),tomoIDX) + + + + tomoChunk = gpuArray(tomogram(cutX:cutX+sizeChunk(1)-1,... + cutY:cutY+sizeChunk(2)-1,... + cutZ:cutZ+sizeChunk(3)-1)); + + % Make a list of the padded regions of the tomogram to exclude from + % statistical calculations + + tomoChunk = tomoChunk - mean(tomoChunk(:)); + tomoChunk = tomoChunk ./ rms(tomoChunk(:)); + + % tomoChunk = real(ifftn(fftn(tomoChunk).*tomoBandpass)); + + tomoChunk = bhF.invFFT(bhF.fwdFFT(tomoChunk,0,0,[bp_vals, pixelSize]),2); + + + if doMedFilt + if ( flgOOM ) + tomoChunk = (medfilt3(tomoChunk,doMedFilt.*[1,1,1])); + else + tomoChunk = gpuArray(medfilt3(tomoChunk,doMedFilt.*[1,1,1])); + end else - tomoChunk = gpuArray(medfilt3(tomoChunk,doMedFilt.*[1,1,1])); - end - else - if ( flgOOM ) - % Leave on CPU - else - tomoChunk = gpuArray(tomoChunk); - statsRadius = gather(statsRadius); + if ( flgOOM ) + % Leave on CPU + else + tomoChunk = gpuArray(tomoChunk); + statsRadius = gather(statsRadius); + end end - end - - [ averageMask, flgOOM ] = BH_movingAverage_2(tomoChunk, statsRadius(1)); - rmsMask = BH_movingAverage_2(tomoChunk.^2, statsRadius(1)); - rmsMask = sqrt(rmsMask - averageMask.^2); - tomoChunk = (tomoChunk - averageMask) ./ rmsMask; - clear rmsMask averageMask -% averageMask = gather(averageMask); - -% [ rmsMask ] = gather(BH_movingRMS_3(tomoChunk, statsRadius(1), averageMask)); - - -% tomoChunk = tomoChunk - averageMask; -% tomoChunk = tomoChunk ./ rmsMask; -% if (save_average_filtered) -% tomoChunk = gpuArray(tomogram(cutX:cutX+sizeChunk(1)-1,... -% cutY:cutY+sizeChunk(2)-1,... -% cutZ:cutZ+sizeChunk(3)-1)); -% avgFiltRec = zeros(size(tomogram),'single'); -% end -% statsRadius(1) -% [ rmsMask ] = BH_movingRMS_2(tomoChunk-averageMask, statsRadius(1)); -% statsRadius(1) -% tomoChunk = tomoChunk ./ rmsMask; -% figure, imshow3D(BH_padZeros3d(gather(averageMask),'fwd',trimValid,'cpu','single')) -% figure, imshow3D(BH_padZeros3d(gather(rmsMask),'fwd',trimValid,'cpu','single')) -% return -% clear avgerageMask - - - tomoChunk = gather(((-1*shouldBeCTF) .* tomoChunk )).*validAreaMask; - -% tomoChunk = tomoChunk .* (-1*shouldBeCTF); % This is backwards, but I don't know why - tmp_sum = sum(tomoChunk(validAreaMask > 0.1)); - fullX = fullX + tmp_sum; - fullX2 = fullX2 + gather(tmp_sum.^2); - fullnX = fullnX + gather(prod(sizeChunk)); - - tomoStack(:,:,:,tomoIDX) = tomoChunk; - - tomoCoords(tomoIDX,:) = [cutX,cutY,cutZ]; - tomoIDX = tomoIDX + 1; + + [ averageMask, flgOOM ] = BH_movingAverage_2(tomoChunk, statsRadius(1)); + rmsMask = BH_movingAverage_2(tomoChunk.^2, statsRadius(1)); + rmsMask = sqrt(rmsMask - averageMask.^2); + + tomoChunk = (tomoChunk - averageMask) ./ rmsMask; + clear rmsMask averageMask + + tomoChunk = gather(tomoChunk .*validCalcMask); + + tmp_sum = sum(tomoChunk(:)); + + fullX = fullX + gather(tmp_sum); + fullX2 = fullX2 + gather(tmp_sum.^2); + fullnX = fullnX + gather(prod(sizeChunk)); + + if (test_half) + % The default is to return uint16 on the same device (host in this case) + tomoStack(:,:,:,tomoIDX) = emc_halfcast(tomoChunk); + else + tomoStack(:,:,:,tomoIDX) = tomoChunk; + end + + tomoCoords(tomoIDX,:) = [cutX,cutY,cutZ]; + tomoIDX = tomoIDX + 1; end % end of loop over Z chunks end % end of loop over Y chunks end % end of loop over X chunks + % Normalize the global variance globalVariance = (fullX2/fullnX) - (fullX/fullnX)^2; -%fprintf('After local normalization, scaling also the global variance %3.3e\n',globalVariance); -for iChunk = 1:tomoIDX-1 - tomoStack(:,:,:,iChunk) = tomoStack(:,:,:,iChunk) ./ sqrt(globalVariance); -end - -clear tomoWedgeMask bandpassFilter statBinary validAreaMask tomoChunk +clear tomoWedgeMask validCalcMask bandpassFilter statBinary tomoChunk %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% kVal = 0; - -currentGlobalAngle = 1; + ANGLE_LIST = zeros(nAngles(1),3, 'single'); -nComplete = 0; totalTime = 0; firstLoopOverTomo = true; - % Center the spectrum by multiplication not swapping (this should just - % be in the fourierTransformer class if it isn't already) - % swapPhase(obj, inputVol, direction) with fwd should do it - [dU,dV,dW] = BH_multi_gridCoordinates(size(tomoStack(:,:,:,1)),... - 'Cartesian','GPU', ... - {'none'},1,1,0); - - swapQuadrants = exp((-2i*pi).*(dU.*(floor(size(dU,1)/2)+1) + ... - (dV.*(floor(size(dV,2)/2)+1) + ... - (dW.*(floor(size(dW,3)/2)+1))))); - clear dU dV dW - - swapQuadrants = swapQuadrants(1:floor(size(swapQuadrants,1)/2)+1,:,:); -if (use_new_grid_search) - theta_search = 1:gridSearch.number_of_out_of_plane_angles; +if (emc.use_new_grid_search) + theta_search = gridSearch.active_theta_positions; else theta_search = 1:size(angleStep,1); end -for iAngle = theta_search - - if (use_new_grid_search) - theta = gridSearch.parameter_map.theta(iAngle); - numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); - else - theta = angleStep(iAngle,1); - phiStep = angleStep(iAngle,3); - numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - end +tomoIDX = 1; +firstLoopOverAngles = true; + +% Avoid repeated allocations +tempPAD = zeros(size(templateBIN) + padBIN(1,:) + padBIN(2,:),'single','gpuArray'); - tempImg = gpuArray(templateBIN); %%%%% NEW switch to bin +use_only_once = false; +template_interpolator = ''; +[template_interpolator, ~] = interpolator(gpuArray(templateBIN),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', use_only_once); - - interpolationNormFactor = sum(abs(tempImg(:)).^2); +% templateMask_interpolator = ''; +% [templateMask_interpolator, ~] = interpolator(gpuArray(templateMask),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', use_only_once); +for iTomo = 1:nTomograms + currentGlobalAngle = 1; + currentSearchPosition = 0; - clear referenceStack tempFilter - % Calculate all references for each out of plane tilt only once - referenceStack = zeros([sizeTempBIN,numRefIter], 'single', 'gpuArray'); - - tomoIDX = 1; - firstLoopOverAngle = true; + fprintf('Working on tomo chunk %d/%d from %s\n', iTomo, nTomograms, mapName); - % Avoid repeated allocations - tempPAD = zeros(size(tempImg) + padBIN(1,:) + padBIN(2,:),'single','gpuArray'); - tempPADMask = tempPAD; - template_interpolator = ''; - [template_interpolator, ~] = interpolator(tempImg,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + % Iterate over the tomogram pulling each chunk one at a time. + % for iTomo = 1:nTomograms sqp loop + tic; + iCut = tomoCoords(iTomo,:); + % reset the angle count and value at the begining of loop + % inside, while each new outer loop changes the start values. - templateMask_interpolator = ''; - [templateMask_interpolator, ~] = interpolator(gpuArray(templateMask),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - - - - - % Iterate over the tomogram pulling each chunk one at a time. - for iTomo = 1:nTomograms - tic; - iCut = tomoCoords(tomoIDX,:); - % reset the angle count and value at the begining of loop - % inside, while each new outer loop changes the start values. - -% nAngle = angleIncStart; - intraLoopAngle = 1; - - % Truth value to initialize temp results matrix each new tomo - % chunk. - firstLoopOverChunk = true; + % Truth value to initialize temp results matrix each new tomo + % chunk. + firstLoopOverChunk = true; - if (use_new_grid_search) - fprintf('Working on tilt(%d/%d) tomoChunk(idx%d/%d)\t' ... - ,iAngle,gridSearch.number_of_out_of_plane_angles, tomoIDX,nTomograms); - else - fprintf('working on tilt(%d/%d) tomoChunk(idx%d/%d)\t' ... - ,iAngle,size(angleStep,1), tomoIDX,nTomograms); - end - - - tomoFou = gpuArray(tomoStack(:,:,:,tomoIDX)); - tomoFou = swapQuadrants.*bhF.fwdFFT(tomoFou); + + if (test_half) + % Convert and return on GPU + tomoFou = emc_halfcast(tomoStack(:,:,:,iTomo), true); + else + tomoFou = gpuArray(tomoStack(:,:,:,iTomo)); + end + tomoFou = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)), 'fwd'); + for iAngle = theta_search - if (use_new_grid_search) + + if (emc.use_new_grid_search) + theta = gridSearch.parameter_map.theta(iAngle); + else + theta = angleStep(iAngle,1); + phiStep = angleStep(iAngle,3); + end + + if (emc.use_new_grid_search) phi_search = gridSearch.parameter_map.phi{iAngle}; else phi_search = 0:angleStep(iAngle,2); end for iAzimuth = phi_search + % currentSearchPosition = currentSearchPosition + 1; + % fprintf('Working search position %d/%d for tomo chunk %d/%d from %s\n',currentSearchPosition, nAngles/length(inPlaneSearch), iTomo, nTomograms, mapName); - if (use_new_grid_search) + if (emc.use_new_grid_search) phi = iAzimuth; else - phi = phiStep * iAzimuth; + phi = phiStep * iAzimuth; end - + for iInPlane = inPlaneSearch + psi = iInPlane; - + %calc references only on first chunk - if (firstLoopOverAngle) - + if (firstLoopOverAngles) ANGLE_LIST(currentGlobalAngle,:) = [phi, theta, psi - phi]; - end - - [ tempRot ] = template_interpolator.interp3d(... - [phi, theta, psi - phi],... - [1,1,1],rotConvention,... - 'forward','C1'); - - - - tempPAD = tempPAD .* 0; - tempPAD(padBIN(1,1)+1: end - padBIN(2,1), ... - padBIN(1,2)+1: end - padBIN(2,2), ... - padBIN(1,3)+1: end - padBIN(2,3)) = tempRot; - - tempFou = conj(bhF.fwdFFT(tempPAD)); - - - ccfmap = BH_padZeros3d((real(single(... - bhF.invFFT(tomoFou.*tempFou)))),...%./(tomoNorm.*tempNorm))))),... - trimValid(1,:),trimValid(2,:),'GPU','single'); -% + + + % rather than using padzeros + tempPAD = tempPAD .* 0; + tempPAD(padBIN(1,1)+1: end - padBIN(2,1), ... + padBIN(1,2)+1: end - padBIN(2,2), ... + padBIN(1,3)+1: end - padBIN(2,3)) = template_interpolator.interp3d(... + [phi, theta, psi - phi],... + [0,0,0],rotConvention,... + 'forward','C1'); + + + tempPAD = tempPAD - mean(tempPAD(:)); + + + ccfmap = BH_padZeros3d(real(single(... + bhF.invFFT(tomoFou.* conj(bhF.fwdFFT(tempPAD))))),...%./(tomoNorm.*tempNorm))))),... + trimValid(1,:),trimValid(2,:),'GPU','single'); + % + ccfmap = ccfmap ./ std(ccfmap(:)); - if ( tmpDecoy > 0 ) - - if (firstLoopOverAngle) - - decoy = BH_padZeros3d(BH_reScale3d(tempRot./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - padDecoy(1,:),padDecoy(2,:),'GPU','single'); - else - % Probably just make a second decoy stack to avoid - % re-interpolating. If it works, then do this. - error('This is temp broken with new interpolator'); -% decoy = BH_padZeros3d(BH_reScale3d(referenceStack(:,:,:,intraLoopAngle)./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... -% padDecoy(1,:),padDecoy(2,:),'GPU','single'); - end - - - - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*conj(fftn(decoy)))))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU','single'); - - - elseif ( tmpDecoy < 0 ) - - % Just use the mirror image of the template, i.e. take the conj - % (of the conj) so just the padded FFT of the ref. - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*tempFou)))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU','single'); - - end - clear tempRot + % If first loop over tomo, initialize the storage volumes, if % first loop over the chunk but not over the tomo, pull storage % chunks from storage volume. - if (firstLoopOverTomo && firstLoopOverChunk) - %store ccfmap as complex with phase = angle of reference - magTmp = ccfmap; - if ( tmpDecoy ) - decoyTmp = decoy; - end - angTmp = ones(size(magTmp), 'single','gpuArray'); - - if (rescale_mip) - sum_of_x_tmp = ccfmap; - sum_of_x2_tmp = ccfmap.^2; - end - - firstLoopOverTomo = false; - firstLoopOverChunk = false; - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - - elseif (firstLoopOverChunk) - % These double cuts are old, and don't really make sense. Make - % this more consistant with current operations when there is - % time. - magTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - if ( tmpDecoy ) - decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyTmp = gpuArray(decoyTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); + if (firstLoopOverChunk) + %store ccfmap as complex with phase = angle of reference + magTmp = ccfmap; + angTmp = ones(size(magTmp), 'single','gpuArray'); + + if (measure_noise_variance) + ccfmap(abs(ccfmap) > 3) = 0; + sumTmp = ccfmap; + sumSqTmp = ccfmap.^2; end - - magTmp = gpuArray(magTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - firstLoopOverChunk = false; - + else + % update higher values of ccfmap with new reference if applicable. replaceTmp = ( magTmp < ccfmap ); magTmp(replaceTmp) = ccfmap(replaceTmp); angTmp(replaceTmp) = currentGlobalAngle; + + if (measure_noise_variance) + ccfmap(abs(ccfmap) > 3) = 0; + sumTmp = sumTmp + ccfmap; + sumSqTmp = sumSqTmp + ccfmap.^2; + end + - if (rescale_mip) - - sum_of_x_tmp = sum_of_x(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sum_of_x_tmp = gpuArray(sum_of_x_tmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - sum_of_x_tmp = sum_of_x_tmp + ccfmap; - - sum_of_x2_tmp = sum_of_x2(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sum_of_x2_tmp = gpuArray(sum_of_x2_tmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - sum_of_x2_tmp = sum_of_x2_tmp + ccfmap.^2; - end - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; clear replaceTmp - - else - % update higher values of ccfmap with new reference if applicable. - - - replaceTmp = ( magTmp < ccfmap ); - - - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - if ( tmpDecoy ) - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - - if (rescale_mip) - sum_of_x_tmp = sum_of_x_tmp + ccfmap; - sum_of_x2_tmp = sum_of_x2_tmp + ccfmap.^2; - end - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp - end - nComplete = nComplete + 1; - end - end - - % After searching all angles on this chunk, but out meaningful - % portion for storage. - + end % end if firstLoopOverChunk + currentGlobalAngle = currentGlobalAngle + 1; + + end % end psi loop over in plane angles + end % end phi loop over azimuth angles + end % end theta loop over out of plane angles + + % FIXME this double cutting and temporary allocation is ridiculous. + magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + + + magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); + angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); + + + RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; + + clear magStoreTmp + + RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; + clear angStoreTmp + + if (measure_noise_variance) + sumStoreTmp = RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + sumSqStoreTmp = RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); - % FIXME this double cutting and temporary allocation is ridiculous. - magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - - - magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); - angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); - - - RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; - - clear magStoreTmp - - RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; - clear angStoreTmp - if ( tmpDecoy ) - decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(decoyTmp); - RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = decoyStoreTmp; - - end + sumStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(sumTmp); + sumSqStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(sumSqTmp); - if ( rescale_mip ) - sum_of_x_store = sum_of_x(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sum_of_x_store(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sum_of_x_tmp); - sum_of_x(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sum_of_x_store; - clear sum_of_x_store - - sum_of_x2_store = sum_of_x2(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sum_of_x2_store(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sum_of_x2_tmp); - sum_of_x2(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sum_of_x2_store; - clear sum_of_x2_store - end - tomoTime = toc; - totalTime = totalTime + toc; timeEstimate = totalTime * (nTomograms*nAngles(1)./(nComplete-1)); - fprintf('elapsed time = %f s est remain %f s\n', tomoTime, timeEstimate); - tomoIDX = tomoIDX + 1; - firstLoopOverAngle = false; - currentGlobalAngle = currentGlobalAngle - intraLoopAngle + 1; - end - - currentGlobalAngle = currentGlobalAngle + intraLoopAngle - 1; + + RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = sumStoreTmp; + + clear sumStoreTmp + + RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = sumSqStoreTmp; + clear sumSqStoreTmp + + firstLoopOverAngles = false; + end end %save('angle_list.txt','angle_list','-ascii'); clear tomoStack % Cut out the post padding used to iterate over the tomogram RESULTS_peak = RESULTS_peak(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); -%RESULTS_peak(RESULTS_peak < 0) = 0; + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); +%RESULTS_peak(RESULTS_peak < 0) = 0; RESULTS_angle = RESULTS_angle(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); - -if ( tmpDecoy ) - RESULTS_decoy = RESULTS_decoy(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); -% RESULTS_decoy = RESULTS_decoy ./ std(RESULTS_decoy(:)); - RESULTS_decoy(RESULTS_decoy < 1) = 1; - + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); + +if (measure_noise_variance) + RESULTS_sum = RESULTS_sum(1+tomoPre(1):end-tomoPost(1),... + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); + RESULTS_sum_sq = RESULTS_sum_sq(1+tomoPre(1):end-tomoPost(1),... + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); end -if ( rescale_mip ) - sum_of_x = sum_of_x(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)) ./ currentGlobalAngle; - - sum_of_x2 = sum_of_x2(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)) ./ currentGlobalAngle; - - %SAVE_IMG(sum_of_x,'sum_of_x.mrc'); - %SAVE_IMG(sum_of_x2,'sum_of_x2.mrc'); - %SAVE_IMG(RESULTS_peak,'prescaling.mrc'); - - RESULTS_peak = RESULTS_peak - sum_of_x; - sum_of_x = sqrt(sum_of_x2 - sum_of_x.^2); - clear sum_of_x2; -% SAVE_IMG(sum_of_x,'stddev.mrc'); -% mov = mean(sum_of_x(:)); -% % sov = std(sum_of_x(:)); -% sov = 0; -% sum_of_x(sum_of_x < (mov - 1*sov)) = max(sum_of_x(:)); -% SAVE_IMG(sum_of_x,'stddev_clipped.mrc'); - - RESULTS_peak = RESULTS_peak ./ sum_of_x; - - RESULTS_peak = RESULTS_peak - mean(RESULTS_peak(:)); - RESULTS_peak = RESULTS_peak ./ rms(RESULTS_peak(:)); - clear sum_of_x; -end + + + gpuDevice(useGPU); clear bhF @@ -1048,17 +731,18 @@ resultsOUT = sprintf('./%s/%s_convmap.mrc',convTMPNAME,mapName); anglesOUT = sprintf('./%s/%s_angles.mrc',convTMPNAME,mapName); angleListOUT = sprintf('./%s/%s_angles.list',convTMPNAME,mapName); -SAVE_IMG(MRCImage(mag),resultsOUT); -SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); -if ( tmpDecoy ) - decoyOUT = sprintf('./%s/%s_decoy.mrc',convTMPNAME,mapName); - SAVE_IMG(MRCImage((RESULTS_decoy)),decoyOUT); - diffOUT = sprintf('./%s/%s_convmap-decoy.mrc',convTMPNAME,mapName); - decoyLogical = mag < RESULTS_decoy; - mag(decoyLogical) = 0; - mag(~decoyLogical) = mag(~decoyLogical) - RESULTS_decoy(~decoyLogical); clear RESULTS_decoy - SAVE_IMG(MRCImage((mag)),diffOUT); +SAVE_IMG(mag,{resultsOUT,'half'}); +noiseVarOUT = sprintf('./%s/%s_noise_variance.mrc',convTMPNAME,mapName); + +if (measure_noise_variance) + n_angles_searched = sum(any(ANGLE_LIST,2)); + noiseVar = (RESULTS_sum_sq./n_angles_searched - (RESULTS_sum./n_angles_searched).^2); + noiseVar(noiseVar == 0) = 1; + + SAVE_IMG(noiseVar,{noiseVarOUT,'half'}); end +% SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); + angleFILE = fopen(angleListOUT,'w'); fprintf(angleFILE,'%2.2f\t%2.2f\t%2.2f\n', ANGLE_LIST'); fclose(angleFILE); @@ -1070,8 +754,8 @@ % out and padding back in.) Also pad by size of removal mask (subtract this from % coordinates) mag = mag(szK(1)+1:end - szK(1), ... - szK(2)+1:end - szK(2), ... - szK(3)+1:end - szK(3)); + szK(2)+1:end - szK(2), ... + szK(3)+1:end - szK(3)); mag = BH_padZeros3d(mag,szK+rmDim,szK+rmDim, 'cpu', 'single'); %dev.FreeMemory; %%%Ang = angle(RESULTS_peak); %clear Results @@ -1094,14 +778,14 @@ [MAX, coord] = max(mag(:)); -peakMat = zeros(peakThreshold,10*nPeaks); +peakMat = zeros(peakThreshold,10*emc.nPeaks); n = 1; fprintf('rmDim %f szK %f\n', rmDim,szK); removalMask = BH_mask3d(eraseMaskType,[2,2,2].*rmDim+1,eraseMaskRadius,[0,0,0]); rmInt = interpolator(gpuArray(removalMask),[0,0,0],[0,0,0],rotConvention ,'forward','C1'); -symOps = interpolator(gpuArray(removalMask),[0,0,0],[0,0,0],rotConvention ,'forward',symmetry); +symOps = interpolator(gpuArray(removalMask),[0,0,0],[0,0,0],rotConvention ,'forward',emc.symmetry); maskCutOff = 0.98; nIncluded = gather(sum(sum(sum(removalMask > maskCutOff)))); @@ -1119,7 +803,7 @@ if (nTries > 1000) error('Did not find an appropriate erase mask'); end - + end if ignore_threshold @@ -1127,80 +811,86 @@ end this_try = 0; -while n <= 2.*peakThreshold && (this_try < max_tries) && MAX > highThr -this_try = this_try + 1; - -% -% Some indicies come back as an error, even when they seem like the -% should be fine. I'm not sure why, and I should think about this -% more, but for now, just set that one index to zero (instead of a -% whole box) and move on with life. It looks like the index that is -% kicking out the error is equal to -1*numberofreferences, which -% might be an issue because that corresonds to the positive upper -% limit of the reference index. Ignoring it still seems to be okay -% but it bothers me not to know. - - -[i,j,k] = ind2sub(sizeTomo,coord); -try - c = gather([i,j,k]); -catch - print('Ran into some trouble gathering the i,j,k. Breaking out\n'); - break +if (ignore_threshold) + search_limit = peakThreshold +else + search_limit = 2 .* peakThreshold end +while n <= search_limit && (this_try < max_tries) && MAX > highThr + this_try = this_try + 1; + + % + % Some indicies come back as an error, even when they seem like the + % should be fine. I'm not sure why, and I should think about this + % more, but for now, just set that one index to zero (instead of a + % whole box) and move on with life. It looks like the index that is + % kicking out the error is equal to -1*numberofreferences, which + % might be an issue because that corresonds to the positive upper + % limit of the reference index. Ignoring it still seems to be okay + % but it bothers me not to know. + + + [i,j,k] = ind2sub(sizeTomo,coord); + try + c = gather([i,j,k]); + catch + fprint('Ran into some trouble gathering the i,j,k. Breaking out\n'); + break + end + if Ang(gather(coord)) > 0 - + % box for removal and center of mass calc, use a larger box if multiple % peaks are being saved. - bDist = 1+round(log(nPeaks)); + bDist = 1+round(log(emc.nPeaks)); clI = c(1) - bDist; chI = c(1) + bDist; clJ = c(2) - bDist; chJ = c(2) + bDist; clK = c(3) - bDist; chK = c(3) + bDist; - + magBox = mag(clI:chI,clJ:chJ,clK:chK); angBox = Ang(clI:chI,clJ:chJ,clK:chK); - + [cmX, cmY, cmZ] = ndgrid(-1*bDist:1*bDist, ... - -1*bDist:1*bDist, ... - -1*bDist:1*bDist ); - - cMass = [ sum(sum(sum(magBox.*cmX))) ; ... - sum(sum(sum(magBox.*cmY))) ; ... - sum(sum(sum(magBox.*cmZ))) ] ./ sum(magBox(:)); - - + -1*bDist:1*bDist, ... + -1*bDist:1*bDist ); + + cMass = [ sum(sum(sum(magBox.*cmX))) ; ... + sum(sum(sum(magBox.*cmY))) ; ... + sum(sum(sum(magBox.*cmZ))) ] ./ sum(magBox(:)); + + % Switching from centered to lower left coordinates and subtracting the - % padding + % padding cenP = c + cMass' - rmDim; - - -% % % % If the most frequent peak is unique use it; -% % % [peakM, ~, peakC] = mode(angBox(:)); -% % % if length(peakC) == 1 && peakM -% % % % Need to ensure the mode is none zero which is possible. -% % % peakMat(n,4:6) = ANGLE_LIST(peakM,:); -% % % topPeak = peakM; -% % % else - % Otherwise use the value at the max for the peak val; - peakMat(n,4:6) = ANGLE_LIST(Ang(coord),:); - topPeak = Ang(coord); -% % % end + + + % % % % If the most frequent peak is unique use it; + % % % [peakM, ~, peakC] = mode(angBox(:)); + % % % if length(peakC) == 1 && peakM + % % % % Need to ensure the mode is none zero which is possible. + % % % peakMat(n,4:6) = ANGLE_LIST(peakM,:); + % % % topPeak = peakM; + % % % else + % Otherwise use the value at the max for the peak val; + peakMat(n,4:6) = ANGLE_LIST(Ang(coord),:); + topPeak = Ang(coord); + % % % end peakMat(n,1:3) = gather(samplingRate.*cenP); peakMat(n,10) = gather(MAX); - + iSNR = 0; - if nPeaks > 1 + if emc.nPeaks > 1 possible_angles = gather(magBox); - possible_angles(angBox == topPeak) = 0; + possible_angles(angBox == topPeak) = 0; nRandom = 2; - for iPeak = 2:nPeaks + for iPeak = 2:emc.nPeaks useRandom = false; @@ -1208,7 +898,7 @@ [~, cAng] = max(possible_angles(:)); topPeak = angBox(cAng); iSNR = gather(mean( possible_angles(angBox == topPeak))); - possible_angles(angBox == topPeak) = 0; + possible_angles(angBox == topPeak) = 0; Ang(cAng) if topPeak <= 0 || Ang(cAng) <= 0 useRandom = true; @@ -1226,52 +916,52 @@ % If we've used up all the possible peaks, just insert a random % Incrementally far from the original iAngles = [ randn(1) .* (nRandom.^2) + peakMat(n,1), ... - randn(1) .* (nRandom.^2) + peakMat(n,2), ... - randn(1) .* (nRandom.^2) + peakMat(n,3)]; + randn(1) .* (nRandom.^2) + peakMat(n,2), ... + randn(1) .* (nRandom.^2) + peakMat(n,3)]; if nRandom < 10 - nRandom = nRandom + 1; + nRandom = nRandom + 1; end end peakMat(n,[1:3]+10*(iPeak-1)) = gather(samplingRate.*cenP); - peakMat(n,[4:6]+10*(iPeak-1)) = iAngles; + peakMat(n,[4:6]+10*(iPeak-1)) = iAngles; peakMat(n,10+10*(iPeak-1)) = iSNR; -% % % oldPeaks = ( angBox == peakM | oldPeaks ); + % % % oldPeaks = ( angBox == peakM | oldPeaks ); end end - - - -% rmMask = BH_resample3d(removalMask,peakMat(n,4:6),[0,0,0],rotConvention ,'GPU','forward'); + + + + % rmMask = BH_resample3d(removalMask,peakMat(n,4:6),[0,0,0],rotConvention ,'GPU','forward'); rmMask = rmInt.interp3d(gather(peakMat(n,4:6)),[0,0,0],rotConvention,'forward','C1'); - + % Invert after resampling so that zeros introduced by not extrapolating % the corners are swapped to ones, i.e. not removed. -% rmMask = (1-rmMask); + % rmMask = (1-rmMask); mag(c(1)-rmDim:c(1)+rmDim,... - c(2)-rmDim:c(2)+rmDim,... - c(3)-rmDim:c(3)+rmDim) = ... - mag(c(1)-rmDim:c(1)+rmDim,... - c(2)-rmDim:c(2)+rmDim,... - c(3)-rmDim:c(3)+rmDim) .* (rmMask< maskCutOff); - -% % % peakMat(n,10) = (gather(MAX) - Tmean)./Tstd; % record stds above mean + c(2)-rmDim:c(2)+rmDim,... + c(3)-rmDim:c(3)+rmDim) = ... + mag(c(1)-rmDim:c(1)+rmDim,... + c(2)-rmDim:c(2)+rmDim,... + c(3)-rmDim:c(3)+rmDim) .* (rmMask< maskCutOff); + + % % % peakMat(n,10) = (gather(MAX) - Tmean)./Tstd; % record stds above mean n = n + 1; if ~mod(n,100) n end - + else - Ang(gather(coord)); - mag(coord) = 0; + Ang(gather(coord)); + mag(coord) = 0; end - - -[MAX, coord] = max(mag(:)); - + + + [MAX, coord] = max(mag(:)); + end peakMat = peakMat( ( peakMat(:,1)>0 ),:); @@ -1288,56 +978,43 @@ errID = fopen(sprintf('./%s/%s.errID',convTMPNAME,mapName)); - -n=1 +n=1; +nSym=1; for i = 1:length(peakMat(:,1)) - if all(peakMat(i,1:3)) + if all(peakMat(i,1:3)) + + iSym = mod(nSym,symOps.nSymMats)+1; + % Generate a uniform distribution over the in-plane + % randomizations + + r = reshape(BH_defineMatrix(peakMat(i,4:6), rotConvention , 'inv') * symOps.symmetry_matrices{iSym},1,9); + nSym = nSym + 1; + fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... + '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... + i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... + peakMat(i,4:6),r,1); + + if emc.nPeaks > 1 + for iPeak = 2:emc.nPeaks - if SYMMETRY > 1 - % Generate a uniform distribution over the in-plane - % randomizations - iSym = rem( n + SYMMETRY, SYMMETRY)+1; - r = reshape(BH_defineMatrix(peakMat(i,4:6), rotConvention , 'inv') *... - symOps.symmetry_matrices{iSym},1,9); - else - r = reshape(BH_defineMatrix(peakMat(i,4:6), rotConvention , 'inv'),1,9); - end + iSym = mod(nSym,symOps.nSymMats)+1; + r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), rotConvention , 'inv') * symOps.symmetry_matrices{iSym},1,9); + nSym = nSym + 1; fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... - peakMat(i,4:6),r,1); - - if nPeaks > 1 - - for iPeak = 2:nPeaks - if SYMMETRY > 1 - % Generate a uniform distribution over the in-plane - % randomizations - iSym = rem( n + SYMMETRY, SYMMETRY)+1; - r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), rotConvention , 'inv')*... - symOps.symmetry_matrices{iSym},1,9); - else - r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), rotConvention , 'inv'),1,9); - end - fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,[1:3]+10*(iPeak-1)), ... - peakMat(i,[4:6]+10*(iPeak-1)),r,1); - end - - - end - - fprintf(fileID,'\n'); - - - - - fprintf(fileID2,'%f %f %f\n',peakMat(i,1:3)./samplingRate); - + '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... + i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,[1:3]+10*(iPeak-1)), ... + peakMat(i,[4:6]+10*(iPeak-1)),r,1); + - n = n +1; - end + end + end + + fprintf(fileID,'\n'); + fprintf(fileID2,'%f %f %f\n',peakMat(i,1:3)./samplingRate); + + + n = n + 1; + end end %lastIndex = find(fieldOUT(:,4),1,'last'); @@ -1345,6 +1022,8 @@ fclose(fileID); fclose(fileID2); + + system(sprintf('point2model -number 1 -sphere 3 -scat ./%s/%s.pos ./%s/%s.mod', convTMPNAME,mapName,convTMPNAME, mapName)); fileID = fopen(sprintf('./%s/%s.path',convTMPNAME,mapName),'w'); @@ -1352,9 +1031,8 @@ fclose(fileID); +fprintf('Total execution time : %f seconds\n', seconds(datetime("now")-startTime)); -fprintf('Total execution time : %f seconds\n', etime(clock, startTime)); - end % end of templateSearch3d function diff --git a/alignment/emC_autoAlign b/alignment/emC_autoAlign index 7237b54d..053028a6 100755 --- a/alignment/emC_autoAlign +++ b/alignment/emC_autoAlign @@ -306,6 +306,7 @@ echo $iEcho && iEcho=$(($iEcho+1)) -AngleOffset ${tiltAngleOffset} > tiltAlign.log else # run without local until an intial stable global solution is found + echo "${EMC_TILTALIGN} -ModelFile ${pName}.fid -ImageFile ${pName}.preali -ImagesAreBinned ${iBin} -OutputModelFile ${pName}.3dmod -OutputResidualFile ${pName}.resid -OutputFidXYZFile ${pName}fid.xyz -OutputTiltFile ${pName}.tlt -OutputTransformFile ${pName}.tltxf_nonScaled -RotationAngle 0.0 -TiltFile ${pName}.rawtlt -AngleOffset ${tiltAngleOffset} -RotOption 1 -RotDefaultGrouping 3 -TiltOption ${TILT_OPTION} -TiltDefaultGrouping 3 -MagOption ${MAG_OPTION} -MagDefaultGrouping 3 -BeamTiltOption 0 -ResidualReportCriterion 1.0 -SurfacesToAnalyze 1 -MetroFactor 0.25 -MaximumCycles 1000 -KFactorScaling 1.0 -NoSeparateTiltGroups 2 -AxisZShift 1000 > tiltAlign.log" > ./.${inp}_tilalign.sh tiltalign \ -ModelFile ${pName}.fid \ -ImageFile ${pName}.preali \ diff --git a/backend/__init__.py b/backend/__init__.py new file mode 100644 index 00000000..12a5da15 --- /dev/null +++ b/backend/__init__.py @@ -0,0 +1 @@ +# emClarity FastAPI backend diff --git a/backend/api/__init__.py b/backend/api/__init__.py new file mode 100644 index 00000000..d6360184 --- /dev/null +++ b/backend/api/__init__.py @@ -0,0 +1 @@ +# API route modules diff --git a/backend/api/jobs.py b/backend/api/jobs.py new file mode 100644 index 00000000..b05fe592 --- /dev/null +++ b/backend/api/jobs.py @@ -0,0 +1,87 @@ +"""API endpoints for job monitoring and management. + +Routes: + GET /api/jobs - List all jobs + GET /api/jobs/{id} - Get a specific job's status + GET /api/jobs/{id}/log - Stream or fetch job log output + DELETE /api/jobs/{id} - Cancel a running job +""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException +from fastapi.responses import StreamingResponse + +from backend.models.job import Job, JobListResponse, JobStatus +from backend.services.job_service import JobService + +router = APIRouter(prefix="/api/jobs", tags=["jobs"]) + +# Shared job service instance (must match the one used in workflow.py) +# In a production setup, this would be injected via dependency injection. +# For this scaffold, we import the same module-level instance. +from backend.api.workflow import _job_service + + +@router.get("", response_model=JobListResponse) +async def list_jobs(status: JobStatus | None = None) -> JobListResponse: + """List all tracked jobs, optionally filtered by status. + + Query parameters: + status: Filter by job status (pending, running, completed, failed, cancelled) + """ + jobs = _job_service.list_jobs(status=status) + return JobListResponse(jobs=jobs, total=len(jobs)) + + +@router.get("/{job_id}", response_model=Job) +async def get_job(job_id: str) -> Job: + """Get the current status of a specific job.""" + job = _job_service.get_job(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") + return job + + +@router.get("/{job_id}/log") +async def get_job_log(job_id: str, stream: bool = False, tail: int = 100): + """Fetch or stream a job's log output. + + Query parameters: + stream: If true, return a Server-Sent Events stream + tail: Number of lines to return (non-streaming mode, default 100) + """ + job = _job_service.get_job(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") + + if stream: + # Server-Sent Events stream + async def event_stream(): + async for chunk in _job_service.stream_log(job_id): + # SSE format: each message is "data: ...\n\n" + for line in chunk.splitlines(): + yield f"data: {line}\n\n" + yield "event: done\ndata: stream ended\n\n" + + return StreamingResponse( + event_stream(), + media_type="text/event-stream", + headers={ + "Cache-Control": "no-cache", + "Connection": "keep-alive", + }, + ) + + # Non-streaming: return last N lines + log_content = _job_service.read_log(job_id, tail=tail) + return {"job_id": job_id, "log": log_content} + + +@router.delete("/{job_id}", response_model=Job) +async def cancel_job(job_id: str) -> Job: + """Cancel a running job by sending SIGTERM to its process.""" + job = _job_service.cancel_job(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") + return job diff --git a/backend/api/parameters.py b/backend/api/parameters.py new file mode 100644 index 00000000..0102b1ef --- /dev/null +++ b/backend/api/parameters.py @@ -0,0 +1,162 @@ +"""API endpoints for parameter schema, file I/O, and validation. + +Routes: + GET /api/v1/parameters/schema - Return the full parameter schema (v1, wrapped) + POST /api/v1/parameters/validate - Validate a dict of parameter values (v1) + GET /api/parameters/schema - Return the full parameter schema (legacy, flat list) + GET /api/parameters/file/{path} - Load a parameter file from disk + POST /api/parameters/file - Save a parameter file to disk + POST /api/parameters/validate - Validate parameter values (legacy) +""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from backend.models.parameter import ( + ParameterDefinition, + ParameterFile, + ParameterSchemaResponse, + ParameterValidationRequest, + ParameterValidationResult, + ParameterValue, +) +from backend.services.parameter_service import ParameterService + +router = APIRouter(prefix="/api/parameters", tags=["parameters"]) + +# v1 router serves the schema wrapped in {"parameters": [...]} as the +# frontend E2E tests expect. +v1_router = APIRouter(prefix="/api/v1/parameters", tags=["parameters-v1"]) + +# Singleton service instance +_service = ParameterService() + + +# ---- v1 endpoint (wrapped response) ------------------------------------ + +@v1_router.get("/schema", response_model=ParameterSchemaResponse) +async def get_parameter_schema_v1() -> ParameterSchemaResponse: + """Return the complete parameter schema wrapped in a JSON object. + + Response body: ``{"parameters": [, ...]}`` + + This is the preferred endpoint for the React frontend. + """ + return ParameterSchemaResponse(parameters=_service.get_schema()) + + +@v1_router.get("/file/{path:path}", response_model=ParameterFile) +async def load_parameter_file_v1(path: str) -> ParameterFile: + """Load and parse a MATLAB-style parameter file (v1). + + Reads the file at the given server-side filesystem path, parses all + ``key = value`` assignments, and transparently migrates any deprecated + parameter names (e.g. ``flgCCCcutoff`` → ``ccc_cutoff``) before + returning the result. + + Args: + path: Absolute or relative filesystem path to the ``.m`` file. + + Returns: + A :class:`ParameterFile` with parsed and migrated parameter values. + + Raises: + 404: When the file does not exist at the given path. + """ + try: + return _service.load_parameter_file_v1(path) + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Parameter file not found: {path}") + + +@v1_router.post("/file", response_model=ParameterFile) +async def save_parameter_file_v1(param_file: ParameterFile) -> ParameterFile: + """Write parameter values to a MATLAB-style ``.m`` file (v1). + + Creates any missing parent directories. The output format is + compatible with the emClarity MATLAB parameter file parser. + + Args: + param_file: The parameter file to write, including the target path + and the list of ``{name, value}`` pairs. + + Returns: + The saved :class:`ParameterFile` echoed back on success. + + Raises: + 500: When the file cannot be written (e.g. permission denied). + """ + try: + _service.save_parameter_file(param_file) + return param_file + except OSError as exc: + raise HTTPException(status_code=500, detail=f"Failed to save file: {exc}") + + +@v1_router.post("/validate", response_model=ParameterValidationResult) +async def validate_parameters_v1( + request: ParameterValidationRequest, +) -> ParameterValidationResult: + """Validate a flat dict of parameter values against the schema. + + Accepts ``{"parameters": {"PIXEL_SIZE": 1.35e-10, ...}}`` and returns + a validation result. Deprecated parameter names (e.g. ``flgCCCcutoff``) + are transparently translated to their canonical form before validation. + """ + return _service.validate_parameters_dict(request.parameters) + + +# ---- legacy endpoint (flat list, backward compat) ----------------------- + +@router.get("/schema", response_model=list[ParameterDefinition]) +async def get_parameter_schema() -> list[ParameterDefinition]: + """Return the complete parameter schema as a flat list. + + The schema describes every parameter that emClarity accepts, + including type, range constraints, and descriptions. The frontend + uses this to render dynamic parameter forms. + + .. deprecated:: + Prefer ``GET /api/v1/parameters/schema`` which wraps the + response in ``{"parameters": [...]}``. + """ + return _service.get_schema() + + +@router.get("/file/{path:path}", response_model=ParameterFile) +async def load_parameter_file(path: str) -> ParameterFile: + """Load and parse a MATLAB-style parameter file. + + The path should be an absolute filesystem path or relative to + the server's working directory. + """ + try: + return _service.load_parameter_file(path) + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Parameter file not found: {path}") + + +@router.post("/file", response_model=ParameterFile) +async def save_parameter_file(param_file: ParameterFile) -> ParameterFile: + """Write parameter values to a .m file on disk. + + Creates parent directories if they do not exist. + """ + try: + _service.save_parameter_file(param_file) + return param_file + except OSError as exc: + raise HTTPException(status_code=500, detail=f"Failed to save file: {exc}") + + +@router.post("/validate", response_model=ParameterValidationResult) +async def validate_parameters( + parameters: list[ParameterValue], +) -> ParameterValidationResult: + """Validate a list of parameter values against the schema. + + Returns a result indicating whether all values are valid, along + with any errors or warnings. + """ + return _service.validate_parameters(parameters) diff --git a/backend/api/projects.py b/backend/api/projects.py new file mode 100644 index 00000000..e8464461 --- /dev/null +++ b/backend/api/projects.py @@ -0,0 +1,61 @@ +"""API endpoints for emClarity project management. + +Routes: + POST /api/projects - Create a new project + GET /api/projects/{path} - Load project state + GET /api/projects/{path}/tilt-series - List tilt series in a project +""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from backend.models.project import Project, TiltSeries +from backend.services.project_service import ProjectService + +router = APIRouter(prefix="/api/projects", tags=["projects"]) + +_service = ProjectService() + + +class CreateProjectRequest(BaseModel): + """Request body for creating a new project.""" + + name: str = Field(..., description="Project name") + path: str = Field(..., description="Absolute path for the project directory") + + +@router.post("", response_model=Project) +async def create_project(request: CreateProjectRequest) -> Project: + """Create a new emClarity project directory. + + Sets up the standard directory structure (rawData/, fixedStacks/, + aliStacks/, cache/, convmap/, FSC/, logFile/). + """ + try: + return _service.create_project(request.name, request.path) + except OSError as exc: + raise HTTPException(status_code=500, detail=f"Failed to create project: {exc}") + + +@router.get("/{path:path}/tilt-series", response_model=list[TiltSeries]) +async def list_tilt_series(path: str) -> list[TiltSeries]: + """List tilt series found in the project's rawData/ directory.""" + try: + return _service.list_tilt_series(f"/{path}") + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Project not found: /{path}") + + +@router.get("/{path:path}", response_model=Project) +async def load_project(path: str) -> Project: + """Load project state by inspecting the directory structure. + + The backend determines the current pipeline state by checking + which processing directories contain data. + """ + try: + return _service.load_project(f"/{path}") + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Project not found: /{path}") diff --git a/backend/api/router.py b/backend/api/router.py new file mode 100644 index 00000000..e0be6b5d --- /dev/null +++ b/backend/api/router.py @@ -0,0 +1,34 @@ +"""Main API router that aggregates all endpoint modules. + +Import this router in main.py and include it on the FastAPI app. +""" + +from __future__ import annotations + +from fastapi import APIRouter + +from backend.api import jobs, parameters, projects, system, workflow +from backend.api import v1_filesystem, v1_parameters, v1_projects, v1_workflow, v1_system, v1_jobs, v1_utilities, v1_viewer, v1_environment + +router = APIRouter() + +router.include_router(parameters.router) +router.include_router(parameters.v1_router) +router.include_router(projects.router) +router.include_router(v1_projects.router) +router.include_router(workflow.router) +# v1_workflow must come AFTER v1_projects so the shared _projects dict is +# populated before any route handlers run. +router.include_router(v1_workflow.router) +router.include_router(jobs.router) +router.include_router(system.router) +# V1 system and jobs endpoints +router.include_router(v1_system.router) +# v1_jobs/schema must be registered before v1_jobs/{job_id} to avoid +# "schema" being interpreted as a job ID. +router.include_router(v1_jobs.router) +router.include_router(v1_utilities.router) +router.include_router(v1_filesystem.router) +router.include_router(v1_viewer.router) +router.include_router(v1_environment.router) +router.include_router(v1_parameters.router) diff --git a/backend/api/system.py b/backend/api/system.py new file mode 100644 index 00000000..d2e9e077 --- /dev/null +++ b/backend/api/system.py @@ -0,0 +1,32 @@ +"""API endpoints for system information (GPU, CPU, memory). + +Routes: + GET /api/system/gpus - Detect available NVIDIA GPUs + GET /api/system/info - Full system information +""" + +from __future__ import annotations + +from fastapi import APIRouter + +from backend.services.system_service import GpuInfo, SystemInfo, SystemService + +router = APIRouter(prefix="/api/system", tags=["system"]) + +_service = SystemService() + + +@router.get("/gpus", response_model=list[GpuInfo]) +async def detect_gpus() -> list[GpuInfo]: + """Detect NVIDIA GPUs via nvidia-smi. + + Returns an empty list if no GPUs are found or nvidia-smi is + not available. + """ + return _service.detect_gpus() + + +@router.get("/info", response_model=SystemInfo) +async def get_system_info() -> SystemInfo: + """Return system information including CPU cores, RAM, and GPUs.""" + return _service.get_system_info() diff --git a/backend/api/v1_environment.py b/backend/api/v1_environment.py new file mode 100644 index 00000000..1c839856 --- /dev/null +++ b/backend/api/v1_environment.py @@ -0,0 +1,250 @@ +"""V1 Environment API endpoints. + +Provides endpoints for validating executable paths, testing SSH connectivity, +and checking for required system dependencies. + +POST /api/v1/environment/validate-path + body: { path: str } + Returns { valid: bool, version: str | null, error: str | null } + Checks existence, executability, and attempts to retrieve version string. + +POST /api/v1/environment/test-ssh + body: { host: str, user: str | null, port: int } + Returns { connected: bool, error: str | null, latency_ms: float | null } + Shells out to the system ssh command; does NOT use paramiko. + +GET /api/v1/environment/check-dependencies + Returns { dependencies: [{ name: str, path: str | null, found: bool, version: str | null }] } + Checks for emClarity, IMOD (imodinfo), and CUDA Toolkit (nvcc). +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import time + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from backend.utils.machine_config import get_registry_dir + +router = APIRouter(prefix="/api/v1/environment", tags=["environment-v1"]) + +# --------------------------------------------------------------------------- +# Whitelist of environment variables that may be resolved via the API +# --------------------------------------------------------------------------- + +ALLOWED_ENV_VARS = {"EMCLARITY_PATH", "IMOD_DIR", "CUDA_HOME", "IMOD_BIN"} + + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + +def _get_version(path: str) -> str | None: + """Try --version then -v to retrieve a version string from an executable. + + Returns the first non-empty line from stdout (preferred) or stderr, or + None if both attempts fail or produce no output. + """ + for flag in ("--version", "-v"): + try: + result = subprocess.run( + [path, flag], + capture_output=True, + text=True, + timeout=10, + ) + output = result.stdout.strip() or result.stderr.strip() + if output: + return output.splitlines()[0].strip() + except (subprocess.TimeoutExpired, FileNotFoundError, PermissionError, OSError): + continue + return None + + +# --------------------------------------------------------------------------- +# POST /validate-path +# --------------------------------------------------------------------------- + +class ValidatePathRequest(BaseModel): + path: str + + +class ValidatePathResponse(BaseModel): + valid: bool + version: str | None + error: str | None + + +@router.post("/validate-path", response_model=ValidatePathResponse) +def validate_path(request: ValidatePathRequest) -> ValidatePathResponse: + """Validate that a path exists, is executable, and optionally return its version.""" + path = request.path + + if not path: + return ValidatePathResponse(valid=False, version=None, error="Path must not be empty") + + if not os.path.exists(path): + return ValidatePathResponse(valid=False, version=None, error=f"Path does not exist: {path}") + + if not os.path.isfile(path): + return ValidatePathResponse(valid=False, version=None, error=f"Path is not a file: {path}") + + if not os.access(path, os.X_OK): + return ValidatePathResponse(valid=False, version=None, error=f"Path is not executable: {path}") + + version = _get_version(path) + + return ValidatePathResponse(valid=True, version=version, error=None) + + +# --------------------------------------------------------------------------- +# POST /test-ssh +# --------------------------------------------------------------------------- + +class TestSshRequest(BaseModel): + host: str + user: str | None = None + port: int = Field(default=22, ge=1, le=65535) + + +class TestSshResponse(BaseModel): + connected: bool + error: str | None + latency_ms: float | None + + +@router.post("/test-ssh", response_model=TestSshResponse) +def test_ssh(request: TestSshRequest) -> TestSshResponse: + """Test SSH connectivity to a remote host by shelling out to the ssh command.""" + user_at_host = f"{request.user}@{request.host}" if request.user else request.host + + cmd = [ + "ssh", + "-o", "ConnectTimeout=5", + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-p", str(request.port), + "--", + user_at_host, + "true", + ] + + start = time.perf_counter() + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=10, + ) + latency_ms = (time.perf_counter() - start) * 1000.0 + + if result.returncode == 0: + return TestSshResponse(connected=True, error=None, latency_ms=latency_ms) + else: + error_text = result.stderr.strip() or f"SSH exited with code {result.returncode}" + return TestSshResponse(connected=False, error=error_text, latency_ms=None) + + except subprocess.TimeoutExpired: + return TestSshResponse(connected=False, error="SSH connection timed out", latency_ms=None) + except FileNotFoundError: + return TestSshResponse(connected=False, error="ssh executable not found", latency_ms=None) + except PermissionError as exc: + return TestSshResponse(connected=False, error=f"Permission denied running ssh: {exc}", latency_ms=None) + except OSError as exc: + return TestSshResponse(connected=False, error=f"OS error running ssh: {exc}", latency_ms=None) + + +# --------------------------------------------------------------------------- +# GET /check-dependencies +# --------------------------------------------------------------------------- + +class DependencyInfo(BaseModel): + name: str + path: str | None + found: bool + version: str | None + + +class CheckDependenciesResponse(BaseModel): + dependencies: list[DependencyInfo] + + +_DEPENDENCY_SPECS: list[tuple[str, str, list[str]]] = [ + # (display_name, binary_name, extra_paths_to_check) + ("emClarity", "emClarity", ["/usr/local/bin/emClarity"]), + ("IMOD", "imodinfo", ["/usr/local/IMOD/bin/imodinfo"]), + ("CUDA Toolkit", "nvcc", ["/usr/local/cuda/bin/nvcc"]), +] + + +def _find_binary(binary_name: str, extra_paths: list[str]) -> str | None: + """Return the first accessible path for a binary, or None.""" + found = shutil.which(binary_name) + if found: + return found + for candidate in extra_paths: + if os.path.isfile(candidate) and os.access(candidate, os.X_OK): + return candidate + return None + + +@router.get("/check-dependencies", response_model=CheckDependenciesResponse) +def check_dependencies() -> CheckDependenciesResponse: + """Check for required system dependencies and return their status.""" + results: list[DependencyInfo] = [] + + for display_name, binary_name, extra_paths in _DEPENDENCY_SPECS: + binary_path = _find_binary(binary_name, extra_paths) + if binary_path is None: + results.append(DependencyInfo(name=display_name, path=None, found=False, version=None)) + else: + version = _get_version(binary_path) + results.append(DependencyInfo(name=display_name, path=binary_path, found=True, version=version)) + + return CheckDependenciesResponse(dependencies=results) + + +# --------------------------------------------------------------------------- +# GET /resolve-env +# --------------------------------------------------------------------------- + +class ResolveEnvResponse(BaseModel): + value: str | None + found: bool + + +@router.get("/resolve-env", response_model=ResolveEnvResponse) +def resolve_env(var: str) -> ResolveEnvResponse: + """Resolve a whitelisted environment variable.""" + if var not in ALLOWED_ENV_VARS: + raise HTTPException( + status_code=403, + detail=f"Environment variable '{var}' is not in the allowed list", + ) + value = os.environ.get(var) + return ResolveEnvResponse(value=value, found=value is not None) + + +# --------------------------------------------------------------------------- +# GET /registry-path +# --------------------------------------------------------------------------- + + +class RegistryPathResponse(BaseModel): + path: str + + +# Freeze the registry path at module-import time so the endpoint always +# returns the value that is actually in use (consistent with v1_projects). +_REGISTRY_PATH: str = str(get_registry_dir()) + + +@router.get("/registry-path", response_model=RegistryPathResponse) +def get_registry_path() -> RegistryPathResponse: + """Return the current registry directory path.""" + return RegistryPathResponse(path=_REGISTRY_PATH) diff --git a/backend/api/v1_filesystem.py b/backend/api/v1_filesystem.py new file mode 100644 index 00000000..61f9c301 --- /dev/null +++ b/backend/api/v1_filesystem.py @@ -0,0 +1,291 @@ +"""V1 Filesystem browse API endpoint. + +Provides a directory listing endpoint for server-side filesystem navigation. +The endpoint returns only real (non-symlink) subdirectories, allowing frontend +components to let users pick project directories without exposing files. + +Response contract (documented for TASK-002b stub): + GET /api/v1/filesystem/browse?path= + 200 OK: + { + "path": "/absolute/real/path", + "parent": "/absolute/real" | null, // null only at filesystem root "/" + "entries": [ + {"name": "subdir", "type": "directory", "path": "/absolute/real/path/subdir"} + ] + } + + Error responses all use FastAPI's standard HTTPException body: + {"detail": ""} + with Content-Type: application/json. + + 400: path traversal ('..'), null byte, relative path, not-a-directory, too long + 403: permission denied reading the directory + 404: path does not exist (including race-condition removal during listing) +""" + +from __future__ import annotations + +import logging +import os +import stat +from dataclasses import dataclass +from pathlib import Path +from typing import Literal + +from fastapi import APIRouter, HTTPException, Query +from pydantic import BaseModel + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1/filesystem", tags=["filesystem-v1"]) + +# Linux PATH_MAX +_PATH_MAX = 4096 + + +# --------------------------------------------------------------------------- +# Response models +# --------------------------------------------------------------------------- + + +class FilesystemEntry(BaseModel): + """A single directory entry returned by the browse endpoint.""" + + name: str + type: Literal["directory"] + path: str + + +class BrowseResponse(BaseModel): + """Response body for GET /api/v1/filesystem/browse.""" + + path: str + parent: str | None + entries: list[FilesystemEntry] + + +# --------------------------------------------------------------------------- +# Path validation helper +# --------------------------------------------------------------------------- + + +@dataclass +class _BrowsePath: + """Holds both the OS-level real path and a display path for error messages. + + ``real`` – fully resolved via os.path.realpath; used for all OS + operations (stat, scandir) and the response ``path`` field. + ``display`` – normalised (os.path.normpath) but *not* symlink-resolved; + used in error-message detail strings so that the path shown + to the caller matches what they sent, even on systems where + the temp-dir prefix crosses a symlink boundary (e.g. macOS + where /tmp → /private/tmp). + """ + + real: Path + display: str + + +def _validate_browse_path(path: str | None) -> _BrowsePath: + """Validate and normalise a browse path, returning a ``_BrowsePath``. + + Applies the following checks in order: + 1. Empty / missing → default to home directory. + 2. Length >= PATH_MAX → 400. + 3. Null byte present → 400. + 4. '..' component present (after URL-decode, which FastAPI already did) → 400. + 5. No leading '/' (relative path) → 400. + 6. os.path.normpath to collapse duplicates / trailing slashes. + 7. os.path.realpath to resolve any symlinks in the path itself. + + Existence / type / permission checks are left to the route handler so that + the error responses (404, 400, 403) come from the right layer. + """ + # 1. Empty / missing → home directory + if path is None or not path.strip(): + try: + home = Path.home() + except RuntimeError as exc: + raise HTTPException( + status_code=500, + detail=( + "Cannot determine user home directory: " + "the HOME directory could not be resolved" + ), + ) from exc + real = Path(os.path.realpath(str(home))) + return _BrowsePath(real=real, display=str(real)) + + path = path.strip() + + # 2. Length guard (Linux PATH_MAX = 4096 bytes; measure in UTF-8 bytes, not + # Unicode code points, so multi-byte characters are counted correctly). + # Use >= so that exactly 4096 bytes is also rejected: PATH_MAX includes + # the NUL terminator, meaning valid paths are at most 4095 bytes long. + # A path of exactly 4096 bytes reaches the OS and triggers ENAMETOOLONG, + # which would otherwise surface as a 404 rather than a proper 400. + if len(path.encode("utf-8")) >= _PATH_MAX: + raise HTTPException( + status_code=400, + detail=f"Path too long: maximum length is {_PATH_MAX - 1} bytes", + ) + + # 3. Null byte guard + if "\x00" in path: + raise HTTPException( + status_code=400, + detail="Path contains invalid null byte", + ) + + # 4. Path traversal guard: check each component for literal '..' + # FastAPI URL-decodes query parameters before the handler runs, so + # %2e%2e%2f will already be decoded to '../' when we receive it here. + if ".." in path.split("/"): + raise HTTPException( + status_code=400, + detail="Path traversal via '..' components is not allowed", + ) + + # 5. Must be absolute (start with '/') + if not path.startswith("/"): + raise HTTPException( + status_code=400, + detail="Path must be absolute", + ) + + # 6. Normalise: collapse '//', remove trailing '/', etc. + normalized = os.path.normpath(path) + + # 7. Resolve any symlinks in the path itself so the response always shows + # the real on-disk location. + real = os.path.realpath(normalized) + + return _BrowsePath(real=Path(real), display=normalized) + + +# --------------------------------------------------------------------------- +# Route handler +# --------------------------------------------------------------------------- + + +@router.get("/browse", response_model=BrowseResponse) +def browse_filesystem( + path: str | None = Query(default=None, description="Absolute path to browse"), +) -> BrowseResponse: + """List subdirectories at the given path. + + Only real directories are returned; regular files and symbolic links are + excluded. Non-UTF-8 filenames are silently skipped so they never cause a + 500 JSON-serialization error. + + Query parameters: + path: Absolute filesystem path to browse. + Empty or missing → server user's home directory. + Whitespace-only → stripped to empty, treated as missing. + """ + browse_path = _validate_browse_path(path) + real_path = browse_path.real + # display_path is the caller-supplied path after normpath (but before + # realpath), used in error-detail strings so they match what the caller + # sent even when the path crosses a symlink boundary (e.g. macOS /tmp → + # /private/tmp). + display_path = browse_path.display + + # --- Existence check ----------------------------------------------------------- + # Use Path.stat() rather than Path.exists() because Path.exists() catches + # OSError internally and returns False for errors it ignores (ENOENT, + # ENOTDIR, EBADF, ELOOP). If we used Path.exists() we would need a + # second stat() call anyway to distinguish 404 from 403. Calling + # Path.stat() directly lets us catch PermissionError and other OSErrors + # once, in the right layer, with the right HTTP status codes. + try: + path_stat = real_path.stat() + except PermissionError: + raise HTTPException( + status_code=403, + detail=f"Permission denied accessing: {display_path}", + ) + except OSError: + # Covers FileNotFoundError, NotADirectoryError, and other stat failures. + raise HTTPException( + status_code=404, + detail=f"Path not found: {display_path}", + ) + + # --- Must be a directory ------------------------------------------------------- + # Reuse the stat result so there is no additional syscall and no risk of an + # unguarded PermissionError from a separate Path.is_dir() call. + if not stat.S_ISDIR(path_stat.st_mode): + raise HTTPException( + status_code=400, + detail=f"Path is not a directory: {display_path}", + ) + + # --- Compute parent and response path ----------------------------------------- + # The response `path` field uses the realpath so that symlink targets are + # resolved transparently (test: test_symlink_path_resolves_to_real). + real_path_str = str(real_path) + parent: str | None = None if real_path_str == "/" else str(real_path.parent) + + # Entry paths are built from real_path_str (the fully-resolved real path) + # so they remain consistent with response.path. This is essential when the + # browse target is itself a symlink: using display_path (pre-realpath) would + # produce entry paths rooted at the symlink path while response.path is the + # resolved real path, violating the documented response contract. + entry_base = real_path_str + + # --- Scan entries -------------------------------------------------------------- + entries: list[FilesystemEntry] = [] + try: + with os.scandir(real_path) as it: + for entry in it: + # Only include real (non-symlink) directories. + # DirEntry.is_dir(follow_symlinks=False) returns False for symlinks + # even when they point to directories, so this single check excludes + # both regular files and all symlinks. + if not entry.is_dir(follow_symlinks=False): + continue + + # Guard against non-UTF-8 names that would break JSON serialization. + # Python represents such names using surrogateescape, which cannot be + # encoded to valid UTF-8. + try: + name: str = entry.name + name.encode("utf-8") + except UnicodeEncodeError: + log.debug("Skipping non-UTF-8 directory entry in %s", real_path) + continue + + # Build the absolute path without introducing double slashes at root. + if entry_base == "/": + entry_path = f"/{name}" + else: + entry_path = f"{entry_base}/{name}" + + entries.append( + FilesystemEntry(name=name, type="directory", path=entry_path) + ) + + except PermissionError: + raise HTTPException( + status_code=403, + detail=f"Permission denied reading directory: {display_path}", + ) + except FileNotFoundError: + # Race condition: directory was removed between the existence check above + # and the scandir call. Surface as 404, never 500. + raise HTTPException( + status_code=404, + detail=f"Path not found (removed during listing): {display_path}", + ) + except OSError as exc: + # Catch remaining OS-level errors (e.g. EIO, ENAMETOOLONG from the + # kernel) so they never bubble up as an unhandled 500. + log.exception("OS error scanning directory %s", real_path) + raise HTTPException( + status_code=500, + detail=f"I/O error reading directory: {display_path}", + ) from exc + + return BrowseResponse(path=real_path_str, parent=parent, entries=entries) diff --git a/backend/api/v1_jobs.py b/backend/api/v1_jobs.py new file mode 100644 index 00000000..eca4bde4 --- /dev/null +++ b/backend/api/v1_jobs.py @@ -0,0 +1,162 @@ +"""V1 API endpoints for job tracking and management. + +Routes: + GET /api/v1/jobs - List jobs, optionally filtered by project_id + GET /api/v1/jobs/schema - Return the Job schema definition + GET /api/v1/jobs/{id} - Get a specific job's status (404 if not found) + GET /api/v1/jobs/{id}/log - Fetch the tail of a job's log file + DELETE /api/v1/jobs/{id} - Cancel a running job +""" + +from __future__ import annotations + +import uuid +from datetime import datetime, timezone +from enum import Enum +from typing import Any + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +router = APIRouter(prefix="/api/v1/jobs", tags=["jobs-v1"]) + + +# --------------------------------------------------------------------------- +# Models +# --------------------------------------------------------------------------- + + +class JobStatus(str, Enum): + """Lifecycle states for a tracked job.""" + + PENDING = "PENDING" + RUNNING = "RUNNING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + CANCELLED = "CANCELLED" + + +class Job(BaseModel): + """Represents a single emClarity command execution.""" + + id: str = Field(..., description="Unique job identifier (UUID)") + project_id: str = Field(..., description="Project this job belongs to") + command: str = Field(..., description="The emClarity command (e.g. 'autoAlign')") + status: JobStatus = Field( + default=JobStatus.PENDING, + description="Current job status", + ) + created_at: datetime = Field( + default_factory=lambda: datetime.now(tz=timezone.utc), + description="When the job was created (ISO-8601)", + ) + updated_at: datetime = Field( + default_factory=lambda: datetime.now(tz=timezone.utc), + description="When the job was last updated (ISO-8601)", + ) + pid: int | None = Field(default=None, description="OS process ID") + exit_code: int | None = Field(default=None, description="Process exit code") + error_message: str | None = Field(default=None, description="Error summary if failed") + log_path: str | None = Field(default=None, description="Path to the job's log file") + + +# --------------------------------------------------------------------------- +# In-memory job registry (keyed by job ID) +# --------------------------------------------------------------------------- + +_jobs: dict[str, Job] = {} + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + + +@router.get("/schema", response_model=dict[str, Any]) +async def get_job_schema() -> dict[str, Any]: + """Return the Job model schema definition. + + Provides field descriptions and status enum values for client validation. + """ + schema = Job.model_json_schema() + return schema + + +@router.get("", response_model=list[Job]) +async def list_jobs(project_id: str | None = None) -> list[Job]: + """List all tracked jobs, optionally filtered by project_id. + + Query parameters: + project_id: Filter jobs to a specific project (optional) + """ + jobs = list(_jobs.values()) + if project_id is not None: + jobs = [j for j in jobs if j.project_id == project_id] + return sorted(jobs, key=lambda j: j.created_at, reverse=True) + + +@router.get("/{job_id}", response_model=Job) +async def get_job(job_id: str) -> Job: + """Get the current status of a specific job. + + Returns 404 if the job does not exist. + """ + job = _jobs.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") + return job + + +@router.get("/{job_id}/log", response_model=dict[str, Any]) +async def get_job_log(job_id: str, tail: int = 100) -> dict[str, Any]: + """Fetch the tail of a job's log file. + + Query parameters: + tail: Number of lines to return from the end of the log (default 100) + + Returns 404 if the job does not exist. + """ + job = _jobs.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") + + log_content = "" + if job.log_path: + try: + from pathlib import Path # noqa: PLC0415 + + log_file = Path(job.log_path) + if log_file.exists(): + lines = log_file.read_text(encoding="utf-8", errors="replace").splitlines() + log_content = "\n".join(lines[-tail:]) + except OSError: + log_content = "" + + return {"job_id": job_id, "log": log_content} + + +@router.delete("/{job_id}", response_model=Job) +async def cancel_job(job_id: str) -> Job: + """Cancel a running or pending job. + + Marks the job status as CANCELLED. For running jobs the OS process + would normally receive SIGTERM; in this in-memory implementation the + status is updated directly. + + Returns 404 if the job does not exist. + """ + job = _jobs.get(job_id) + if job is None: + raise HTTPException(status_code=404, detail=f"Job not found: {job_id}") + + if job.status in (JobStatus.PENDING, JobStatus.RUNNING): + updated = job.model_copy( + update={ + "status": JobStatus.CANCELLED, + "updated_at": datetime.now(tz=timezone.utc), + } + ) + _jobs[job_id] = updated + return updated + + return job diff --git a/backend/api/v1_parameters.py b/backend/api/v1_parameters.py new file mode 100644 index 00000000..fef0ca1f --- /dev/null +++ b/backend/api/v1_parameters.py @@ -0,0 +1,243 @@ +"""Parameter snapshot endpoints for saving, listing, loading, and exporting. + +Provides: +- POST /api/v1/projects/{project_id}/parameter-snapshots + Save a parameter snapshot to the project's parameters/ directory. +- GET /api/v1/projects/{project_id}/parameter-snapshots + List all snapshots sorted by date descending. +- GET /api/v1/projects/{project_id}/parameter-snapshots/{snapshot_id} + Load full parameter values for a specific snapshot. +- POST /api/v1/projects/{project_id}/parameter-snapshots/{snapshot_id}/export-m + Export a snapshot to MATLAB .m format. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from backend.api.v1_projects import _get_project +from backend.services.parameter_service import ParameterService + +log = logging.getLogger(__name__) + +router = APIRouter( + prefix="/api/v1/projects/{project_id}/parameter-snapshots", + tags=["parameter-snapshots-v1"], +) + +_parameter_service = ParameterService() + + +# --------------------------------------------------------------------------- +# Request / Response models +# --------------------------------------------------------------------------- + +class CreateSnapshotRequest(BaseModel): + """Request body for creating a parameter snapshot.""" + parameters: dict[str, Any] = Field( + ..., description="Parameter key-value pairs to snapshot" + ) + + +class CreateSnapshotResponse(BaseModel): + """Response after successfully creating a parameter snapshot.""" + snapshot_id: str + filename: str + created_at: str + + +class ExportMResponse(BaseModel): + """Response after exporting a snapshot to .m format.""" + m_file_path: str + + +class SnapshotListItem(BaseModel): + """Metadata for a single snapshot in a listing.""" + snapshot_id: str + filename: str + created_at: str + + +class SnapshotListResponse(BaseModel): + """Response containing a list of parameter snapshots.""" + snapshots: list[SnapshotListItem] + + +class SnapshotDetailResponse(BaseModel): + """Response containing a full parameter snapshot with its data.""" + snapshot_id: str + parameters: dict[str, Any] + created_at: str + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + +@router.post("", response_model=CreateSnapshotResponse) +def create_snapshot( + project_id: str, + body: CreateSnapshotRequest, +) -> CreateSnapshotResponse: + """Save a parameter snapshot to the project's parameters/ directory. + + Generates a UUID-based snapshot file, writes the parameters as JSON, + and enforces a retention cap of 50 snapshots per project (oldest deleted). + """ + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + + project_dir = Path(record.directory) + if not project_dir.is_dir(): + raise HTTPException( + status_code=404, + detail=f"Project directory does not exist: {record.directory}", + ) + + try: + snapshot_id, filename, created_at = _parameter_service.save_snapshot( + project_dir, body.parameters + ) + except Exception as exc: + log.exception("Failed to save parameter snapshot for project %s", project_id) + raise HTTPException( + status_code=500, + detail=f"Failed to save snapshot: {exc}", + ) from exc + + return CreateSnapshotResponse( + snapshot_id=snapshot_id, + filename=filename, + created_at=created_at, + ) + + +@router.post("/{snapshot_id}/export-m", response_model=ExportMResponse) +def export_snapshot_m( + project_id: str, + snapshot_id: str, +) -> ExportMResponse: + """Export a parameter snapshot to MATLAB .m format. + + Reads the snapshot JSON identified by *snapshot_id*, converts to .m + format, and writes the file alongside the JSON snapshot. + """ + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + + project_dir = Path(record.directory) + params_dir = project_dir / "parameters" + if not params_dir.is_dir(): + raise HTTPException( + status_code=404, + detail=f"No parameters directory found for project {project_id}", + ) + + # Find the snapshot file by matching the UUID prefix in filenames + try: + matching = [ + f for f in params_dir.iterdir() + if f.name.startswith(f"snapshot_{snapshot_id}") and f.suffix == ".json" + ] + except OSError as exc: + log.exception("Failed to list parameters directory for project %s", project_id) + raise HTTPException( + status_code=500, + detail=f"Failed to list snapshots directory: {exc}", + ) from exc + + if not matching: + raise HTTPException( + status_code=404, + detail=f"Snapshot {snapshot_id} not found in project {project_id}", + ) + + if len(matching) > 1: + raise HTTPException( + status_code=422, + detail=f"Snapshot ID prefix '{snapshot_id}' is ambiguous: matches {len(matching)} files", + ) + + snapshot_path = matching[0] + + try: + m_file_path = _parameter_service.export_snapshot_to_m(snapshot_path) + except Exception as exc: + log.exception("Failed to export snapshot %s to .m format", snapshot_id) + raise HTTPException( + status_code=500, + detail=f"Failed to export snapshot: {exc}", + ) from exc + + return ExportMResponse(m_file_path=str(m_file_path)) + + +@router.get("", response_model=SnapshotListResponse) +def list_snapshots(project_id: str) -> SnapshotListResponse: + """List all parameter snapshots sorted by date descending.""" + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + + project_dir = Path(record.directory) + if not project_dir.is_dir(): + raise HTTPException( + status_code=404, + detail=f"Project directory does not exist: {record.directory}", + ) + + try: + items = _parameter_service.list_snapshots(project_dir) + except Exception as exc: + log.exception("Failed to list snapshots for project %s", project_id) + raise HTTPException( + status_code=500, + detail=f"Failed to list snapshots: {exc}", + ) from exc + + return SnapshotListResponse( + snapshots=[SnapshotListItem(**item) for item in items] + ) + + +@router.get("/{snapshot_id}", response_model=SnapshotDetailResponse) +def get_snapshot(project_id: str, snapshot_id: str) -> SnapshotDetailResponse: + """Load a specific parameter snapshot.""" + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project {project_id} not found") + + project_dir = Path(record.directory) + if not project_dir.is_dir(): + raise HTTPException( + status_code=404, + detail=f"Project directory does not exist: {record.directory}", + ) + + try: + data = _parameter_service.load_snapshot(project_dir, snapshot_id) + except FileNotFoundError as exc: + raise HTTPException( + status_code=404, + detail=f"Snapshot {snapshot_id} not found in project {project_id}", + ) from exc + except ValueError as exc: + raise HTTPException( + status_code=422, + detail=str(exc), + ) from exc + except Exception as exc: + log.exception("Failed to load snapshot %s for project %s", snapshot_id, project_id) + raise HTTPException( + status_code=500, + detail=f"Failed to load snapshot: {exc}", + ) from exc + + return SnapshotDetailResponse(**data) diff --git a/backend/api/v1_projects.py b/backend/api/v1_projects.py new file mode 100644 index 00000000..2dca6a7b --- /dev/null +++ b/backend/api/v1_projects.py @@ -0,0 +1,592 @@ +"""V1 Project management API endpoints. + +Provides CRUD operations for emClarity projects using UUID-based identifiers. +Projects are backed by on-disk directory structures following emClarity conventions. + +The in-memory registry is persisted to ``/projects.json`` so that +project IDs remain valid across backend restarts. The registry directory +defaults to ``~/.emclarity`` but can be overridden via the +``EMCLARITY_REGISTRY_DIR`` environment variable (see +:func:`backend.utils.machine_config.get_registry_dir`). +""" + +from __future__ import annotations + +import logging +import re +import threading +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field, ValidationError + +from backend.models.project import ProjectState, TiltSeries +from backend.models.project_settings import ProjectSettings, ProjectSettingsPatch +from backend.services.project_service import ProjectService +from backend.utils.machine_config import get_registry_dir +from backend.utils.safe_json import locked_json_read, locked_json_read_write + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/v1/projects", tags=["projects-v1"]) + +# --------------------------------------------------------------------------- +# Persistence +# --------------------------------------------------------------------------- + +_REGISTRY_DIR = get_registry_dir() +_REGISTRY_DIR.mkdir(parents=True, exist_ok=True) +_REGISTRY_FILE = _REGISTRY_DIR / "projects.json" + +# In-process lock protecting _projects dict access. +_registry_lock = threading.Lock() + + +def _save_registry() -> None: + """Persist the in-memory project registry to disk using dual-locking. + + Serialises the current in-memory ``_projects`` dict through the + locked-JSON read-modify-write pattern so that concurrent threads + and processes cannot corrupt the file. + """ + try: + with _registry_lock: + snapshot = {k: v.model_dump(mode="json") for k, v in _projects.items()} + + def _replace(_existing: Any) -> dict[str, Any]: + return snapshot + + locked_json_read_write(_REGISTRY_FILE, _replace) + except Exception as exc: # noqa: BLE001 + log.error("Could not persist project registry: %s", exc) + raise HTTPException( + status_code=500, + detail=f"Failed to persist project registry: {exc}", + ) from exc + + +def _load_registry() -> None: + """Load the project registry from disk (called once at module import). + + Uses :func:`locked_json_read` so that a concurrent writer + cannot produce a partial read, without redundantly rewriting the file. + """ + if not _REGISTRY_FILE.exists(): + return + try: + raw = locked_json_read(_REGISTRY_FILE) + if raw is None: + return + + with _registry_lock: + for project_id, record_data in raw.items(): + try: + _projects[project_id] = _ProjectRecord(**record_data) + except Exception as exc: # noqa: BLE001 + log.warning("Skipping corrupt registry entry %s: %s", project_id, exc) + except Exception as exc: # noqa: BLE001 + log.warning("Could not load project registry from %s: %s", _REGISTRY_FILE, exc) + + +def _get_projects() -> dict[str, _ProjectRecord]: + """Return a snapshot of the in-memory registry (thread-safe).""" + with _registry_lock: + return dict(_projects) + + +def _get_project(project_id: str) -> _ProjectRecord | None: + """Look up a single project by ID (thread-safe).""" + with _registry_lock: + return _projects.get(project_id) + + +def _set_project(project_id: str, record: _ProjectRecord) -> None: + """Insert or update a project in the in-memory registry (thread-safe).""" + with _registry_lock: + _projects[project_id] = record + + +# --------------------------------------------------------------------------- +# In-memory project registry (keyed by UUID string) +# --------------------------------------------------------------------------- + +class _ProjectRecord(BaseModel): + """Internal record stored in memory for each created project.""" + + id: str + name: str + directory: str + state: ProjectState + parameters: dict[str, Any] + current_cycle: int = 0 + last_accessed: str | None = None + settings: ProjectSettings = Field(default_factory=ProjectSettings) + + +_projects: dict[str, _ProjectRecord] = {} +_project_service = ProjectService() + +# Load persisted registry at startup +_load_registry() + + +# --------------------------------------------------------------------------- +# Request / Response models +# --------------------------------------------------------------------------- + + +class CreateProjectRequest(BaseModel): + """Payload for creating a new project.""" + + name: str = Field(..., description="Human-readable project name") + directory: str = Field(..., description="Absolute path to the project directory on disk") + parameters: dict[str, Any] = Field( + default_factory=dict, + description="Initial emClarity parameters", + ) + + +class LoadProjectRequest(BaseModel): + """Payload for loading an existing project by its filesystem path.""" + + directory: str = Field(..., description="Absolute path to an existing emClarity project") + name: str | None = Field(None, description="Override the project name (defaults to dir name)") + + +class ProjectResponse(BaseModel): + """Response body for project endpoints.""" + + id: str + name: str + directory: str + state: str + parameters: dict[str, Any] + current_cycle: int = 0 + last_accessed: str | None = None + + +class ProjectStatisticsResponse(BaseModel): + """Response body for the project statistics endpoint.""" + + project_id: str + particle_count: int | None = None + resolution_angstrom: float | None = None + tilt_series_count: int + + +class TiltSeriesListResponse(BaseModel): + """Response body for tilt series listing.""" + + tilt_series: list[TiltSeries] + + +# --------------------------------------------------------------------------- +# Path safety helpers +# --------------------------------------------------------------------------- + +# Patterns that indicate path traversal or clearly unsafe inputs +_UNSAFE_PATH_PATTERNS = re.compile(r"\.\.|/etc/|/proc/|/sys/|/dev/") + + +def _validate_project_path(directory: str) -> Path: + """Return a resolved absolute path after basic safety checks. + + Raises HTTPException 400 if the path looks dangerous or non-absolute. + """ + if not directory or not directory.strip(): + raise HTTPException(status_code=400, detail="Directory path must not be empty") + + if _UNSAFE_PATH_PATTERNS.search(directory): + raise HTTPException( + status_code=400, + detail="Directory path contains unsafe components", + ) + + path = Path(directory) + if not path.is_absolute(): + raise HTTPException( + status_code=400, + detail="Directory path must be absolute (start with /)", + ) + + return path + + +# --------------------------------------------------------------------------- +# Statistics helpers +# --------------------------------------------------------------------------- + +def _count_particles(project_dir: Path) -> int | None: + """Count particles from .coords files in the recon/ directory. + + Each line in a .coords file represents one particle coordinate triplet. + Returns None if no .coords files are found. + """ + recon_dir = project_dir / "recon" + if not recon_dir.exists(): + return None + + total = 0 + found_any = False + for coords_file in recon_dir.glob("*.coords"): + try: + lines = coords_file.read_text().splitlines() + # Count non-empty, non-comment lines + count = sum( + 1 for line in lines + if line.strip() and not line.strip().startswith("#") + ) + total += count + found_any = True + except OSError: + continue + + return total if found_any else None + + +def _detect_best_resolution(project_dir: Path) -> float | None: + """Parse FSC text files to find the best resolution achieved. + + Only reads files matching the canonical emClarity FSC naming pattern + (*_fsc_GLD.txt) and extracts the resolution from the last data line. + The resolution line must follow the format: + + where spatial_frequency is in 1/Å units (reciprocal space). + + Returns the resolution in ÅngstrÃļms, or None if not determinable. + """ + fsc_dir = project_dir / "FSC" + if not fsc_dir.exists(): + return None + + best_angstrom: float | None = None + + # Only match the canonical FSC output files; ignore PDFs and other outputs + fsc_pattern = re.compile(r"^[\w\-]+_fsc_GLD\.txt$") + + for fsc_file in fsc_dir.glob("*_fsc_GLD.txt"): + if not fsc_pattern.match(fsc_file.name): + continue + try: + lines = fsc_file.read_text().splitlines() + except OSError: + continue + + # Each line: <1/Å_frequency> + # The FSC=0.143 threshold marks the resolution limit. + # We find the last line where FSC > 0.143 (or the highest freq where FSCâ‰Ĩ0.143). + resolution_freq: float | None = None + implausible_warned = False # emit at most one warning per file + for line in lines: + line = line.strip() + if not line or line.startswith("#"): + continue + parts = line.split() + if len(parts) < 2: + continue + try: + freq = float(parts[0]) + fsc_val = float(parts[1]) + except ValueError: + continue + # freq must be a realistic reciprocal-space value (0 < freq ≤ 1.0 1/Å). + # The upper bound of 1.0 corresponds to ~1 Å resolution, which is the + # practical physical limit for cryo-EM; values beyond this indicate + # misinterpreted units or corrupt data. + if freq <= 0 or freq > 1.0: + continue + if fsc_val >= 0.143: + angstrom = 1.0 / freq + # Accept physically meaningful cryo-EM resolutions up to 200 Å. + # No lower bound: sub-2 Å results are valid for high-resolution structures. + # Guard is applied per-line so one implausible line cannot silently + # discard an otherwise valid resolution result from the same file. + if angstrom <= 200.0: + resolution_freq = freq + elif not implausible_warned: + log.warning( + "Discarding implausible resolution(s) from %s " + "(first offender: %.2f Å exceeds 200 Å upper bound; " + "likely a unit mismatch).", + fsc_file, + angstrom, + ) + implausible_warned = True + + if resolution_freq is not None: + angstrom = 1.0 / resolution_freq + if best_angstrom is None or angstrom < best_angstrom: + best_angstrom = angstrom + + return round(best_angstrom, 2) if best_angstrom is not None else None + + +# --------------------------------------------------------------------------- +# Response builder +# --------------------------------------------------------------------------- + + +def _to_response(record: _ProjectRecord) -> ProjectResponse: + """Convert an internal record to an API response.""" + return ProjectResponse( + id=record.id, + name=record.name, + directory=record.directory, + state=record.state.value, + parameters=record.parameters, + current_cycle=record.current_cycle, + last_accessed=record.last_accessed, + ) + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + + +@router.get("", response_model=list[ProjectResponse]) +async def list_projects() -> list[ProjectResponse]: + """Return all registered projects sorted by last_accessed descending. + + Projects that have never been accessed (``last_accessed`` is ``None``) + appear after all projects with a timestamp. + """ + projects = _get_projects() + + # Partition into accessed and never-accessed, sort accessed newest-first + with_ts = [(k, v) for k, v in projects.items() if v.last_accessed is not None] + without_ts = [(k, v) for k, v in projects.items() if v.last_accessed is None] + with_ts.sort(key=lambda item: item[1].last_accessed or "", reverse=True) + sorted_items = with_ts + without_ts + + return [_to_response(rec) for _, rec in sorted_items] + + +@router.patch("/{project_id}/accessed", response_model=ProjectResponse) +async def mark_project_accessed(project_id: str) -> ProjectResponse: + """Touch a project's last_accessed timestamp (set to current UTC time). + + Used by the frontend to track recently accessed projects. + Returns 404 if the project ID is not found. + """ + with _registry_lock: + record = _projects.get(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + record.last_accessed = datetime.now(timezone.utc).isoformat() + # record is already in _projects; no need to re-insert + + _save_registry() + + return _to_response(record) + + +@router.delete("/{project_id}", status_code=204) +async def deregister_project(project_id: str) -> None: + """Remove a project from the registry (does NOT delete files on disk). + + Used by the frontend to hide a project from the recent-projects list. + Returns 404 if the project ID is not found. + """ + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + + with _registry_lock: + _projects.pop(project_id, None) + _save_registry() + + +@router.post("", status_code=201, response_model=ProjectResponse) +async def create_project(request: CreateProjectRequest) -> ProjectResponse: + """Create a new emClarity project. + + Creates the standard directory structure on disk and registers the + project in memory with an UNINITIALIZED state. + """ + _validate_project_path(request.directory) + + # Create the directory structure on disk + _project_service.create_project(name=request.name, path=request.directory) + + project_id = str(uuid.uuid4()) + record = _ProjectRecord( + id=project_id, + name=request.name, + directory=request.directory, + state=ProjectState.UNINITIALIZED, + parameters=request.parameters, + current_cycle=0, + ) + _set_project(project_id, record) + _save_registry() + + return _to_response(record) + + +@router.post("/load", status_code=200, response_model=ProjectResponse) +async def load_project(request: LoadProjectRequest) -> ProjectResponse: + """Load an existing emClarity project by filesystem path. + + Inspects the directory structure to determine the pipeline state. + Registers the project in the registry and returns a stable project ID. + + Security: only absolute paths without traversal components are accepted. + """ + project_path = _validate_project_path(request.directory) + + if not project_path.exists() or not project_path.is_dir(): + raise HTTPException( + status_code=404, + detail=f"Directory not found: {request.directory}", + ) + + # Check if this directory is already registered (by resolved path) + resolved = str(project_path.resolve()) + for existing_id, existing_record in _get_projects().items(): + if Path(existing_record.directory).resolve() == Path(resolved): + return _to_response(existing_record) + + # Load project state from disk + try: + project = _project_service.load_project(request.directory) + except FileNotFoundError as exc: + raise HTTPException(status_code=404, detail=str(exc)) from exc + + project_name = request.name or project.name + project_id = str(uuid.uuid4()) + record = _ProjectRecord( + id=project_id, + name=project_name, + directory=project.path, + state=project.state, + parameters={}, + current_cycle=project.current_cycle, + ) + _set_project(project_id, record) + _save_registry() + + return _to_response(record) + + +@router.get("/{project_id}", response_model=ProjectResponse) +async def get_project(project_id: str) -> ProjectResponse: + """Return the current state and metadata for a project. + + Returns 404 if the project ID is not found. + """ + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + + return _to_response(record) + + +@router.get("/{project_id}/statistics", response_model=ProjectStatisticsResponse) +async def get_project_statistics(project_id: str) -> ProjectStatisticsResponse: + """Return computed statistics for a project. + + Inspects the project directory to count particles and estimate resolution. + Returns 404 if the project ID is not found. + """ + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + + project_dir = Path(record.directory) + tilt_series = _project_service.list_tilt_series(record.directory) + + particle_count = _count_particles(project_dir) + resolution = _detect_best_resolution(project_dir) + + return ProjectStatisticsResponse( + project_id=project_id, + particle_count=particle_count, + resolution_angstrom=resolution, + tilt_series_count=len(tilt_series), + ) + + +@router.get("/{project_id}/tilt-series") +async def list_tilt_series(project_id: str) -> TiltSeriesListResponse: + """List tilt series for a project. + + Returns an empty list for new projects with no data in rawData/. + Returns 404 if the project ID is not found. + """ + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + + tilt_series = _project_service.list_tilt_series(record.directory) + + return TiltSeriesListResponse(tilt_series=tilt_series) + + +@router.get("/{project_id}/settings") +async def get_project_settings(project_id: str) -> ProjectSettings: + """Return the settings for a project.""" + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + return record.settings + + +@router.patch("/{project_id}/settings") +async def update_project_settings(project_id: str, patch: ProjectSettingsPatch) -> ProjectSettings: + """Partial update of project settings. + + Accepts a typed partial-update model, merges provided fields with + existing settings. Uses locked write pattern for concurrent safety. + """ + record = _get_project(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + + # Get only the fields that were explicitly provided in the request body + provided = patch.model_dump(exclude_unset=True) + + with _registry_lock: + record = _projects.get(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + + # Merge: existing settings dict + provided fields + existing = record.settings.model_dump() + + for key, value in provided.items(): + if key == "run_profiles": + if value is None: + raise HTTPException( + status_code=422, + detail="run_profiles must be a list, not null", + ) + existing["run_profiles"] = value + elif key == "executable_paths": + if value is None: + raise HTTPException( + status_code=422, + detail="executable_paths must be an object, not null", + ) + existing["executable_paths"] = value + elif key == "system_params" and value is not None and isinstance(value, dict): + if existing.get("system_params") is not None: + existing["system_params"].update(value) + else: + existing["system_params"] = value + else: + existing[key] = value + + try: + record.settings = ProjectSettings(**existing) + except ValidationError as exc: + raise HTTPException( + status_code=422, + detail=exc.errors(), + ) + + _save_registry() + return record.settings diff --git a/backend/api/v1_system.py b/backend/api/v1_system.py new file mode 100644 index 00000000..d0090abf --- /dev/null +++ b/backend/api/v1_system.py @@ -0,0 +1,32 @@ +"""V1 API endpoints for system information (GPU, CPU, memory). + +Routes: + GET /api/v1/system/info - Full system information + GET /api/v1/system/gpus - Detect available NVIDIA GPUs +""" + +from __future__ import annotations + +from fastapi import APIRouter + +from backend.services.system_service import GpuInfo, SystemInfo, SystemService + +router = APIRouter(prefix="/api/v1/system", tags=["system-v1"]) + +_service = SystemService() + + +@router.get("/info", response_model=SystemInfo) +async def get_system_info_v1() -> SystemInfo: + """Return system information including CPU cores, RAM, hostname, and GPUs.""" + return _service.get_system_info() + + +@router.get("/gpus", response_model=list[GpuInfo]) +async def detect_gpus_v1() -> list[GpuInfo]: + """Detect NVIDIA GPUs via nvidia-smi. + + Returns an empty list if no GPUs are found or nvidia-smi is + not available. + """ + return _service.detect_gpus() diff --git a/backend/api/v1_utilities.py b/backend/api/v1_utilities.py new file mode 100644 index 00000000..af70542c --- /dev/null +++ b/backend/api/v1_utilities.py @@ -0,0 +1,178 @@ +"""V1 API endpoints for standalone utility operations. + +Routes: + POST /api/v1/utilities/check - Run emClarity check to verify installation + POST /api/v1/utilities/mask - Run emClarity mask command + POST /api/v1/utilities/rescale - Run emClarity rescale command + POST /api/v1/utilities/geometry - Run emClarity geometry operations +""" + +from __future__ import annotations + +import shlex +import subprocess + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +router = APIRouter(prefix="/api/v1/utilities", tags=["utilities-v1"]) + +# --------------------------------------------------------------------------- +# Response models +# --------------------------------------------------------------------------- + +VALID_GEOMETRY_OPERATIONS: set[str] = { + "RemoveClasses", + "RemoveFraction", + "RemoveLowScoringParticles", + "RestoreParticles", + "PrintGeometry", +} + + +class CheckResult(BaseModel): + """Result of running emClarity check.""" + + success: bool = Field(..., description="Whether the check passed") + output: str = Field(default="", description="Standard output from the command") + errors: str = Field(default="", description="Standard error output from the command") + + +class CommandResult(BaseModel): + """Result of running an emClarity utility command.""" + + success: bool = Field(..., description="Whether the command succeeded") + output: str = Field(default="", description="Command output (stdout + stderr)") + command: str = Field(default="", description="The command that was executed") + + +class GeometryResult(CommandResult): + """Result of running an emClarity geometry operation.""" + + operation: str = Field(..., description="The geometry operation that was run") + + +# --------------------------------------------------------------------------- +# Request models +# --------------------------------------------------------------------------- + + +class MaskRequest(BaseModel): + """Parameters for the emClarity mask command.""" + + param_file: str = Field(..., description="Path to the parameter file (.m)") + tilt_series_name: str = Field(..., description="Name of the tilt series") + + +class RescaleRequest(BaseModel): + """Parameters for the emClarity rescale command.""" + + param_file: str = Field(..., description="Path to the parameter file (.m)") + target_pixel_size: float = Field( + ..., + gt=0, + description="Target pixel size in Angstroms", + ) + + +class GeometryRequest(BaseModel): + """Parameters for an emClarity geometry operation.""" + + param_file: str = Field(..., description="Path to the parameter file (.m)") + operation: str = Field(..., description="Geometry operation to perform") + cycle: int = Field(default=1, ge=1, description="Processing cycle number") + + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + + +def _run(cmd: list[str], timeout: int = 300) -> tuple[bool, str, str]: + """Run a subprocess command and return (success, stdout, stderr).""" + try: + result = subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout, + ) + return result.returncode == 0, result.stdout, result.stderr + except FileNotFoundError: + return ( + False, + "", + "emClarity executable not found in PATH. " + "Please ensure emClarity is installed and accessible.", + ) + except subprocess.TimeoutExpired as exc: + raise HTTPException( + status_code=504, + detail=f"Command timed out after {timeout} seconds", + ) from exc + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + + +@router.post("/check", response_model=CheckResult) +async def run_system_check() -> CheckResult: + """Run emClarity check to verify installation and all dependencies.""" + success, stdout, stderr = _run(["emClarity", "check"], timeout=60) + return CheckResult(success=success, output=stdout, errors=stderr) + + +@router.post("/mask", response_model=CommandResult) +async def run_mask(request: MaskRequest) -> CommandResult: + """Run emClarity mask command to create particle masks.""" + cmd = ["emClarity", "mask", request.param_file, request.tilt_series_name] + cmd_str = shlex.join(cmd) + success, stdout, stderr = _run(cmd) + combined = stdout + (f"\n{stderr}" if stderr else "") + return CommandResult(success=success, output=combined.strip(), command=cmd_str) + + +@router.post("/rescale", response_model=CommandResult) +async def run_rescale(request: RescaleRequest) -> CommandResult: + """Run emClarity rescale command to rescale volumes to a new pixel size.""" + cmd = [ + "emClarity", + "rescale", + request.param_file, + str(request.target_pixel_size), + ] + cmd_str = shlex.join(cmd) + success, stdout, stderr = _run(cmd) + combined = stdout + (f"\n{stderr}" if stderr else "") + return CommandResult(success=success, output=combined.strip(), command=cmd_str) + + +@router.post("/geometry", response_model=GeometryResult) +async def run_geometry(request: GeometryRequest) -> GeometryResult: + """Run an emClarity geometry operation on the particle set.""" + if request.operation not in VALID_GEOMETRY_OPERATIONS: + raise HTTPException( + status_code=422, + detail=( + f"Unknown geometry operation: {request.operation!r}. " + f"Valid operations: {sorted(VALID_GEOMETRY_OPERATIONS)}" + ), + ) + cmd = [ + "emClarity", + "geometry", + request.operation, + request.param_file, + str(request.cycle), + ] + cmd_str = shlex.join(cmd) + success, stdout, stderr = _run(cmd) + combined = stdout + (f"\n{stderr}" if stderr else "") + return GeometryResult( + success=success, + output=combined.strip(), + command=cmd_str, + operation=request.operation, + ) diff --git a/backend/api/v1_viewer.py b/backend/api/v1_viewer.py new file mode 100644 index 00000000..d5303123 --- /dev/null +++ b/backend/api/v1_viewer.py @@ -0,0 +1,104 @@ +"""V1 Viewer launcher API endpoint. + +Provides endpoints to launch an external viewer program (e.g. 3dmod, ChimeraX) +as a non-blocking subprocess. + +POST /api/v1/viewer/launch + body: { viewer_path: str, args: list[str] } + Returns { launched: true, pid: int } on success. + 400 if path is not executable (exists but not executable). + 404 if path does not exist. + Subprocess launched with list form, never shell=True. + +GET /api/v1/viewer/default + Returns { viewer_path: str | null } — the currently stored default viewer path. + +PUT /api/v1/viewer/default + body: { viewer_path: str } + Validates path exists (does not need to be running, just a file that exists). + Stores it in module-level variable. Returns 200 on success. +""" + +from __future__ import annotations + +import os +import subprocess + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel + +router = APIRouter(prefix="/api/v1/viewer", tags=["viewer-v1"]) + +_default_viewer_path: str | None = None + + +class LaunchRequest(BaseModel): + viewer_path: str + args: list[str] = [] + + +class LaunchResponse(BaseModel): + launched: bool + pid: int + + +class DefaultViewerResponse(BaseModel): + viewer_path: str | None + + +class SetDefaultRequest(BaseModel): + viewer_path: str + + +@router.post("/launch", response_model=LaunchResponse) +def launch_viewer(request: LaunchRequest) -> LaunchResponse: + """Launch an external viewer as a non-blocking subprocess.""" + viewer_path = request.viewer_path + + if not os.path.exists(viewer_path): + raise HTTPException(status_code=404, detail=f"Viewer not found: {viewer_path}") + + if not os.path.isfile(viewer_path): + raise HTTPException(status_code=400, detail=f"Path is not a file: {viewer_path}") + + if not os.access(viewer_path, os.X_OK): + raise HTTPException( + status_code=400, detail=f"Path is not executable: {viewer_path}" + ) + + try: + proc = subprocess.Popen( + [viewer_path] + request.args, + start_new_session=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + except OSError as exc: + raise HTTPException( + status_code=500, detail=f"OS error launching viewer: {exc}" + ) from exc + + return LaunchResponse(launched=True, pid=proc.pid) + + +@router.get("/default", response_model=DefaultViewerResponse) +def get_default_viewer() -> DefaultViewerResponse: + """Return the currently stored default viewer path.""" + return DefaultViewerResponse(viewer_path=_default_viewer_path) + + +@router.put("/default", response_model=DefaultViewerResponse) +def set_default_viewer(request: SetDefaultRequest) -> DefaultViewerResponse: + """Set the default viewer path. Validates that the path exists.""" + global _default_viewer_path + + viewer_path = request.viewer_path + + if not os.path.exists(viewer_path): + raise HTTPException(status_code=404, detail=f"Viewer not found: {viewer_path}") + + if not os.path.isfile(viewer_path): + raise HTTPException(status_code=400, detail=f"Path is not a file: {viewer_path}") + + _default_viewer_path = viewer_path + return DefaultViewerResponse(viewer_path=_default_viewer_path) diff --git a/backend/api/v1_workflow.py b/backend/api/v1_workflow.py new file mode 100644 index 00000000..177b9bfe --- /dev/null +++ b/backend/api/v1_workflow.py @@ -0,0 +1,327 @@ +"""V1 Workflow state machine API endpoints. + +Provides workflow state management endpoints for emClarity projects using +UUID-based project identifiers. + +Routes: + GET /api/v1/workflow/state-machine - Full state machine definition + GET /api/v1/workflow/{project_id}/available-commands - Commands available in current state + POST /api/v1/workflow/{project_id}/run - Run a command (enforces prerequisites) +""" + +from __future__ import annotations + +from typing import Any + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from backend.models.project import ProjectState + +router = APIRouter(prefix="/api/v1/workflow", tags=["workflow-v1"]) + +# --------------------------------------------------------------------------- +# State machine definition +# --------------------------------------------------------------------------- + +# Maps each project state to the set of commands that are allowed (enabled) +# in that state. Only commands whose prerequisites have been completed +# should appear here. +_STATE_ALLOWED_COMMANDS: dict[ProjectState, list[str]] = { + ProjectState.UNINITIALIZED: [ + "autoAlign", + ], + ProjectState.TILT_ALIGNED: [ + "autoAlign", + "ctf estimate", + ], + ProjectState.CTF_ESTIMATED: [ + "autoAlign", + "ctf estimate", + "ctf 3d", + ], + ProjectState.RECONSTRUCTED: [ + "autoAlign", + "ctf estimate", + "ctf 3d", + "templateSearch", + ], + ProjectState.PARTICLES_PICKED: [ + "autoAlign", + "ctf estimate", + "ctf 3d", + "templateSearch", + "init", + ], + ProjectState.INITIALIZED: [ + "autoAlign", + "ctf estimate", + "ctf 3d", + "templateSearch", + "init", + "avg", + ], + ProjectState.CYCLE_N: [ + "autoAlign", + "ctf estimate", + "ctf 3d", + "templateSearch", + "init", + "avg", + "alignRaw", + "tomoCPR", + "pca", + "cluster", + "fsc", + "reconstruct", + ], + ProjectState.EXPORT: [ + "autoAlign", + "ctf estimate", + "ctf 3d", + "templateSearch", + "init", + "avg", + "alignRaw", + "tomoCPR", + "pca", + "cluster", + "fsc", + "reconstruct", + ], + ProjectState.DONE: [], +} + +# Full state machine definition for client consumption +_STATE_MACHINE: dict[str, Any] = { + "states": { + "UNINITIALIZED": { + "description": "No processing has started; raw data only", + "available_commands": ["autoAlign"], + "transitions": { + "autoAlign": "TILT_ALIGNED", + }, + }, + "TILT_ALIGNED": { + "description": "Tilt-series alignment complete", + "available_commands": ["autoAlign", "ctf estimate"], + "transitions": { + "ctf estimate": "CTF_ESTIMATED", + }, + }, + "CTF_ESTIMATED": { + "description": "CTF estimation complete", + "available_commands": ["autoAlign", "ctf estimate", "ctf 3d"], + "transitions": { + "ctf 3d": "RECONSTRUCTED", + }, + }, + "RECONSTRUCTED": { + "description": "3D tomograms reconstructed", + "available_commands": ["autoAlign", "ctf estimate", "ctf 3d", "templateSearch"], + "transitions": { + "templateSearch": "PARTICLES_PICKED", + }, + }, + "PARTICLES_PICKED": { + "description": "Template search / particle picking complete", + "available_commands": [ + "autoAlign", "ctf estimate", "ctf 3d", "templateSearch", "init", + ], + "transitions": { + "init": "INITIALIZED", + }, + }, + "INITIALIZED": { + "description": "Project initialised; subTomoMeta.mat created", + "available_commands": [ + "autoAlign", "ctf estimate", "ctf 3d", "templateSearch", "init", "avg", + ], + "transitions": { + "avg": "CYCLE_N", + }, + }, + "CYCLE_N": { + "description": "Iterative alignment/averaging in progress", + "available_commands": [ + "autoAlign", "ctf estimate", "ctf 3d", "templateSearch", "init", + "avg", "alignRaw", "tomoCPR", "pca", "cluster", "fsc", "reconstruct", + ], + "transitions": { + "reconstruct": "EXPORT", + }, + }, + "EXPORT": { + "description": "Final reconstruction exported to cisTEM format", + "available_commands": [ + "autoAlign", "ctf estimate", "ctf 3d", "templateSearch", "init", + "avg", "alignRaw", "tomoCPR", "pca", "cluster", "fsc", "reconstruct", + ], + "transitions": {}, + }, + "DONE": { + "description": "Processing complete", + "available_commands": [], + "transitions": {}, + }, + }, + "initial_state": "UNINITIALIZED", +} + +# --------------------------------------------------------------------------- +# Request / response models +# --------------------------------------------------------------------------- + + +class CommandEntry(BaseModel): + """Describes a single available command.""" + + name: str = Field(..., description="Command name as passed to emClarity CLI") + description: str = Field(default="", description="Human-readable description") + + +class AvailableCommandsResponse(BaseModel): + """Response body for the available-commands endpoint.""" + + project_id: str + state: str + commands: list[CommandEntry] + + +class RunCommandRequest(BaseModel): + """Request body for running a workflow command.""" + + command: str = Field(..., description="emClarity command to execute") + args: dict[str, Any] = Field( + default_factory=dict, + description="Command-specific arguments", + ) + param_file: str | None = Field( + default=None, + description="Path to the .m parameter file for this run", + ) + + +class RunCommandResponse(BaseModel): + """Response body after successfully accepting a run request.""" + + project_id: str + command: str + status: str = "accepted" + message: str = "" + param_file: str | None = None + + +# --------------------------------------------------------------------------- +# Human-readable descriptions per command +# --------------------------------------------------------------------------- + +_COMMAND_DESCRIPTIONS: dict[str, str] = { + "autoAlign": "Align raw tilt-series images using fiducial or patch tracking", + "ctf estimate": "Estimate defocus and astigmatism for each tilt image", + "ctf 3d": "Apply 3D CTF correction and reconstruct tomograms", + "templateSearch": "Search for particles using a 3D template", + "init": "Initialise the sub-tomogram averaging project", + "avg": "Compute the sub-tomogram average from aligned particles", + "alignRaw": "Refine particle orientations against the current average", + "tomoCPR": "Refine tilt-series geometry using current particle positions", + "pca": "Principal component analysis for heterogeneity detection", + "cluster": "Classify particles based on PCA eigenvectors", + "fsc": "Compute Fourier Shell Correlation for resolution estimation", + "reconstruct": "Generate the final high-resolution 3D reconstruction", +} + + +# --------------------------------------------------------------------------- +# Shared project registry accessor +# --------------------------------------------------------------------------- + +def _get_projects() -> dict[str, Any]: + """Return the shared in-memory project registry. + + Imported lazily to avoid circular-import issues at module load time. + """ + from backend.api.v1_projects import _projects # noqa: PLC0415 + return _projects # type: ignore[return-value] + + +# --------------------------------------------------------------------------- +# Routes +# --------------------------------------------------------------------------- + + +@router.get("/state-machine") +async def get_state_machine() -> dict[str, Any]: + """Return the full state machine definition. + + Describes every state, its available commands, and allowed transitions. + Clients use this to render pipeline progress UIs and validate user actions. + """ + return _STATE_MACHINE + + +@router.get("/{project_id}/available-commands", response_model=AvailableCommandsResponse) +async def get_available_commands(project_id: str) -> AvailableCommandsResponse: + """Return the commands available for a project in its current state. + + Only commands whose prerequisites have been satisfied are included. + Returns 404 if the project ID is unknown. + """ + projects = _get_projects() + record = projects.get(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + + state: ProjectState = record.state + allowed_names = _STATE_ALLOWED_COMMANDS.get(state, []) + + commands = [ + CommandEntry( + name=name, + description=_COMMAND_DESCRIPTIONS.get(name, ""), + ) + for name in allowed_names + ] + + return AvailableCommandsResponse( + project_id=project_id, + state=state.value.upper(), + commands=commands, + ) + + +@router.post("/{project_id}/run", response_model=RunCommandResponse) +async def run_command(project_id: str, request: RunCommandRequest) -> RunCommandResponse: + """Execute an emClarity pipeline command for a project. + + Enforces the state machine: returns 409 if the requested command is not + available in the project's current state (i.e. prerequisites not met). + Returns 404 if the project ID is unknown. + """ + projects = _get_projects() + record = projects.get(project_id) + if record is None: + raise HTTPException(status_code=404, detail=f"Project '{project_id}' not found") + + state: ProjectState = record.state + allowed_names = _STATE_ALLOWED_COMMANDS.get(state, []) + + if request.command not in allowed_names: + raise HTTPException( + status_code=409, + detail=( + f"Command '{request.command}' is not available in state " + f"'{state.value.upper()}'. " + f"Available commands: {allowed_names}" + ), + ) + + # Command is permitted – in a full implementation this would launch a job. + # For now we return 'accepted' and let the caller poll for job status. + return RunCommandResponse( + project_id=project_id, + command=request.command, + status="accepted", + message=f"Command '{request.command}' accepted for execution", + param_file=request.param_file, + ) diff --git a/backend/api/workflow.py b/backend/api/workflow.py new file mode 100644 index 00000000..90f20e14 --- /dev/null +++ b/backend/api/workflow.py @@ -0,0 +1,128 @@ +"""API endpoints for workflow management and command execution. + +Routes: + GET /api/workflow/commands - List available pipeline commands + POST /api/workflow/execute - Execute a pipeline command + GET /api/workflow/state/{path} - Get pipeline state for a project +""" + +from __future__ import annotations + +from fastapi import APIRouter, HTTPException + +from backend.models.job import Job +from backend.models.workflow import CommandInfo, CommandRequest, WorkflowState +from backend.services.job_service import JobService +from backend.services.project_service import ProjectService +from backend.services.workflow_service import WorkflowService + +router = APIRouter(prefix="/api/workflow", tags=["workflow"]) + +_workflow_service = WorkflowService() +_project_service = ProjectService() +_job_service = JobService() + + +@router.get("/commands", response_model=list[CommandInfo]) +async def list_commands() -> list[CommandInfo]: + """Return metadata for all available pipeline commands. + + Includes human-readable labels, descriptions, and prerequisite + information that the frontend uses to enable/disable buttons. + """ + return _workflow_service.list_commands() + + +@router.post("/execute", response_model=Job) +async def execute_command(request: CommandRequest) -> Job: + """Execute an emClarity pipeline command. + + Launches the command as a background subprocess and returns a Job + object that can be polled for status. The project must have a + parameter file set. + """ + # We need a project path from the request parameters + project_path = request.parameters.get("project_path") + if not project_path: + raise HTTPException( + status_code=400, + detail="'project_path' must be provided in parameters", + ) + + # Load project to find the parameter file + try: + project = _project_service.load_project(project_path) + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Project not found: {project_path}") + + param_file = project.parameter_file + if not param_file: + raise HTTPException( + status_code=400, + detail="No parameter file found in project directory", + ) + + # Build and launch the CLI command + cli_args = _workflow_service.build_cli_command(request, param_file) + job = _job_service.start_job( + command=request.command, + cli_args=cli_args, + project_path=project_path, + ) + + return job + + +@router.get("/state/{path:path}", response_model=WorkflowState) +async def get_workflow_state(path: str) -> WorkflowState: + """Determine which commands are available for a project. + + Inspects the project directory to figure out which steps have + been completed, then calculates which commands can be run next. + """ + try: + project = _project_service.load_project(f"/{path}") + except FileNotFoundError: + raise HTTPException(status_code=404, detail=f"Project not found: /{path}") + + # Map project state to completed commands (simplified heuristic) + from backend.models.project import ProjectState + from backend.models.workflow import PipelineCommand + + state_to_commands: dict[ProjectState, list[PipelineCommand]] = { + ProjectState.UNINITIALIZED: [], + ProjectState.TILT_ALIGNED: [PipelineCommand.AUTO_ALIGN], + ProjectState.CTF_ESTIMATED: [ + PipelineCommand.AUTO_ALIGN, + PipelineCommand.CTF_ESTIMATE, + ], + ProjectState.RECONSTRUCTED: [ + PipelineCommand.AUTO_ALIGN, + PipelineCommand.CTF_ESTIMATE, + PipelineCommand.CTF_3D, + ], + ProjectState.PARTICLES_PICKED: [ + PipelineCommand.AUTO_ALIGN, + PipelineCommand.CTF_ESTIMATE, + PipelineCommand.CTF_3D, + PipelineCommand.TEMPLATE_SEARCH, + ], + ProjectState.INITIALIZED: [ + PipelineCommand.AUTO_ALIGN, + PipelineCommand.CTF_ESTIMATE, + PipelineCommand.CTF_3D, + PipelineCommand.TEMPLATE_SEARCH, + PipelineCommand.INIT, + ], + ProjectState.CYCLE_N: [ + PipelineCommand.AUTO_ALIGN, + PipelineCommand.CTF_ESTIMATE, + PipelineCommand.CTF_3D, + PipelineCommand.TEMPLATE_SEARCH, + PipelineCommand.INIT, + PipelineCommand.AVG, + ], + } + + completed = state_to_commands.get(project.state, []) + return _workflow_service.get_workflow_state(completed) diff --git a/backend/main.py b/backend/main.py new file mode 100644 index 00000000..6187cb7e --- /dev/null +++ b/backend/main.py @@ -0,0 +1,80 @@ +"""FastAPI application entry point for the emClarity backend. + +Start the server with: + uvicorn backend.main:app --reload --port 8000 + +The backend serves both the REST API and the production React frontend +(built to frontend/dist/). All /api/* requests are handled by the API +router; everything else falls through to the SPA's index.html. +""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.responses import FileResponse, HTMLResponse +from fastapi.staticfiles import StaticFiles + +from backend.api.router import router + +app = FastAPI( + title="emClarity Backend", + description="REST API for the emClarity cryo-EM processing pipeline", + version="0.1.0", +) + +# CORS — only needed when the frontend is served from a separate origin +# (e.g. a Vite dev server during active frontend development). In the +# normal production workflow the frontend is served from the same origin +# so CORS is not involved. +app.add_middleware( + CORSMiddleware, + allow_origins=[ + "http://localhost:5173", # Vite dev server (default port) + "http://127.0.0.1:5173", + ], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(router) + + +@app.get("/api/health") +async def health_check(): + """Simple health check endpoint.""" + return {"status": "ok", "service": "emClarity backend"} + + +# --------------------------------------------------------------------------- +# Serve the production React build (frontend/dist/) +# --------------------------------------------------------------------------- + +_FRONTEND_DIR = Path(__file__).resolve().parent.parent / "frontend" / "dist" + +if _FRONTEND_DIR.is_dir(): + # Serve static assets (JS, CSS, images) at /assets/... + _assets_dir = _FRONTEND_DIR / "assets" + if _assets_dir.is_dir(): + app.mount( + "/assets", + StaticFiles(directory=str(_assets_dir)), + name="frontend-assets", + ) + + @app.get("/{path:path}", response_model=None) + async def spa_fallback(request: Request, path: str): + """Serve static files from the frontend build, falling back to + index.html for client-side routes (SPA behaviour).""" + # Try to serve the exact file (favicon.svg, etc.) + file_path = _FRONTEND_DIR / path + if path and file_path.is_file(): + return FileResponse(file_path) + # Everything else → index.html (React Router handles the route) + index = _FRONTEND_DIR / "index.html" + if index.is_file(): + return FileResponse(index) + return HTMLResponse("Frontend not built. Run: cd frontend && npm run build", status_code=503) diff --git a/backend/models/__init__.py b/backend/models/__init__.py new file mode 100644 index 00000000..57d42bca --- /dev/null +++ b/backend/models/__init__.py @@ -0,0 +1,20 @@ +from backend.models.parameter import ParameterDefinition, ParameterFile, ParameterValue +from backend.models.project import Project, ProjectState +from backend.models.project_settings import ProjectSettings, ProjectSettingsPatch, RunProfile +from backend.models.workflow import CommandRequest, PipelineCommand +from backend.models.job import Job, JobStatus + +__all__ = [ + "ParameterDefinition", + "ParameterFile", + "ParameterValue", + "Project", + "ProjectState", + "ProjectSettings", + "ProjectSettingsPatch", + "RunProfile", + "CommandRequest", + "PipelineCommand", + "Job", + "JobStatus", +] diff --git a/backend/models/job.py b/backend/models/job.py new file mode 100644 index 00000000..ef6ab220 --- /dev/null +++ b/backend/models/job.py @@ -0,0 +1,49 @@ +"""Pydantic models for job tracking and subprocess management. + +Each emClarity command execution is tracked as a Job with its own +process ID, log file, and status lifecycle. +""" + +from __future__ import annotations + +from datetime import datetime +from enum import Enum + +from pydantic import BaseModel, Field + +from backend.models.workflow import PipelineCommand + + +class JobStatus(str, Enum): + """Lifecycle states for a running job.""" + + PENDING = "pending" + RUNNING = "running" + COMPLETED = "completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +class Job(BaseModel): + """Represents a single command execution.""" + + id: str = Field(..., description="Unique job identifier (UUID)") + command: PipelineCommand = Field(..., description="The pipeline command being executed") + status: JobStatus = Field(default=JobStatus.PENDING, description="Current job status") + pid: int | None = Field(default=None, description="OS process ID (set when running)") + start_time: datetime | None = Field(default=None, description="When the job started") + end_time: datetime | None = Field(default=None, description="When the job finished") + log_path: str | None = Field(default=None, description="Path to the job's log file") + exit_code: int | None = Field(default=None, description="Process exit code (set when done)") + project_path: str = Field(..., description="Project this job belongs to") + error_message: str | None = Field( + default=None, + description="Error summary if the job failed", + ) + + +class JobListResponse(BaseModel): + """Response model for listing jobs.""" + + jobs: list[Job] = Field(default_factory=list) + total: int = Field(default=0, description="Total number of jobs") diff --git a/backend/models/parameter.py b/backend/models/parameter.py new file mode 100644 index 00000000..cc2a541b --- /dev/null +++ b/backend/models/parameter.py @@ -0,0 +1,150 @@ +"""Pydantic models for emClarity parameter handling. + +Parameters control every aspect of the cryo-EM processing pipeline. +Each parameter has a definition (schema) and a concrete value when used +in a parameter file. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + + +class ParameterType(str, Enum): + """Supported parameter value types. + + The golden schema (from BH_parseParameterFile.m) uses: numeric, + numeric_array, string, boolean. Legacy backend code also references + integer, float, vector, and enum; those are kept for backward + compatibility with existing backend tests. + """ + + STRING = "string" + INTEGER = "integer" + FLOAT = "float" + BOOLEAN = "boolean" + VECTOR = "vector" # e.g., [1, 2, 3] - common in cryo-EM for 3D dimensions + ENUM = "enum" # constrained string choice + NUMERIC = "numeric" # golden schema numeric type + NUMERIC_ARRAY = "numeric_array" # golden schema array type + + +class ParameterCategory(str, Enum): + """Logical groupings for parameters in the UI. + + The golden schema uses: alignment, classification, ctf, + disk_management, dose, fsc, hardware, masking, metadata, microscope, + template_matching, tomoCPR. Legacy values (general, reconstruction, + templateSearch, tomogram, system) are kept for backward compatibility. + """ + + GENERAL = "general" + ALIGNMENT = "alignment" + CTF = "ctf" + RECONSTRUCTION = "reconstruction" + MASKING = "masking" + CLASSIFICATION = "classification" + TEMPLATE_SEARCH = "templateSearch" + TOMOGRAM = "tomogram" + SYSTEM = "system" + MICROSCOPE = "microscope" + HARDWARE = "hardware" + METADATA = "metadata" + DISK_MANAGEMENT = "disk_management" + DOSE = "dose" + FSC = "fsc" + TEMPLATE_MATCHING = "template_matching" + TOMOCPR = "tomoCPR" + + +class ParameterDefinition(BaseModel): + """Schema definition for a single emClarity parameter. + + Describes the name, type, constraints, and documentation for a + parameter. Used by the frontend to render appropriate input widgets + and perform client-side validation. + """ + + name: str = Field(..., description="Parameter name as used in the .m parameter file") + type: ParameterType = Field(..., description="Value type for validation and UI rendering") + required: bool = Field(default=False, description="Whether the parameter must be set") + default: Any = Field(default=None, description="Default value if not explicitly set") + range: list[float] | None = Field( + default=None, + description="[min, max] range for numeric parameters", + ) + allowed_values: list[Any] | None = Field( + default=None, + description="Allowed values for enum-type parameters", + ) + description: str = Field(default="", description="Human-readable description") + category: ParameterCategory = Field( + default=ParameterCategory.GENERAL, + description="UI grouping category", + ) + units: str | None = Field(default=None, description="Physical units (e.g., angstroms, degrees)") + + +class ParameterSchemaResponse(BaseModel): + """API response wrapper for the parameter schema. + + The ``/api/v1/parameters/schema`` endpoint returns this object so + that the frontend receives a JSON object ``{"parameters": [...]}`` + rather than a bare array. + """ + + parameters: list[ParameterDefinition] = Field( + ..., + description="List of parameter definitions from the golden schema", + ) + + +class ParameterValue(BaseModel): + """A concrete parameter name-value pair.""" + + name: str = Field(..., description="Parameter name") + value: Any = Field(..., description="Parameter value") + + +class ParameterFile(BaseModel): + """Represents a complete emClarity parameter file. + + Contains all parameter values and the filesystem path where the + file is (or will be) stored. + """ + + parameters: list[ParameterValue] = Field( + default_factory=list, + description="List of parameter name-value pairs", + ) + path: str = Field(..., description="Filesystem path to the parameter file") + + +class ParameterValidationResult(BaseModel): + """Result of validating a set of parameter values against the schema.""" + + valid: bool = Field(..., description="Whether all parameters passed validation") + errors: list[str] = Field( + default_factory=list, + description="List of validation error messages", + ) + warnings: list[str] = Field( + default_factory=list, + description="List of non-fatal validation warnings", + ) + + +class ParameterValidationRequest(BaseModel): + """Request body for the v1 parameter validation endpoint. + + Accepts parameters as a flat dict ``{name: value}`` which is more + ergonomic than the legacy list-of-objects format. + """ + + parameters: dict[str, Any] = Field( + ..., + description="Flat mapping of parameter name to value", + ) diff --git a/backend/models/project.py b/backend/models/project.py new file mode 100644 index 00000000..bbc96621 --- /dev/null +++ b/backend/models/project.py @@ -0,0 +1,73 @@ +"""Pydantic models for emClarity project state management. + +An emClarity project progresses through a series of well-defined states +as tilt-series data is processed from raw micrographs to final 3D +reconstructions. +""" + +from __future__ import annotations + +from enum import Enum + +from pydantic import BaseModel, Field + + +class ProjectState(str, Enum): + """Processing states an emClarity project passes through. + + The pipeline is sequential: each state requires all prior states + to have been completed successfully. + """ + + UNINITIALIZED = "uninitialized" + TILT_ALIGNED = "tilt_aligned" + CTF_ESTIMATED = "ctf_estimated" + RECONSTRUCTED = "reconstructed" + PARTICLES_PICKED = "particles_picked" + INITIALIZED = "initialized" + CYCLE_N = "cycle_n" + EXPORT = "export" + DONE = "done" + + +class TiltSeries(BaseModel): + """Metadata for a single tilt series within a project.""" + + name: str = Field(..., description="Tilt series identifier (e.g., 'tilt1')") + stack_path: str | None = Field( + default=None, + description="Path to the raw tilt-series stack (.st file)", + ) + rawtlt_path: str | None = Field( + default=None, + description="Path to the raw tilt angles file (.rawtlt)", + ) + aligned: bool = Field(default=False, description="Whether tilt-series alignment is complete") + ctf_estimated: bool = Field(default=False, description="Whether CTF has been estimated") + + +class Project(BaseModel): + """Top-level representation of an emClarity project. + + Tracks the overall processing state and the collection of tilt + series being processed. + """ + + name: str = Field(..., description="Project name") + path: str = Field(..., description="Absolute path to the project directory") + state: ProjectState = Field( + default=ProjectState.UNINITIALIZED, + description="Current pipeline state", + ) + current_cycle: int = Field( + default=0, + description="Current refinement cycle number (0 = not yet cycling)", + ) + tilt_series: list[TiltSeries] = Field( + default_factory=list, + description="Tilt series in this project", + ) + parameter_file: str | None = Field( + default=None, + description="Path to the active parameter file", + ) diff --git a/backend/models/project_settings.py b/backend/models/project_settings.py new file mode 100644 index 00000000..364a8c28 --- /dev/null +++ b/backend/models/project_settings.py @@ -0,0 +1,40 @@ +"""Typed Pydantic models for per-project settings.""" +from __future__ import annotations + +from typing import Any + +from pydantic import BaseModel, Field + +# Sentinel to distinguish "field not provided" from "field set to None" +_UNSET = object() + + +class RunProfile(BaseModel): + """A single run profile configuration.""" + name: str + gpu_count: int = 1 + cpu_cores: int = 4 + scratch_disk: str | None = None + command_template: str | None = None + + +class ProjectSettings(BaseModel): + """Per-project settings stored in the project registry.""" + run_profiles: list[RunProfile] = Field(default_factory=list) + selected_run_profile: str | None = None + system_params: dict[str, Any] | None = None + viewer_path: str | None = None + executable_paths: dict[str, str] = Field(default_factory=dict) + + +class ProjectSettingsPatch(BaseModel): + """Typed partial-update model for PATCH /settings (defect 9). + + All fields are optional. Only provided fields are merged into the + existing ProjectSettings. + """ + run_profiles: list[RunProfile] | None = None + selected_run_profile: str | None = None + system_params: dict[str, Any] | None = None + viewer_path: str | None = None + executable_paths: dict[str, str] | None = None diff --git a/backend/models/workflow.py b/backend/models/workflow.py new file mode 100644 index 00000000..c858b287 --- /dev/null +++ b/backend/models/workflow.py @@ -0,0 +1,100 @@ +"""Pydantic models for emClarity workflow and pipeline commands. + +Each processing step in the emClarity pipeline maps to a command that +can be executed with specific parameters. This module defines the +command vocabulary and request/response models. +""" + +from __future__ import annotations + +from enum import Enum +from typing import Any + +from pydantic import BaseModel, Field + + +class PipelineCommand(str, Enum): + """Available emClarity processing commands. + + These map to the subcommands accepted by the `emClarity` entry point. + The ordering here reflects the typical processing sequence. + """ + + AUTO_ALIGN = "autoAlign" + CTF_ESTIMATE = "ctf estimate" + CTF_3D = "ctf 3d" + TEMPLATE_SEARCH = "templateSearch" + INIT = "init" + AVG = "avg" + ALIGN_RAW = "alignRaw" + TOMO_CPR = "tomoCPR" + PCA = "pca" + CLUSTER = "cluster" + FSC = "fsc" + RECONSTRUCT = "reconstruct" + + +# Defines which commands are available at each pipeline stage and their +# required predecessor commands. +COMMAND_PREREQUISITES: dict[PipelineCommand, list[PipelineCommand]] = { + PipelineCommand.AUTO_ALIGN: [], + PipelineCommand.CTF_ESTIMATE: [PipelineCommand.AUTO_ALIGN], + PipelineCommand.CTF_3D: [PipelineCommand.CTF_ESTIMATE], + PipelineCommand.TEMPLATE_SEARCH: [PipelineCommand.CTF_3D], + PipelineCommand.INIT: [PipelineCommand.TEMPLATE_SEARCH], + PipelineCommand.AVG: [PipelineCommand.INIT], + PipelineCommand.ALIGN_RAW: [PipelineCommand.AVG], + PipelineCommand.TOMO_CPR: [PipelineCommand.ALIGN_RAW], + PipelineCommand.PCA: [PipelineCommand.AVG], + PipelineCommand.CLUSTER: [PipelineCommand.PCA], + PipelineCommand.FSC: [PipelineCommand.AVG], + PipelineCommand.RECONSTRUCT: [PipelineCommand.AVG], +} + + +class CommandInfo(BaseModel): + """Describes a pipeline command for the frontend.""" + + command: PipelineCommand + label: str = Field(..., description="Human-readable command name") + description: str = Field(default="", description="What this command does") + prerequisites: list[PipelineCommand] = Field( + default_factory=list, + description="Commands that must complete before this one", + ) + + +class CommandRequest(BaseModel): + """Request to execute a pipeline command.""" + + command: PipelineCommand = Field(..., description="The command to execute") + parameters: dict[str, Any] = Field( + default_factory=dict, + description="Parameter overrides for this command", + ) + tilt_series_name: str | None = Field( + default=None, + description="Specific tilt series to process (None = all)", + ) + cycle: int = Field( + default=0, + description="Refinement cycle number", + ) + gpu_ids: list[int] | None = Field( + default=None, + description="GPU device IDs to use (None = auto-detect)", + ) + + +class WorkflowState(BaseModel): + """Current state of the processing pipeline for a project.""" + + completed_commands: list[PipelineCommand] = Field( + default_factory=list, + description="Commands that have completed successfully", + ) + available_commands: list[PipelineCommand] = Field( + default_factory=list, + description="Commands that can be run given the current state", + ) + current_cycle: int = Field(default=0, description="Current refinement cycle") diff --git a/backend/requirements.txt b/backend/requirements.txt new file mode 100644 index 00000000..73f37af6 --- /dev/null +++ b/backend/requirements.txt @@ -0,0 +1,4 @@ +fastapi>=0.100.0 +uvicorn[standard]>=0.23.0 +pydantic>=2.0.0 +python-multipart diff --git a/backend/services/__init__.py b/backend/services/__init__.py new file mode 100644 index 00000000..83886304 --- /dev/null +++ b/backend/services/__init__.py @@ -0,0 +1,13 @@ +from backend.services.parameter_service import ParameterService +from backend.services.project_service import ProjectService +from backend.services.workflow_service import WorkflowService +from backend.services.job_service import JobService +from backend.services.system_service import SystemService + +__all__ = [ + "ParameterService", + "ProjectService", + "WorkflowService", + "JobService", + "SystemService", +] diff --git a/backend/services/job_service.py b/backend/services/job_service.py new file mode 100644 index 00000000..a4eac4bc --- /dev/null +++ b/backend/services/job_service.py @@ -0,0 +1,195 @@ +"""Service for managing emClarity subprocess execution and monitoring. + +Each command execution is tracked as a Job with a unique ID, subprocess +PID, and log file. Jobs can be listed, inspected, and cancelled. +""" + +from __future__ import annotations + +import signal +import subprocess +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import AsyncIterator + +from backend.models.job import Job, JobStatus +from backend.models.workflow import PipelineCommand + + +class JobService: + """Manages emClarity subprocesses.""" + + def __init__(self) -> None: + # In-memory job registry (keyed by job ID) + self._jobs: dict[str, Job] = {} + # Active subprocess handles (keyed by job ID) + self._processes: dict[str, subprocess.Popen[bytes]] = {} + + def start_job( + self, + command: PipelineCommand, + cli_args: list[str], + project_path: str, + log_dir: str | None = None, + ) -> Job: + """Launch a command as a background subprocess. + + Args: + command: The pipeline command being run. + cli_args: Full CLI argument list (e.g., ["emClarity", "init", ...]). + project_path: Project directory (used as cwd). + log_dir: Directory for log files. Defaults to project_path/logFile/. + """ + job_id = str(uuid.uuid4()) + + if log_dir is None: + log_dir = str(Path(project_path) / "logFile") + Path(log_dir).mkdir(parents=True, exist_ok=True) + + log_path = str(Path(log_dir) / f"{command.value}_{job_id[:8]}.log") + + job = Job( + id=job_id, + command=command, + status=JobStatus.PENDING, + log_path=log_path, + project_path=project_path, + ) + + try: + log_fh = open(log_path, "wb") # noqa: SIM115 + proc = subprocess.Popen( + cli_args, + cwd=project_path, + stdout=log_fh, + stderr=subprocess.STDOUT, + ) + job.status = JobStatus.RUNNING + job.pid = proc.pid + job.start_time = datetime.now(tz=timezone.utc) + + self._processes[job_id] = proc + except FileNotFoundError as exc: + job.status = JobStatus.FAILED + job.error_message = f"Command not found: {exc}" + except OSError as exc: + job.status = JobStatus.FAILED + job.error_message = str(exc) + + self._jobs[job_id] = job + return job + + def get_job(self, job_id: str) -> Job | None: + """Return a job by ID, refreshing its status if still running.""" + job = self._jobs.get(job_id) + if job is not None: + self._refresh_status(job) + return job + + def list_jobs( + self, status: JobStatus | None = None + ) -> list[Job]: + """List all tracked jobs, optionally filtered by status.""" + for job in self._jobs.values(): + self._refresh_status(job) + + jobs = list(self._jobs.values()) + if status is not None: + jobs = [j for j in jobs if j.status == status] + + return sorted(jobs, key=lambda j: j.start_time or datetime.min, reverse=True) + + def cancel_job(self, job_id: str) -> Job | None: + """Send SIGTERM to a running job's process.""" + job = self._jobs.get(job_id) + if job is None: + return None + + proc = self._processes.get(job_id) + if proc is not None and proc.poll() is None: + proc.send_signal(signal.SIGTERM) + job.status = JobStatus.CANCELLED + job.end_time = datetime.now(tz=timezone.utc) + + return job + + def read_log(self, job_id: str, tail: int = 100) -> str: + """Read the last N lines of a job's log file.""" + job = self._jobs.get(job_id) + if job is None or job.log_path is None: + return "" + + log_path = Path(job.log_path) + if not log_path.exists(): + return "" + + lines = log_path.read_text(errors="replace").splitlines() + return "\n".join(lines[-tail:]) + + async def stream_log( + self, job_id: str, poll_interval: float = 0.5 + ) -> AsyncIterator[str]: + """Yield new log lines as they are written (for SSE streaming). + + This is a simple tail-follow implementation suitable for the + /jobs/{id}/log streaming endpoint. + """ + import asyncio + + job = self._jobs.get(job_id) + if job is None or job.log_path is None: + return + + log_path = Path(job.log_path) + last_pos = 0 + + while True: + if log_path.exists(): + with open(log_path, "r", errors="replace") as fh: + fh.seek(last_pos) + new_data = fh.read() + last_pos = fh.tell() + + if new_data: + yield new_data + + # Stop streaming if the job is no longer running + self._refresh_status(job) + if job.status not in (JobStatus.PENDING, JobStatus.RUNNING): + # One final read to catch any remaining output + if log_path.exists(): + with open(log_path, "r", errors="replace") as fh: + fh.seek(last_pos) + final = fh.read() + if final: + yield final + break + + await asyncio.sleep(poll_interval) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + def _refresh_status(self, job: Job) -> None: + """Update job status from the subprocess return code.""" + if job.status not in (JobStatus.PENDING, JobStatus.RUNNING): + return + + proc = self._processes.get(job.id) + if proc is None: + return + + rc = proc.poll() + if rc is None: + return # Still running + + job.exit_code = rc + job.end_time = datetime.now(tz=timezone.utc) + + if rc == 0: + job.status = JobStatus.COMPLETED + else: + job.status = JobStatus.FAILED + job.error_message = f"Process exited with code {rc}" diff --git a/backend/services/parameter_service.py b/backend/services/parameter_service.py new file mode 100644 index 00000000..d559367f --- /dev/null +++ b/backend/services/parameter_service.py @@ -0,0 +1,654 @@ +"""Service for loading, saving, and validating emClarity parameter files. + +Parameter files are MATLAB-style .m files with key-value pairs that +control every aspect of the cryo-EM processing pipeline. +""" + +from __future__ import annotations + +import json +import logging +import re +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +log = logging.getLogger(__name__) + +from backend.models.parameter import ( + ParameterCategory, + ParameterDefinition, + ParameterFile, + ParameterType, + ParameterValidationResult, + ParameterValue, +) + +# Path to the parameter schema JSON generated from BH_parseParameterFile.m +_SCHEMA_PATH = Path(__file__).parent.parent.parent / "autonomous-build" / "templates" / "phase0-artifacts" / "parameter_schema.json" + + +class ParameterService: + """Handles parameter schema, file I/O, and validation.""" + + def __init__(self) -> None: + self._schema: list[ParameterDefinition] | None = None + + def get_schema(self) -> list[ParameterDefinition]: + """Return the parameter schema, loading from disk on first call. + + If the schema JSON file is not found, returns a minimal built-in + set of core parameters so the backend remains functional. + """ + if self._schema is not None: + return self._schema + + if _SCHEMA_PATH.exists(): + self._schema = self._load_schema_from_json(_SCHEMA_PATH) + else: + self._schema = self._builtin_schema() + + return self._schema + + def load_parameter_file(self, path: str) -> ParameterFile: + """Parse a MATLAB-style parameter file into structured data. + + Lines are expected in the format: + paramName = value + Comment lines starting with '%' are ignored. + """ + file_path = Path(path) + if not file_path.exists(): + raise FileNotFoundError(f"Parameter file not found: {path}") + + parameters: list[ParameterValue] = [] + content = file_path.read_text() + + for line in content.splitlines(): + line = line.strip() + # Skip empty lines and MATLAB comments + if not line or line.startswith("%"): + continue + + match = re.match(r"^(\w+)\s*=\s*(.+?)(?:;?\s*(?:%.*)?)?$", line) + if match: + name = match.group(1) + raw_value = match.group(2).strip().rstrip(";") + value = self._parse_value(raw_value) + parameters.append(ParameterValue(name=name, value=value)) + + return ParameterFile(parameters=parameters, path=path) + + def save_parameter_file(self, param_file: ParameterFile) -> None: + """Write parameter values to a MATLAB-style .m file atomically. + + Uses :func:`atomic_write_text` (temp file + ``os.replace()``) to + prevent partial writes from corrupting the parameter file on disk. + """ + from backend.utils.safe_json import atomic_write_text + + file_path = Path(param_file.path) + + lines: list[str] = [] + lines.append("% emClarity parameter file") + lines.append("% Generated by emClarity backend") + lines.append("") + + for pv in param_file.parameters: + value_str = self._format_value(pv.value) + lines.append(f"{pv.name} = {value_str};") + + content = "\n".join(lines) + "\n" + + atomic_write_text(file_path, content) + + def validate_parameters( + self, parameters: list[ParameterValue] + ) -> ParameterValidationResult: + """Validate parameter values against the schema.""" + schema = self.get_schema() + schema_map = {d.name: d for d in schema} + + errors: list[str] = [] + warnings: list[str] = [] + + # Check required parameters + provided_names = {p.name for p in parameters} + for defn in schema: + if defn.required and defn.name not in provided_names: + errors.append(f"Required parameter '{defn.name}' is missing") + + # Validate each provided value + for pv in parameters: + defn = schema_map.get(pv.name) + if defn is None: + warnings.append(f"Unknown parameter '{pv.name}' - not in schema") + continue + + # Range check for numeric types + if defn.range is not None and isinstance(pv.value, (int, float)): + lo, hi = defn.range + if not (lo <= pv.value <= hi): + errors.append( + f"Parameter '{pv.name}' value {pv.value} " + f"outside allowed range [{lo}, {hi}]" + ) + + # Enum / allowed-values check + if defn.allowed_values is not None: + # Compare both the raw value and its string representation + # to handle mixed-type allowed_values lists from the schema + if pv.value not in defn.allowed_values and str(pv.value) not in [ + str(v) for v in defn.allowed_values + ]: + errors.append( + f"Parameter '{pv.name}' value '{pv.value}' " + f"not in allowed values: {defn.allowed_values}" + ) + + return ParameterValidationResult( + valid=len(errors) == 0, + errors=errors, + warnings=warnings, + ) + + def load_parameter_file_v1(self, path: str) -> ParameterFile: + """Parse a MATLAB-style parameter file with deprecated name migration. + + This is the preferred v1 method. After parsing the raw key-value + pairs, any deprecated parameter names (e.g. ``flgCCCcutoff``) are + transparently translated to their current canonical names before the + result is returned. + + Args: + path: Filesystem path to the ``.m`` parameter file. + + Returns: + A :class:`ParameterFile` whose parameter names have been migrated + to their canonical forms. + + Raises: + FileNotFoundError: When the file does not exist. + """ + param_file = self.load_parameter_file(path) + migrated = self._migrate_deprecated_names(param_file.parameters) + return ParameterFile(parameters=migrated, path=path) + + def _migrate_deprecated_names( + self, parameters: list[ParameterValue] + ) -> list[ParameterValue]: + """Translate any deprecated parameter names to their canonical form. + + Reads the ``deprecated_name`` field from the golden schema to build a + lookup table, then replaces deprecated names in the supplied list. + Parameters whose names are already canonical are returned unchanged. + """ + deprecated_lookup: dict[str, str] = {} + if _SCHEMA_PATH.exists(): + raw = json.loads(_SCHEMA_PATH.read_text()) + entries = raw if isinstance(raw, list) else raw.get("parameters", []) + for entry in entries: + dep = entry.get("deprecated_name") + if dep: + deprecated_lookup[dep] = entry["name"] + + migrated: list[ParameterValue] = [] + for pv in parameters: + canonical = deprecated_lookup.get(pv.name, pv.name) + migrated.append(ParameterValue(name=canonical, value=pv.value)) + return migrated + + def validate_parameters_dict( + self, parameters: dict[str, Any] + ) -> ParameterValidationResult: + """Validate a flat dict of ``{name: value}`` pairs against the schema. + + This is the preferred method for the v1 API endpoint. It handles: + - Deprecated parameter name translation (e.g. ``flgCCCcutoff`` -> + ``ccc_cutoff``) so that legacy parameter files are accepted. + - Type checking: numeric parameters reject non-numeric strings. + - Range validation and required-parameter checks. + + Args: + parameters: Dict mapping parameter names (or deprecated names) + to their values. + + Returns: + A ParameterValidationResult with ``valid=True`` when all checks + pass, otherwise with ``errors`` populated. + """ + schema = self.get_schema() + schema_map = {d.name: d for d in schema} + + # Build deprecated-name -> current-name lookup from the schema JSON. + deprecated_lookup: dict[str, str] = {} + if _SCHEMA_PATH.exists(): + raw = json.loads(_SCHEMA_PATH.read_text()) + entries = raw if isinstance(raw, list) else raw.get("parameters", []) + for entry in entries: + dep = entry.get("deprecated_name") + if dep: + deprecated_lookup[dep] = entry["name"] + + errors: list[str] = [] + warnings: list[str] = [] + + # Translate deprecated names and build a normalised working dict. + normalised: dict[str, Any] = {} + for name, value in parameters.items(): + canonical = deprecated_lookup.get(name, name) + normalised[canonical] = value + + # Check required parameters. + for defn in schema: + if defn.required and defn.name not in normalised: + errors.append(f"Required parameter '{defn.name}' is missing") + + # Numeric types that require a numeric value. + numeric_types = {ParameterType.NUMERIC, ParameterType.FLOAT, ParameterType.INTEGER} + + # Validate each provided value. + for name, value in normalised.items(): + defn = schema_map.get(name) + if defn is None: + warnings.append(f"Unknown parameter '{name}' - not in schema") + continue + + # Type check: numeric parameters must not be non-numeric strings. + if defn.type in numeric_types and isinstance(value, str): + try: + float(value) + except (ValueError, TypeError): + errors.append( + f"Parameter '{name}' expects a numeric value, " + f"got string '{value}'" + ) + continue # skip further checks for this parameter + + # Range check for numeric types. + if defn.range is not None and isinstance(value, (int, float)): + lo, hi = defn.range + if not (lo <= value <= hi): + errors.append( + f"Parameter '{name}' value {value} " + f"outside allowed range [{lo}, {hi}]" + ) + + # Enum / allowed-values check. + if defn.allowed_values is not None: + if value not in defn.allowed_values and str(value) not in [ + str(v) for v in defn.allowed_values + ]: + errors.append( + f"Parameter '{name}' value '{value}' " + f"not in allowed values: {defn.allowed_values}" + ) + + return ParameterValidationResult( + valid=len(errors) == 0, + errors=errors, + warnings=warnings, + ) + + # ------------------------------------------------------------------ + # Snapshot methods + # ------------------------------------------------------------------ + + def save_snapshot( + self, project_dir: Path, parameters: dict[str, Any] + ) -> tuple[str, str, str]: + """Save a parameter snapshot to the project's parameters/ directory. + + Generates a UUID-based filename, writes the parameters as JSON using + :func:`atomic_write`, then enforces the retention cap. + + Args: + project_dir: Root directory of the project. + parameters: Parameter key-value pairs to snapshot. + + Returns: + A tuple of ``(snapshot_id, filename, created_at_iso)``. + """ + from backend.utils.safe_json import atomic_write + + params_dir = project_dir / "parameters" + params_dir.mkdir(parents=True, exist_ok=True) + + snapshot_id = str(uuid.uuid4()) + created_at = datetime.now(timezone.utc).isoformat() + # Filesystem-safe timestamp: replace colons with dashes + safe_timestamp = created_at.replace(":", "-") + filename = f"snapshot_{snapshot_id}_{safe_timestamp}.json" + filepath = params_dir / filename + + snapshot_data = { + "snapshot_id": snapshot_id, + "created_at": created_at, + "parameters": parameters, + } + + atomic_write(filepath, snapshot_data) + + # Enforce retention cap — errors are suppressed so they don't + # propagate as HTTP 500 after a successful save. + try: + self.cleanup_old_snapshots(project_dir) + except Exception: + pass + + return snapshot_id, filename, created_at + + def export_snapshot_to_m(self, snapshot_path: Path) -> Path: + """Read a snapshot JSON and write a .m parameter file alongside it. + + Uses :func:`_format_value` to convert each parameter to MATLAB + syntax, then writes atomically via :func:`atomic_write_text`. + + Args: + snapshot_path: Path to the snapshot JSON file. + + Returns: + The path to the generated ``.m`` file. + + Raises: + FileNotFoundError: When the snapshot JSON does not exist. + """ + from backend.utils.safe_json import atomic_write_text + + if not snapshot_path.exists(): + raise FileNotFoundError(f"Snapshot not found: {snapshot_path}") + + data = json.loads(snapshot_path.read_text(encoding="utf-8")) + parameters: dict[str, Any] = data.get("parameters", {}) + + lines: list[str] = [ + "% emClarity parameter file", + "% Generated by emClarity backend", + "", + ] + + for name, value in parameters.items(): + value_str = self._format_value(value) + lines.append(f"{name} = {value_str};") + + content = "\n".join(lines) + "\n" + + m_path = snapshot_path.with_suffix(".m") + atomic_write_text(m_path, content) + + return m_path + + @staticmethod + def cleanup_old_snapshots(project_dir: Path, keep: int = 50) -> None: + """Delete oldest snapshot files if more than *keep* exist. + + Only considers files matching the ``snapshot_*.json`` pattern in + the project's ``parameters/`` directory. Files are sorted by + modification time; the oldest are removed first. + + Args: + project_dir: Root directory of the project. + keep: Maximum number of snapshots to retain (default 50). + """ + params_dir = project_dir / "parameters" + if not params_dir.is_dir(): + return + + snapshots = sorted( + (f for f in params_dir.iterdir() if f.name.startswith("snapshot_") and f.suffix == ".json"), + key=lambda f: f.stat().st_mtime, + ) + + if len(snapshots) <= keep: + return + + to_delete = snapshots[: len(snapshots) - keep] + for snap_file in to_delete: + try: + snap_file.unlink() + # Also remove companion .m file if it exists + m_file = snap_file.with_suffix(".m") + if m_file.exists(): + m_file.unlink() + except OSError: + pass # best-effort cleanup + + def list_snapshots(self, project_dir: Path) -> list[dict[str, str]]: + """List all parameter snapshots sorted by creation date descending. + + Returns list of dicts with keys: snapshot_id, filename, created_at. + Reads the JSON content to get snapshot_id and created_at. + Sorted by created_at descending (newest first). + """ + params_dir = project_dir / "parameters" + if not params_dir.is_dir(): + return [] + + results: list[dict[str, str]] = [] + for f in params_dir.iterdir(): + if not (f.name.startswith("snapshot_") and f.suffix == ".json"): + continue + try: + data = json.loads(f.read_text(encoding="utf-8")) + results.append({ + "snapshot_id": data["snapshot_id"], + "filename": f.name, + "created_at": data["created_at"], + }) + except (json.JSONDecodeError, KeyError, OSError) as exc: + log.warning( + "Skipping malformed snapshot file %s: %s", f.name, exc + ) + continue + + results.sort(key=lambda r: r["created_at"], reverse=True) + return results + + def load_snapshot(self, project_dir: Path, snapshot_id: str) -> dict[str, Any]: + """Load a specific snapshot by ID. + + Returns dict with keys: snapshot_id, parameters (dict), created_at. + Raises FileNotFoundError if snapshot not found. + """ + params_dir = project_dir / "parameters" + if not params_dir.is_dir(): + raise FileNotFoundError( + f"Parameters directory not found: {params_dir}" + ) + + matching = [ + f for f in params_dir.iterdir() + if f.name.startswith(f"snapshot_{snapshot_id}") and f.suffix == ".json" + ] + + if not matching: + raise FileNotFoundError( + f"Snapshot {snapshot_id} not found" + ) + + if len(matching) > 1: + raise ValueError( + f"Snapshot ID prefix '{snapshot_id}' is ambiguous: " + f"matches {len(matching)} files" + ) + + snapshot_path = matching[0] + try: + data = json.loads(snapshot_path.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + raise ValueError( + f"Snapshot file is corrupted or unreadable: {snapshot_path.name}" + ) from exc + + try: + return { + "snapshot_id": data["snapshot_id"], + "parameters": data.get("parameters", {}), + "created_at": data["created_at"], + } + except KeyError as exc: + raise ValueError( + f"Snapshot file is missing required field {exc}: {snapshot_path.name}" + ) from exc + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _parse_value(raw: str) -> Any: + """Convert a raw MATLAB value string to a Python type.""" + stripped = raw.strip().strip("'\"") + + # Boolean + if stripped.lower() in ("true", "1") and stripped.lower() in ("true",): + return True + if stripped.lower() in ("false", "0") and stripped.lower() in ("false",): + return False + + # Vector: [1, 2, 3] + if stripped.startswith("[") and stripped.endswith("]"): + inner = stripped[1:-1] + parts = re.split(r"[,\s]+", inner.strip()) + try: + return [float(p) if "." in p else int(p) for p in parts if p] + except ValueError: + return stripped + + # Numeric + try: + if "." in stripped: + return float(stripped) + return int(stripped) + except ValueError: + pass + + return stripped + + @staticmethod + def _format_value(value: Any) -> str: + """Format a Python value for writing to a MATLAB parameter file.""" + if value is None: + return "''" + if isinstance(value, bool): + return "1" if value else "0" + if isinstance(value, float): + import math + if math.isnan(value): + return "NaN" + if math.isinf(value): + return "Inf" if value > 0 else "-Inf" + if isinstance(value, list): + parts = ", ".join(str(v) for v in value) + return f"[{parts}]" + if isinstance(value, str): + # Try to parse as a number before wrapping in quotes + try: + int_val = int(value) + return str(int_val) + except (ValueError, TypeError): + pass + try: + float_val = float(value) + return str(float_val) + except (ValueError, TypeError): + pass + return f"'{value}'" + return str(value) + + @staticmethod + def _load_schema_from_json(path: Path) -> list[ParameterDefinition]: + """Load parameter definitions from the generated JSON schema.""" + data = json.loads(path.read_text()) + + definitions: list[ParameterDefinition] = [] + params = data if isinstance(data, list) else data.get("parameters", []) + + for entry in params: + # Map JSON fields to our model, with sensible defaults. + # The golden schema uses: numeric, numeric_array, string, boolean. + param_type = entry.get("type", "string") + type_map = { + "string": ParameterType.STRING, + "integer": ParameterType.INTEGER, + "float": ParameterType.FLOAT, + "number": ParameterType.FLOAT, + "boolean": ParameterType.BOOLEAN, + "vector": ParameterType.VECTOR, + "enum": ParameterType.ENUM, + "numeric": ParameterType.NUMERIC, + "numeric_array": ParameterType.NUMERIC_ARRAY, + } + + category_str = entry.get("category", "general") + try: + category = ParameterCategory(category_str) + except ValueError: + category = ParameterCategory.GENERAL + + definitions.append( + ParameterDefinition( + name=entry["name"], + type=type_map.get(param_type, ParameterType.STRING), + required=entry.get("required", False), + default=entry.get("default"), + range=entry.get("range"), + allowed_values=entry.get("allowed_values"), + description=entry.get("description", ""), + category=category, + units=entry.get("units"), + ) + ) + + return definitions + + @staticmethod + def _builtin_schema() -> list[ParameterDefinition]: + """Return a minimal built-in schema for core parameters. + + Used as a fallback when the full JSON schema file is not available. + """ + return [ + ParameterDefinition( + name="PIXEL_SIZE", + type=ParameterType.FLOAT, + required=True, + description="Pixel size of the raw tilt-series images", + category=ParameterCategory.GENERAL, + units="angstroms", + range=[0.1, 50.0], + ), + ParameterDefinition( + name="SuperResolution", + type=ParameterType.INTEGER, + required=False, + default=0, + description="Super-resolution factor (0 = off, 1 = on)", + category=ParameterCategory.GENERAL, + range=[0, 1], + ), + ParameterDefinition( + name="Ali_samplingRate", + type=ParameterType.VECTOR, + required=False, + default=[4, 3, 2], + description="Sampling rates for alignment cycles", + category=ParameterCategory.ALIGNMENT, + ), + ParameterDefinition( + name="Cls_className", + type=ParameterType.STRING, + required=False, + default="", + description="Class name for classification", + category=ParameterCategory.CLASSIFICATION, + ), + ParameterDefinition( + name="GPU", + type=ParameterType.VECTOR, + required=True, + description="GPU device IDs to use", + category=ParameterCategory.SYSTEM, + ), + ] diff --git a/backend/services/project_service.py b/backend/services/project_service.py new file mode 100644 index 00000000..b84de98a --- /dev/null +++ b/backend/services/project_service.py @@ -0,0 +1,169 @@ +"""Service for emClarity project management. + +A project is a directory on disk that follows the emClarity convention: + rawData/ - original tilt-series stacks + fixedStacks/ - aligned stacks and metadata + aliStacks/ - CTF-corrected aligned stacks + cache/ - temporary reconstructions + convmap/ - template search results + FSC/ - resolution curves + logFile/ - processing logs +""" + +from __future__ import annotations + +from pathlib import Path + +from backend.models.project import Project, ProjectState, TiltSeries + + +# Directories that emClarity expects inside a project +_PROJECT_SUBDIRS = [ + "rawData", + "fixedStacks", + "aliStacks", + "cache", + "convmap", + "FSC", + "logFile", +] + + +class ProjectService: + """Create, load, and inspect emClarity projects.""" + + def create_project(self, name: str, path: str) -> Project: + """Create a new project directory structure. + + Creates the project root and all expected subdirectories. + Returns the initial project model. + """ + project_dir = Path(path) + project_dir.mkdir(parents=True, exist_ok=True) + + for subdir in _PROJECT_SUBDIRS: + (project_dir / subdir).mkdir(exist_ok=True) + + return Project( + name=name, + path=str(project_dir.resolve()), + state=ProjectState.UNINITIALIZED, + current_cycle=0, + tilt_series=[], + ) + + def load_project(self, path: str) -> Project: + """Load project state by inspecting the directory structure. + + Determines the current pipeline state by checking which + directories contain processed data. + """ + project_dir = Path(path) + if not project_dir.exists(): + raise FileNotFoundError(f"Project directory not found: {path}") + + name = project_dir.name + state = self._detect_state(project_dir) + cycle = self._detect_cycle(project_dir) + tilt_series = self._discover_tilt_series(project_dir) + + # Try to find the parameter file + param_file = None + for candidate in project_dir.glob("*.m"): + param_file = str(candidate) + break + + return Project( + name=name, + path=str(project_dir.resolve()), + state=state, + current_cycle=cycle, + tilt_series=tilt_series, + parameter_file=param_file, + ) + + def list_tilt_series(self, path: str) -> list[TiltSeries]: + """Discover tilt series in the project's rawData/ directory.""" + project_dir = Path(path) + return self._discover_tilt_series(project_dir) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _detect_state(project_dir: Path) -> ProjectState: + """Infer the pipeline state from what data exists on disk.""" + # Check in reverse order of the pipeline (most advanced first) + fsc_dir = project_dir / "FSC" + if fsc_dir.exists() and any(fsc_dir.iterdir()): + return ProjectState.CYCLE_N + + convmap_dir = project_dir / "convmap" + if convmap_dir.exists() and any(convmap_dir.iterdir()): + return ProjectState.PARTICLES_PICKED + + ali_dir = project_dir / "aliStacks" + if ali_dir.exists() and any(ali_dir.iterdir()): + return ProjectState.RECONSTRUCTED + + fixed_dir = project_dir / "fixedStacks" + if fixed_dir.exists() and any(fixed_dir.glob("*.fixed")): + return ProjectState.CTF_ESTIMATED + + if fixed_dir.exists() and any(fixed_dir.iterdir()): + return ProjectState.TILT_ALIGNED + + return ProjectState.UNINITIALIZED + + @staticmethod + def _detect_cycle(project_dir: Path) -> int: + """Detect the latest refinement cycle number.""" + cycle_dirs = sorted(project_dir.glob("cycle*")) + if not cycle_dirs: + return 0 + + # Extract the highest cycle number + max_cycle = 0 + for d in cycle_dirs: + try: + n = int(d.name.replace("cycle", "")) + max_cycle = max(max_cycle, n) + except ValueError: + continue + return max_cycle + + @staticmethod + def _discover_tilt_series(project_dir: Path) -> list[TiltSeries]: + """Find tilt-series stacks in rawData/ and fixedStacks/.""" + tilt_series: list[TiltSeries] = [] + seen_names: set[str] = set() + + raw_dir = project_dir / "rawData" + if raw_dir.exists(): + for stack_file in sorted(raw_dir.glob("*.st")): + ts_name = stack_file.stem + if ts_name in seen_names: + continue + seen_names.add(ts_name) + + rawtlt = stack_file.with_suffix(".rawtlt") + fixed_dir = project_dir / "fixedStacks" + + tilt_series.append( + TiltSeries( + name=ts_name, + stack_path=str(stack_file), + rawtlt_path=str(rawtlt) if rawtlt.exists() else None, + aligned=bool( + fixed_dir.exists() + and any(fixed_dir.glob(f"{ts_name}*")) + ), + ctf_estimated=bool( + fixed_dir.exists() + and any(fixed_dir.glob(f"{ts_name}*.fixed")) + ), + ) + ) + + return tilt_series diff --git a/backend/services/system_service.py b/backend/services/system_service.py new file mode 100644 index 00000000..a268e525 --- /dev/null +++ b/backend/services/system_service.py @@ -0,0 +1,207 @@ +"""Service for detecting system hardware: GPUs, CPUs, and memory. + +Uses nvidia-smi for GPU detection and /proc or psutil-style queries +for CPU and memory information. +""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from pydantic import BaseModel, Field + + +class GpuInfo(BaseModel): + """Information about a single GPU device.""" + + index: int = Field(..., description="Device index") + name: str = Field(..., description="GPU model name") + memory_total_mb: int = Field(..., description="Total GPU memory in MB") + memory_used_mb: int = Field(default=0, description="Currently used memory in MB") + memory_free_mb: int = Field(default=0, description="Available memory in MB") + driver_version: str = Field(default="", description="NVIDIA driver version") + cuda_version: str = Field(default="", description="CUDA version") + + +class SystemInfo(BaseModel): + """Aggregated system information.""" + + cpu_count: int = Field(..., description="Number of logical CPU cores") + cpu_count_physical: int = Field(..., description="Number of physical CPU cores") + memory_total_gb: float = Field(..., description="Total system RAM in GB") + memory_available_gb: float = Field(..., description="Available RAM in GB") + hostname: str = Field(default="", description="Machine hostname") + gpus: list[GpuInfo] = Field(default_factory=list, description="Detected GPUs") + + +class SystemService: + """Detect and report system hardware capabilities.""" + + def detect_gpus(self) -> list[GpuInfo]: + """Detect NVIDIA GPUs using nvidia-smi. + + Returns an empty list if nvidia-smi is not available or no + GPUs are found. + """ + try: + result = subprocess.run( + [ + "nvidia-smi", + "--query-gpu=index,name,memory.total,memory.used,memory.free,driver_version", + "--format=csv,noheader,nounits", + ], + capture_output=True, + text=True, + timeout=10, + ) + except (FileNotFoundError, subprocess.TimeoutExpired): + return [] + + if result.returncode != 0: + return [] + + gpus: list[GpuInfo] = [] + for line in result.stdout.strip().splitlines(): + parts = [p.strip() for p in line.split(",")] + if len(parts) < 6: + continue + try: + gpus.append( + GpuInfo( + index=int(parts[0]), + name=parts[1], + memory_total_mb=int(parts[2]), + memory_used_mb=int(parts[3]), + memory_free_mb=int(parts[4]), + driver_version=parts[5], + ) + ) + except (ValueError, IndexError): + continue + + # Try to get CUDA version + cuda_version = self._detect_cuda_version() + for gpu in gpus: + gpu.cuda_version = cuda_version + + return gpus + + def get_system_info(self) -> SystemInfo: + """Gather CPU, memory, and GPU information.""" + cpu_count = os.cpu_count() or 1 + cpu_physical = self._get_physical_cpu_count() + mem_total, mem_available = self._get_memory_info() + + hostname = "" + try: + import socket + hostname = socket.gethostname() + except OSError: + pass + + return SystemInfo( + cpu_count=cpu_count, + cpu_count_physical=cpu_physical, + memory_total_gb=round(mem_total / (1024 ** 3), 2), + memory_available_gb=round(mem_available / (1024 ** 3), 2), + hostname=hostname, + gpus=self.detect_gpus(), + ) + + # ------------------------------------------------------------------ + # Internal helpers + # ------------------------------------------------------------------ + + @staticmethod + def _get_physical_cpu_count() -> int: + """Get physical CPU core count from /proc/cpuinfo.""" + try: + cpuinfo = Path("/proc/cpuinfo").read_text() + physical_ids = set() + core_ids = set() + current_physical = None + + for line in cpuinfo.splitlines(): + if line.startswith("physical id"): + current_physical = line.split(":")[1].strip() + elif line.startswith("core id") and current_physical is not None: + core_id = line.split(":")[1].strip() + physical_ids.add(current_physical) + core_ids.add((current_physical, core_id)) + + if core_ids: + return len(core_ids) + except (FileNotFoundError, PermissionError): + pass + + return os.cpu_count() or 1 + + @staticmethod + def _get_memory_info() -> tuple[int, int]: + """Read total and available memory from /proc/meminfo. + + Returns (total_bytes, available_bytes). + """ + total = 0 + available = 0 + + try: + meminfo = Path("/proc/meminfo").read_text() + for line in meminfo.splitlines(): + if line.startswith("MemTotal:"): + total = int(line.split()[1]) * 1024 # kB to bytes + elif line.startswith("MemAvailable:"): + available = int(line.split()[1]) * 1024 + except (FileNotFoundError, PermissionError, ValueError): + pass + + return total, available + + @staticmethod + def _detect_cuda_version() -> str: + """Try to detect the installed CUDA version.""" + try: + result = subprocess.run( + ["nvidia-smi", "--query-gpu=driver_version", "--format=csv,noheader"], + capture_output=True, + text=True, + timeout=5, + ) + # nvidia-smi also shows CUDA version in the header output + header_result = subprocess.run( + ["nvidia-smi"], + capture_output=True, + text=True, + timeout=5, + ) + if header_result.returncode == 0: + for line in header_result.stdout.splitlines(): + if "CUDA Version:" in line: + parts = line.split("CUDA Version:") + if len(parts) > 1: + return parts[1].strip().rstrip("|").strip() + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + + # Fallback: check nvcc + try: + result = subprocess.run( + ["nvcc", "--version"], + capture_output=True, + text=True, + timeout=5, + ) + if result.returncode == 0: + for line in result.stdout.splitlines(): + if "release" in line.lower(): + # Typical format: "Cuda compilation tools, release 12.2, V12.2.140" + parts = line.split("release") + if len(parts) > 1: + return parts[1].strip().split(",")[0].strip() + except (FileNotFoundError, subprocess.TimeoutExpired): + pass + + return "" diff --git a/backend/services/workflow_service.py b/backend/services/workflow_service.py new file mode 100644 index 00000000..7b0883de --- /dev/null +++ b/backend/services/workflow_service.py @@ -0,0 +1,162 @@ +"""Service for building emClarity commands and tracking pipeline state. + +Translates frontend command requests into the CLI invocations that +emClarity expects, and determines which commands are available given +the current project state. +""" + +from __future__ import annotations + +from backend.models.workflow import ( + COMMAND_PREREQUISITES, + CommandInfo, + CommandRequest, + PipelineCommand, + WorkflowState, +) + +# Human-readable labels and descriptions for each command +_COMMAND_INFO: dict[PipelineCommand, tuple[str, str]] = { + PipelineCommand.AUTO_ALIGN: ( + "Tilt-Series Alignment", + "Align raw tilt-series images using fiducial or patch tracking", + ), + PipelineCommand.CTF_ESTIMATE: ( + "CTF Estimation", + "Estimate defocus and astigmatism for each tilt image", + ), + PipelineCommand.CTF_3D: ( + "3D CTF Correction", + "Apply 3D CTF correction and reconstruct tomograms", + ), + PipelineCommand.TEMPLATE_SEARCH: ( + "Template Search", + "Search for particles using a 3D template", + ), + PipelineCommand.INIT: ( + "Initialize Project", + "Initialize the sub-tomogram averaging project", + ), + PipelineCommand.AVG: ( + "Average", + "Compute the sub-tomogram average from aligned particles", + ), + PipelineCommand.ALIGN_RAW: ( + "Align Particles", + "Refine particle orientations against the current average", + ), + PipelineCommand.TOMO_CPR: ( + "Tilt-Series Refinement", + "Refine tilt-series geometry using current particle positions", + ), + PipelineCommand.PCA: ( + "PCA Analysis", + "Principal component analysis for heterogeneity detection", + ), + PipelineCommand.CLUSTER: ( + "Classification", + "Classify particles based on PCA eigenvectors", + ), + PipelineCommand.FSC: ( + "FSC Calculation", + "Compute Fourier Shell Correlation for resolution estimation", + ), + PipelineCommand.RECONSTRUCT: ( + "Final Reconstruction", + "Generate the final high-resolution 3D reconstruction", + ), +} + + +class WorkflowService: + """Build CLI commands and manage pipeline state.""" + + def list_commands(self) -> list[CommandInfo]: + """Return metadata for all available pipeline commands.""" + result: list[CommandInfo] = [] + for cmd in PipelineCommand: + label, description = _COMMAND_INFO.get(cmd, (cmd.value, "")) + result.append( + CommandInfo( + command=cmd, + label=label, + description=description, + prerequisites=COMMAND_PREREQUISITES.get(cmd, []), + ) + ) + return result + + def get_workflow_state( + self, completed_commands: list[PipelineCommand] + ) -> WorkflowState: + """Determine which commands are available given completed ones. + + A command is available if all its prerequisites are in the + completed set. + """ + completed_set = set(completed_commands) + available: list[PipelineCommand] = [] + + for cmd, prereqs in COMMAND_PREREQUISITES.items(): + if cmd in completed_set: + continue # Already done + if all(p in completed_set for p in prereqs): + available.append(cmd) + + return WorkflowState( + completed_commands=completed_commands, + available_commands=available, + current_cycle=0, + ) + + def build_cli_command(self, request: CommandRequest, param_file: str) -> list[str]: + """Translate a CommandRequest into an emClarity CLI invocation. + + Returns the command as a list of arguments suitable for + subprocess.Popen. + """ + cmd = request.command + parts: list[str] = ["emClarity"] + + if cmd == PipelineCommand.AUTO_ALIGN: + parts.extend([ + "autoAlign", + param_file, + request.tilt_series_name or "", + f"{request.tilt_series_name or ''}.rawtlt", + "0", + ]) + elif cmd == PipelineCommand.CTF_ESTIMATE: + parts.extend([ + "ctf", + "estimate", + param_file, + request.tilt_series_name or "", + ]) + elif cmd == PipelineCommand.CTF_3D: + parts.extend(["ctf", "3d", param_file]) + elif cmd == PipelineCommand.TEMPLATE_SEARCH: + parts.extend(["templateSearch", param_file]) + elif cmd == PipelineCommand.INIT: + parts.extend(["init", param_file]) + elif cmd == PipelineCommand.AVG: + parts.extend([ + "avg", + param_file, + str(request.cycle), + "RawAlignment" if request.cycle == 0 else "NoAlignment", + ]) + elif cmd == PipelineCommand.ALIGN_RAW: + parts.extend(["alignRaw", param_file, str(request.cycle)]) + elif cmd == PipelineCommand.TOMO_CPR: + parts.extend(["tomoCPR", param_file, str(request.cycle)]) + elif cmd == PipelineCommand.PCA: + parts.extend(["pca", param_file, str(request.cycle)]) + elif cmd == PipelineCommand.CLUSTER: + parts.extend(["cluster", param_file, str(request.cycle)]) + elif cmd == PipelineCommand.FSC: + parts.extend(["fsc", param_file, str(request.cycle)]) + elif cmd == PipelineCommand.RECONSTRUCT: + parts.extend(["reconstruct", param_file, str(request.cycle)]) + + return parts diff --git a/backend/tests/__init__.py b/backend/tests/__init__.py new file mode 100644 index 00000000..4993e5ef --- /dev/null +++ b/backend/tests/__init__.py @@ -0,0 +1 @@ +# Backend test package diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 00000000..093e8e75 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,26 @@ +"""Shared pytest fixtures for backend tests.""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from backend.main import app + + +@pytest.fixture +def client() -> TestClient: + """Create a FastAPI test client.""" + return TestClient(app) + + +@pytest.fixture +def tmp_project(tmp_path): + """Create a temporary project directory structure.""" + project_dir = tmp_path / "test_project" + project_dir.mkdir() + + for subdir in ["rawData", "fixedStacks", "aliStacks", "cache", "convmap", "FSC", "logFile"]: + (project_dir / subdir).mkdir() + + return project_dir diff --git a/backend/tests/test_detect_best_resolution.py b/backend/tests/test_detect_best_resolution.py new file mode 100644 index 00000000..c263a68c --- /dev/null +++ b/backend/tests/test_detect_best_resolution.py @@ -0,0 +1,107 @@ +"""Unit tests for _detect_best_resolution logic branches. + +Covers the two guards added in TASK-018: + 1. Invalid-frequency branch: freq <= 0 or freq > 1.0 are skipped. + 2. Implausible-resolution branch: angstrom > 200 is discarded with a warning. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from backend.api.v1_projects import _detect_best_resolution + + +class TestDetectBestResolutionInvalidFrequency: + """Branch 1 – freq <= 0 or freq > 1.0 lines are silently skipped.""" + + def test_zero_frequency_is_ignored(self, tmp_path: Path) -> None: + """A line with freq == 0 must be discarded; no ZeroDivisionError.""" + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + fsc_file = fsc_dir / "test_fsc_GLD.txt" + # freq=0 would cause 1/freq → ZeroDivisionError and must be skipped + fsc_file.write_text("0.0 0.900\n") + assert _detect_best_resolution(tmp_path) is None + + def test_negative_frequency_is_ignored(self, tmp_path: Path) -> None: + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + (fsc_dir / "test_fsc_GLD.txt").write_text("-0.1 0.900\n") + assert _detect_best_resolution(tmp_path) is None + + def test_frequency_above_one_is_ignored(self, tmp_path: Path) -> None: + """freq > 1.0 corresponds to sub-1 Å resolution – reject as implausible units.""" + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + (fsc_dir / "test_fsc_GLD.txt").write_text("1.5 0.900\n") + assert _detect_best_resolution(tmp_path) is None + + def test_valid_frequency_accepted_when_mixed_with_invalid(self, tmp_path: Path) -> None: + """Only valid freq lines contribute; invalid freq lines are skipped.""" + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + fsc_file = fsc_dir / "test_fsc_GLD.txt" + # invalid freq=0, then valid freq=0.1 (→ 10.0 Å), fsc >= 0.143 + fsc_file.write_text("0.0 0.900\n1.5 0.900\n0.1 0.500\n") + result = _detect_best_resolution(tmp_path) + assert result == pytest.approx(10.0, rel=1e-3) + + +class TestDetectBestResolutionImplausibleAngstrom: + """Branch 2 – resolutions > 200 Å are discarded with a warning log.""" + + def test_resolution_above_200_discarded(self, tmp_path: Path) -> None: + """freq = 0.004 → 250 Å must be rejected; function returns None.""" + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + # 1 / 0.004 = 250 Å → above the 200 Å upper bound + (fsc_dir / "test_fsc_GLD.txt").write_text("0.004 0.900\n") + assert _detect_best_resolution(tmp_path) is None + + def test_resolution_exactly_200_accepted(self, tmp_path: Path) -> None: + """Boundary value: 1/0.005 == 200.0 Å is on the inclusive boundary and must be kept.""" + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + (fsc_dir / "test_fsc_GLD.txt").write_text("0.005 0.900\n") + result = _detect_best_resolution(tmp_path) + assert result == pytest.approx(200.0, rel=1e-3) + + def test_resolution_just_above_200_discarded(self, tmp_path: Path) -> None: + """A resolution of ~200.8 Å (freq ~0.00498) is above 200 and must be discarded.""" + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + # 1 / 0.00498 ≈ 200.8 Å + (fsc_dir / "test_fsc_GLD.txt").write_text("0.00498 0.900\n") + assert _detect_best_resolution(tmp_path) is None + + def test_implausible_file_does_not_pollute_best_from_good_file( + self, tmp_path: Path + ) -> None: + """When one FSC file has only implausible resolutions and another has valid ones, + the valid result is returned.""" + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + # Bad file: all resolutions > 200 Å + (fsc_dir / "bad_fsc_GLD.txt").write_text("0.004 0.900\n") + # Good file: 0.1 → 10.0 Å (valid) + (fsc_dir / "good_fsc_GLD.txt").write_text("0.1 0.500\n") + result = _detect_best_resolution(tmp_path) + assert result == pytest.approx(10.0, rel=1e-3) + + def test_implausible_warning_logged(self, tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """A discarded implausible resolution must emit a WARNING log.""" + import logging + + fsc_dir = tmp_path / "FSC" + fsc_dir.mkdir() + (fsc_dir / "test_fsc_GLD.txt").write_text("0.004 0.900\n") + + with caplog.at_level(logging.WARNING, logger="backend.api.v1_projects"): + _detect_best_resolution(tmp_path) + + assert any("200" in record.message for record in caplog.records), ( + "Expected a warning mentioning the 200 Å upper bound" + ) diff --git a/backend/tests/test_filesystem.py b/backend/tests/test_filesystem.py new file mode 100644 index 00000000..f1397107 --- /dev/null +++ b/backend/tests/test_filesystem.py @@ -0,0 +1,342 @@ +"""Tests for the filesystem browse API endpoint (GET /api/v1/filesystem/browse). + +Coverage map (acceptance-criteria checklist): + - default path (no param) + - empty string path (same as default) + - whitespace-only path (same as default) + - explicit valid path with all three response fields verified + - root path (parent: null) + - root-path entry path construction (no double slash) + - path with '..' (400) + - URL-encoded traversal (400) – FastAPI decodes before handler + - null byte (400) + - file path (400) with JSON body containing the offending path + - nonexistent path (404) with JSON body + - permission-denied path (403) with JSON body + - directory containing only files (200, entries: []) + - symlinks excluded (both dir-links and file-links) + - path normalisation (trailing slash stripped) + - symlink path resolved to real path in response + - relative path without leading slash (400, "absolute" in detail) + - path longer than PATH_MAX (400) + - race-condition removal during scandir (404, not 500) + - non-UTF-8 filename silently skipped (200, no 500) + - unauthenticated request (401) — SKIPPED, auth not yet implemented +""" + +from __future__ import annotations + +import os +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from fastapi.testclient import TestClient + +from backend.main import app + + +@pytest.fixture() +def client() -> TestClient: + """Return a TestClient bound to the FastAPI app.""" + return TestClient(app) + + +# --------------------------------------------------------------------------- +# Default-path behaviour (no param / empty / whitespace) +# --------------------------------------------------------------------------- + + +class TestDefaultPath: + def test_no_param_returns_home(self, client: TestClient) -> None: + response = client.get("/api/v1/filesystem/browse") + assert response.status_code == 200 + data = response.json() + # path must equal the real home directory + assert data["path"] == str(Path(os.path.realpath(str(Path.home())))) + assert "parent" in data + assert isinstance(data["entries"], list) + + def test_empty_string_param_returns_home(self, client: TestClient) -> None: + response = client.get("/api/v1/filesystem/browse?path=") + assert response.status_code == 200 + assert response.json()["path"] == str(Path(os.path.realpath(str(Path.home())))) + + def test_whitespace_only_param_returns_home(self, client: TestClient) -> None: + response = client.get("/api/v1/filesystem/browse?path= ") + assert response.status_code == 200 + assert response.json()["path"] == str(Path(os.path.realpath(str(Path.home())))) + + +# --------------------------------------------------------------------------- +# Happy paths – valid directories +# --------------------------------------------------------------------------- + + +class TestValidPaths: + def test_explicit_tmp_all_fields_present(self, client: TestClient) -> None: + response = client.get("/api/v1/filesystem/browse?path=/tmp") + assert response.status_code == 200 + data = response.json() + assert data["path"] == "/tmp" + assert data["parent"] == "/" + assert isinstance(data["entries"], list) + for entry in data["entries"]: + assert "name" in entry + assert entry["type"] == "directory" + assert "path" in entry + + def test_root_path_parent_is_null(self, client: TestClient) -> None: + response = client.get("/api/v1/filesystem/browse?path=/") + assert response.status_code == 200 + data = response.json() + assert data["path"] == "/" + assert data["parent"] is None # JSON null, not the string "null" + assert isinstance(data["entries"], list) + + def test_root_entry_paths_have_no_double_slash(self, client: TestClient) -> None: + response = client.get("/api/v1/filesystem/browse?path=/") + assert response.status_code == 200 + for entry in response.json()["entries"]: + assert not entry["path"].startswith("//"), ( + f"Entry path must not start with //: {entry['path']!r}" + ) + assert entry["path"] == f"/{entry['name']}" + + def test_subdir_entries_correct_paths( + self, client: TestClient, tmp_path: Path + ) -> None: + (tmp_path / "alpha").mkdir() + (tmp_path / "beta").mkdir() + (tmp_path / "file.txt").write_text("content") + + response = client.get(f"/api/v1/filesystem/browse?path={tmp_path}") + assert response.status_code == 200 + data = response.json() + names = {e["name"] for e in data["entries"]} + assert names == {"alpha", "beta"} + for entry in data["entries"]: + expected = f"{tmp_path}/{entry['name']}" + assert entry["path"] == expected + + def test_only_directories_in_entries( + self, client: TestClient, tmp_path: Path + ) -> None: + (tmp_path / "subdir").mkdir() + (tmp_path / "regular_file.txt").write_text("hello") + (tmp_path / "another_file").write_text("world") + + response = client.get(f"/api/v1/filesystem/browse?path={tmp_path}") + assert response.status_code == 200 + data = response.json() + assert len(data["entries"]) == 1 + assert data["entries"][0]["name"] == "subdir" + + def test_symlinks_excluded(self, client: TestClient, tmp_path: Path) -> None: + real_dir = tmp_path / "real_dir" + real_dir.mkdir() + link_dir = tmp_path / "link_to_dir" + link_dir.symlink_to(real_dir) + + real_file = tmp_path / "real_file.txt" + real_file.write_text("content") + link_file = tmp_path / "link_to_file.txt" + link_file.symlink_to(real_file) + + response = client.get(f"/api/v1/filesystem/browse?path={tmp_path}") + assert response.status_code == 200 + names = {e["name"] for e in response.json()["entries"]} + assert "link_to_dir" not in names + assert "link_to_file.txt" not in names + assert "real_dir" in names + + def test_directory_with_only_files_returns_empty_entries( + self, client: TestClient, tmp_path: Path + ) -> None: + (tmp_path / "file1.txt").write_text("a") + (tmp_path / "file2.txt").write_text("b") + + response = client.get(f"/api/v1/filesystem/browse?path={tmp_path}") + assert response.status_code == 200 + assert response.json()["entries"] == [] + + def test_trailing_slash_normalised(self, client: TestClient) -> None: + r_plain = client.get("/api/v1/filesystem/browse?path=/tmp") + r_slash = client.get("/api/v1/filesystem/browse?path=/tmp/") + assert r_plain.status_code == 200 + assert r_slash.status_code == 200 + assert r_plain.json()["path"] == r_slash.json()["path"] == "/tmp" + + def test_symlink_path_resolves_to_real( + self, client: TestClient, tmp_path: Path + ) -> None: + real_dir = tmp_path / "real" + real_dir.mkdir() + link_path = tmp_path / "link_to_real" + link_path.symlink_to(real_dir) + + response = client.get(f"/api/v1/filesystem/browse?path={link_path}") + assert response.status_code == 200 + # The response path must be the resolved real path, not the symlink path. + assert response.json()["path"] == str(real_dir.resolve()) + + +# --------------------------------------------------------------------------- +# Error conditions +# --------------------------------------------------------------------------- + + +class TestErrorConditions: + def test_path_traversal_double_dots_returns_400( + self, client: TestClient + ) -> None: + response = client.get("/api/v1/filesystem/browse?path=../../etc/passwd") + assert response.status_code == 400 + assert "detail" in response.json() + + def test_url_encoded_traversal_returns_400(self, client: TestClient) -> None: + # FastAPI decodes %2e%2e%2f → ../../ before the handler; our validator + # must catch the decoded form. + response = client.get("/api/v1/filesystem/browse?path=%2e%2e%2fetc%2fpasswd") + assert response.status_code == 400 + assert "detail" in response.json() + + def test_null_byte_returns_400(self, client: TestClient) -> None: + response = client.get("/api/v1/filesystem/browse?path=/tmp%00evil") + assert response.status_code == 400 + assert "detail" in response.json() + + def test_file_path_returns_400_with_json_body( + self, client: TestClient, tmp_path: Path + ) -> None: + file_path = tmp_path / "test_file.txt" + file_path.write_text("content") + + response = client.get(f"/api/v1/filesystem/browse?path={file_path}") + assert response.status_code == 400 + assert response.headers["content-type"].startswith("application/json") + data = response.json() + assert "detail" in data + assert str(file_path) in data["detail"] + + def test_nonexistent_path_returns_404_with_json_body( + self, client: TestClient + ) -> None: + response = client.get( + "/api/v1/filesystem/browse?path=/nonexistent/path/xyz" + ) + assert response.status_code == 404 + assert response.headers["content-type"].startswith("application/json") + data = response.json() + assert "detail" in data + assert "/nonexistent/path/xyz" in data["detail"] + + def test_permission_denied_returns_403_with_json_body( + self, client: TestClient, tmp_path: Path + ) -> None: + if os.geteuid() == 0: + pytest.skip("Running as root – chmod 000 does not restrict access") + + restricted = tmp_path / "restricted" + restricted.mkdir() + restricted.chmod(0o000) + try: + response = client.get( + f"/api/v1/filesystem/browse?path={restricted}" + ) + assert response.status_code == 403 + assert response.headers["content-type"].startswith("application/json") + data = response.json() + assert "detail" in data + assert str(restricted) in data["detail"] + finally: + restricted.chmod(0o755) # restore so tmp_path cleanup succeeds + + def test_relative_path_returns_400_with_absolute_message( + self, client: TestClient + ) -> None: + response = client.get("/api/v1/filesystem/browse?path=tmp") + assert response.status_code == 400 + data = response.json() + assert "detail" in data + assert "absolute" in data["detail"].lower() + + def test_path_longer_than_path_max_returns_400( + self, client: TestClient + ) -> None: + # Build a path that is definitely longer than 4096 characters. + long_path = "/" + "a" * 4097 + response = client.get(f"/api/v1/filesystem/browse?path={long_path}") + assert response.status_code == 400 + assert "detail" in response.json() + + def test_race_condition_dir_removed_during_scandir_returns_404( + self, client: TestClient, tmp_path: Path + ) -> None: + """Directory deleted between existence check and os.scandir → 404, not 500.""" + target = tmp_path / "vanishing_dir" + target.mkdir() + + def _raise_fnf(path: object) -> None: + raise FileNotFoundError(f"No such file or directory: {path!r}") + + with patch( + "backend.api.v1_filesystem.os.scandir", side_effect=_raise_fnf + ): + response = client.get( + f"/api/v1/filesystem/browse?path={target}" + ) + + assert response.status_code == 404 + assert "detail" in response.json() + + def test_non_utf8_filename_silently_skipped( + self, client: TestClient, tmp_path: Path + ) -> None: + """Non-UTF-8 directory entries are skipped; response is 200, never 500.""" + (tmp_path / "valid_dir").mkdir() + + # Build a mock DirEntry whose name contains a surrogate character + # (what Python uses for undecodable filesystem bytes via surrogateescape). + bad_entry = MagicMock() + bad_entry.is_dir.return_value = True + bad_entry.is_symlink.return_value = False + bad_entry.name = "bad\udcffname" # surrogate → encode('utf-8') raises + + good_entry = MagicMock() + good_entry.is_dir.return_value = True + good_entry.is_symlink.return_value = False + good_entry.name = "valid_dir" + + mock_cm = MagicMock() + mock_cm.__enter__ = MagicMock( + return_value=iter([bad_entry, good_entry]) + ) + mock_cm.__exit__ = MagicMock(return_value=False) + + with patch( + "backend.api.v1_filesystem.os.scandir", return_value=mock_cm + ): + response = client.get( + f"/api/v1/filesystem/browse?path={tmp_path}" + ) + + assert response.status_code == 200 + names = [e["name"] for e in response.json()["entries"]] + assert "valid_dir" in names + assert "bad\udcffname" not in names + + +# --------------------------------------------------------------------------- +# Authentication (skipped until auth middleware is implemented) +# --------------------------------------------------------------------------- + + +@pytest.mark.skip(reason="auth not yet implemented") +class TestAuthentication: + def test_unauthenticated_request_returns_401( + self, client: TestClient + ) -> None: + # When auth middleware is added, an unauthenticated GET should return 401. + response = client.get("/api/v1/filesystem/browse") + assert response.status_code == 401 diff --git a/backend/tests/test_filesystem_entry_path_contract.py b/backend/tests/test_filesystem_entry_path_contract.py new file mode 100644 index 00000000..4ae5a2fe --- /dev/null +++ b/backend/tests/test_filesystem_entry_path_contract.py @@ -0,0 +1,132 @@ +"""Regression tests for the entry-path contract in the filesystem browse endpoint. + +These tests specifically verify that every entry's ``path`` field is consistent +with ``response.path`` (both rooted at the resolved real path), which is the +contract documented in the module docstring of ``v1_filesystem.py``. + +This file was added as part of TASK-002a/PATCH to provide regression protection +for Defect #1 (entry_base symlink inconsistency). Two existing tests +(``test_only_directories_in_entries`` and ``test_symlink_path_resolves_to_real``) +verify the response structure but leave entry ``path`` values unchecked; this +file closes that gap without modifying the original test file. +""" + +from __future__ import annotations + +from pathlib import Path + +from fastapi.testclient import TestClient + + +class TestEntryPathContract: + """Entry path values must always equal response.path + "/" + entry.name.""" + + def test_only_directories_entry_path_matches_parent( + self, client: TestClient, tmp_path: Path + ) -> None: + """Regression for test_only_directories_in_entries missing entry path check. + + That test verifies one entry named "subdir" is returned but does not + assert entry["path"]. This test ensures the path is correct. + """ + (tmp_path / "subdir").mkdir() + (tmp_path / "regular_file.txt").write_text("hello") + + response = client.get(f"/api/v1/filesystem/browse?path={tmp_path}") + assert response.status_code == 200 + data = response.json() + + assert len(data["entries"]) == 1 + entry = data["entries"][0] + assert entry["name"] == "subdir" + + # The entry path must be consistent with response.path, not with the + # caller-supplied path (which may differ when symlinks are involved). + expected_entry_path = f"{data['path']}/{entry['name']}" + assert entry["path"] == expected_entry_path, ( + f"Entry path {entry['path']!r} is inconsistent with " + f"response.path {data['path']!r}. " + f"Expected: {expected_entry_path!r}" + ) + + def test_symlink_target_entry_paths_use_real_path( + self, client: TestClient, tmp_path: Path + ) -> None: + """Regression for test_symlink_path_resolves_to_real missing entry path check. + + The original test only checks response.path == real_dir.resolve(). + This test additionally verifies that entry paths are rooted at the + resolved real path, not at the symlink path. + + Before Defect #1 was fixed, entry_base used display_path (the symlink + path) so entry.path would be ``/symlink/subdir`` while response.path + was ``/real``. After the fix both must use the resolved real path. + """ + real_dir = tmp_path / "real" + real_dir.mkdir() + # Add a subdirectory so entries is non-empty. + (real_dir / "inner").mkdir() + + link_path = tmp_path / "link_to_real" + link_path.symlink_to(real_dir) + + response = client.get(f"/api/v1/filesystem/browse?path={link_path}") + assert response.status_code == 200 + data = response.json() + + real_path = str(real_dir.resolve()) + # response.path must be the resolved real path. + assert data["path"] == real_path + + # Every entry path must also be rooted at the resolved real path. + assert len(data["entries"]) == 1 + entry = data["entries"][0] + assert entry["name"] == "inner" + + expected_entry_path = f"{real_path}/{entry['name']}" + assert entry["path"] == expected_entry_path, ( + f"Entry path {entry['path']!r} does not match the resolved real " + f"path {real_path!r}. This likely means entry_base still uses " + "display_path (the symlink path) instead of the real path." + ) + + # Explicitly verify entry path is NOT rooted at the symlink path. + symlink_based_path = f"{link_path}/{entry['name']}" + assert entry["path"] != symlink_based_path, ( + "Entry path must not be rooted at the symlink path " + f"{str(link_path)!r}; it must use the real path {real_path!r}." + ) + + def test_all_entry_paths_consistent_with_response_path( + self, client: TestClient, tmp_path: Path + ) -> None: + """Generic contract: for every entry, entry.path == response.path + "/" + entry.name.""" + (tmp_path / "alpha").mkdir() + (tmp_path / "beta").mkdir() + (tmp_path / "gamma").mkdir() + (tmp_path / "file.txt").write_text("content") + + response = client.get(f"/api/v1/filesystem/browse?path={tmp_path}") + assert response.status_code == 200 + data = response.json() + + response_path = data["path"] + for entry in data["entries"]: + expected = f"{response_path}/{entry['name']}" + assert entry["path"] == expected, ( + f"Entry {entry['name']!r}: path {entry['path']!r} != " + f"expected {expected!r} (derived from response.path={response_path!r})" + ) + + def test_root_entry_path_contract(self, client: TestClient) -> None: + """Root entries: path must be '/' + name (no double slash, consistent with response.path).""" + response = client.get("/api/v1/filesystem/browse?path=/") + assert response.status_code == 200 + data = response.json() + assert data["path"] == "/" + + for entry in data["entries"]: + assert entry["path"] == f"/{entry['name']}", ( + f"Root entry {entry['name']!r} path {entry['path']!r} should be " + f"'/{entry['name']}'" + ) diff --git a/backend/tests/test_parameters.py b/backend/tests/test_parameters.py new file mode 100644 index 00000000..86b6c6e2 --- /dev/null +++ b/backend/tests/test_parameters.py @@ -0,0 +1,154 @@ +"""Tests for parameter models, service, and API endpoints.""" + +from __future__ import annotations + +from backend.models.parameter import ( + ParameterCategory, + ParameterDefinition, + ParameterFile, + ParameterType, + ParameterValidationResult, + ParameterValue, +) +from backend.services.parameter_service import ParameterService + + +class TestParameterModels: + """Verify Pydantic model creation and serialization.""" + + def test_parameter_definition_creation(self): + defn = ParameterDefinition( + name="PIXEL_SIZE", + type=ParameterType.FLOAT, + required=True, + default=1.0, + range=[0.1, 50.0], + description="Pixel size in angstroms", + category=ParameterCategory.GENERAL, + units="angstroms", + ) + assert defn.name == "PIXEL_SIZE" + assert defn.type == ParameterType.FLOAT + assert defn.required is True + assert defn.range == [0.1, 50.0] + + def test_parameter_value_creation(self): + pv = ParameterValue(name="GPU", value=[0, 1]) + assert pv.name == "GPU" + assert pv.value == [0, 1] + + def test_parameter_file_creation(self): + pf = ParameterFile( + parameters=[ + ParameterValue(name="PIXEL_SIZE", value=1.35), + ParameterValue(name="GPU", value=[0]), + ], + path="/tmp/test_param.m", + ) + assert len(pf.parameters) == 2 + assert pf.path == "/tmp/test_param.m" + + def test_validation_result(self): + result = ParameterValidationResult( + valid=False, + errors=["Missing required parameter 'PIXEL_SIZE'"], + warnings=["Unknown parameter 'foo'"], + ) + assert result.valid is False + assert len(result.errors) == 1 + assert len(result.warnings) == 1 + + +class TestParameterService: + """Verify the parameter service logic.""" + + def test_get_schema_returns_definitions(self): + service = ParameterService() + schema = service.get_schema() + assert isinstance(schema, list) + assert len(schema) > 0 + assert all(isinstance(d, ParameterDefinition) for d in schema) + + def test_parse_value_integer(self): + assert ParameterService._parse_value("42") == 42 + + def test_parse_value_float(self): + assert ParameterService._parse_value("3.14") == 3.14 + + def test_parse_value_vector(self): + assert ParameterService._parse_value("[1, 2, 3]") == [1, 2, 3] + + def test_parse_value_string(self): + assert ParameterService._parse_value("'hello'") == "hello" + + def test_format_value_bool(self): + assert ParameterService._format_value(True) == "1" + assert ParameterService._format_value(False) == "0" + + def test_format_value_list(self): + assert ParameterService._format_value([1, 2, 3]) == "[1, 2, 3]" + + def test_save_and_load_roundtrip(self, tmp_path): + service = ParameterService() + path = str(tmp_path / "roundtrip.m") + + original = ParameterFile( + parameters=[ + ParameterValue(name="PIXEL_SIZE", value=1.35), + ParameterValue(name="GPU", value=[0, 1]), + ParameterValue(name="Cls_className", value="myclass"), + ], + path=path, + ) + + service.save_parameter_file(original) + loaded = service.load_parameter_file(path) + + assert loaded.path == path + loaded_map = {p.name: p.value for p in loaded.parameters} + assert loaded_map["PIXEL_SIZE"] == 1.35 + assert loaded_map["GPU"] == [0, 1] + assert loaded_map["Cls_className"] == "myclass" + + def test_validate_missing_required(self): + service = ParameterService() + result = service.validate_parameters([]) + # The builtin schema has required params, so validation should flag them + assert result.valid is False + assert any("PIXEL_SIZE" in e for e in result.errors) + + def test_validate_unknown_parameter(self): + service = ParameterService() + result = service.validate_parameters([ + ParameterValue(name="PIXEL_SIZE", value=1.0), + ParameterValue(name="GPU", value=[0]), + ParameterValue(name="TOTALLY_FAKE", value="nope"), + ]) + assert any("TOTALLY_FAKE" in w for w in result.warnings) + + +class TestParameterEndpoints: + """Test the API endpoints via the test client.""" + + def test_get_schema(self, client): + response = client.get("/api/parameters/schema") + assert response.status_code == 200 + data = response.json() + assert isinstance(data, list) + assert len(data) > 0 + + def test_load_missing_file(self, client): + response = client.get("/api/parameters/file/nonexistent.m") + assert response.status_code == 404 + + def test_validate_endpoint(self, client): + response = client.post( + "/api/parameters/validate", + json=[ + {"name": "PIXEL_SIZE", "value": 1.0}, + {"name": "GPU", "value": [0]}, + ], + ) + assert response.status_code == 200 + data = response.json() + assert "valid" in data diff --git a/backend/tests/test_projects.py b/backend/tests/test_projects.py new file mode 100644 index 00000000..0b8e3bfd --- /dev/null +++ b/backend/tests/test_projects.py @@ -0,0 +1,122 @@ +"""Tests for project models, service, and API endpoints.""" + +from __future__ import annotations + +from pathlib import Path + +from backend.models.project import Project, ProjectState, TiltSeries +from backend.services.project_service import ProjectService + + +class TestProjectModels: + """Verify Pydantic model creation.""" + + def test_project_creation(self): + project = Project( + name="test", + path="/tmp/test", + state=ProjectState.UNINITIALIZED, + current_cycle=0, + tilt_series=[], + ) + assert project.name == "test" + assert project.state == ProjectState.UNINITIALIZED + + def test_tilt_series_creation(self): + ts = TiltSeries( + name="tilt1", + stack_path="/data/tilt1.st", + rawtlt_path="/data/tilt1.rawtlt", + aligned=False, + ctf_estimated=False, + ) + assert ts.name == "tilt1" + assert ts.aligned is False + + def test_project_state_values(self): + """Ensure all expected states exist in the enum.""" + expected = { + "uninitialized", + "tilt_aligned", + "ctf_estimated", + "reconstructed", + "particles_picked", + "initialized", + "cycle_n", + "export", + "done", + } + actual = {s.value for s in ProjectState} + assert expected == actual + + +class TestProjectService: + """Verify the project service logic.""" + + def test_create_project(self, tmp_path): + service = ProjectService() + project_path = str(tmp_path / "new_project") + project = service.create_project("my_project", project_path) + + assert project.name == "my_project" + assert project.state == ProjectState.UNINITIALIZED + assert Path(project_path).exists() + assert (Path(project_path) / "rawData").exists() + assert (Path(project_path) / "logFile").exists() + + def test_load_project(self, tmp_project): + service = ProjectService() + project = service.load_project(str(tmp_project)) + + assert project.name == "test_project" + assert project.state == ProjectState.UNINITIALIZED + + def test_detect_tilt_aligned_state(self, tmp_project): + # Create a dummy file in fixedStacks to simulate alignment + (tmp_project / "fixedStacks" / "tilt1_ali.mrc").touch() + service = ProjectService() + project = service.load_project(str(tmp_project)) + assert project.state == ProjectState.TILT_ALIGNED + + def test_discover_tilt_series(self, tmp_project): + # Create dummy tilt-series files + (tmp_project / "rawData" / "tilt1.st").touch() + (tmp_project / "rawData" / "tilt1.rawtlt").touch() + (tmp_project / "rawData" / "tilt2.st").touch() + + service = ProjectService() + series = service.list_tilt_series(str(tmp_project)) + + assert len(series) == 2 + names = {ts.name for ts in series} + assert "tilt1" in names + assert "tilt2" in names + + # tilt1 has a rawtlt file, tilt2 does not + tilt1 = next(ts for ts in series if ts.name == "tilt1") + tilt2 = next(ts for ts in series if ts.name == "tilt2") + assert tilt1.rawtlt_path is not None + assert tilt2.rawtlt_path is None + + +class TestProjectEndpoints: + """Test the API endpoints via the test client.""" + + def test_create_project(self, client, tmp_path): + response = client.post( + "/api/projects", + json={"name": "api_test", "path": str(tmp_path / "api_project")}, + ) + assert response.status_code == 200 + data = response.json() + assert data["name"] == "api_test" + assert data["state"] == "uninitialized" + + def test_load_nonexistent_project(self, client): + response = client.get("/api/projects/nonexistent/path/nowhere") + assert response.status_code == 404 + + def test_health_check(self, client): + response = client.get("/api/health") + assert response.status_code == 200 + assert response.json()["status"] == "ok" diff --git a/backend/tests/test_safe_json.py b/backend/tests/test_safe_json.py new file mode 100644 index 00000000..3b00a589 --- /dev/null +++ b/backend/tests/test_safe_json.py @@ -0,0 +1,286 @@ +"""Tests for backend.utils.safe_json -- atomic writes and dual-locking.""" + +from __future__ import annotations + +import json +import threading +from pathlib import Path + +import pytest + +from backend.utils.safe_json import ( + atomic_write, + atomic_write_text, + locked_json_read, + locked_json_read_write, +) + + +class TestAtomicWrite: + """Verify atomic_write produces valid JSON files.""" + + def test_creates_file(self, tmp_path: Path) -> None: + target = tmp_path / "out.json" + atomic_write(target, {"key": "value"}) + assert target.exists() + assert json.loads(target.read_text()) == {"key": "value"} + + def test_overwrites_existing(self, tmp_path: Path) -> None: + target = tmp_path / "out.json" + atomic_write(target, {"a": 1}) + atomic_write(target, {"b": 2}) + assert json.loads(target.read_text()) == {"b": 2} + + def test_creates_parent_dirs(self, tmp_path: Path) -> None: + target = tmp_path / "sub" / "dir" / "out.json" + atomic_write(target, [1, 2, 3]) + assert json.loads(target.read_text()) == [1, 2, 3] + + def test_no_temp_file_left_on_success(self, tmp_path: Path) -> None: + target = tmp_path / "out.json" + atomic_write(target, {"ok": True}) + assert not target.with_suffix(".tmp").exists() + + +class TestLockedJsonReadWrite: + """Verify locked_json_read_write provides correct read-modify-write.""" + + def test_creates_file_from_none(self, tmp_path: Path) -> None: + target = tmp_path / "new.json" + + def init(data): + assert data is None + return {"count": 0} + + result = locked_json_read_write(target, init) + assert result == {"count": 0} + assert json.loads(target.read_text()) == {"count": 0} + + def test_reads_existing_data(self, tmp_path: Path) -> None: + target = tmp_path / "existing.json" + target.write_text(json.dumps({"items": [1, 2]})) + + def add_item(data): + data["items"].append(3) + return data + + result = locked_json_read_write(target, add_item) + assert result == {"items": [1, 2, 3]} + + def test_transform_error_does_not_corrupt(self, tmp_path: Path) -> None: + target = tmp_path / "safe.json" + target.write_text(json.dumps({"original": True})) + + def bad_transform(data): + raise ValueError("intentional failure") + + with pytest.raises(ValueError, match="intentional failure"): + locked_json_read_write(target, bad_transform) + + # Original data must still be intact + assert json.loads(target.read_text()) == {"original": True} + + +class TestConcurrency: + """Verify that concurrent access via locked_json_read_write is safe.""" + + def test_concurrent_increments(self, tmp_path: Path) -> None: + """Launch 100 threads each incrementing a counter — no lost updates.""" + target = tmp_path / "counter.json" + target.write_text(json.dumps({"count": 0})) + + num_threads = 100 + errors: list[str] = [] + + def increment() -> None: + try: + def _inc(data): + data["count"] += 1 + return data + + locked_json_read_write(target, _inc) + except Exception as exc: + errors.append(str(exc)) + + threads = [threading.Thread(target=increment) for _ in range(num_threads)] + for t in threads: + t.start() + for t in threads: + t.join() + + assert not errors, f"Errors during concurrent increments: {errors}" + + final = json.loads(target.read_text()) + assert final["count"] == num_threads, ( + f"Expected count={num_threads}, got {final['count']} — lost updates detected" + ) + + def test_concurrent_project_creates_no_data_loss(self, tmp_path: Path) -> None: + """Simulate 50+ concurrent project-create operations. + + Each thread adds a unique project entry to a shared registry JSON. + After all threads complete: + - All projects must be present (no data loss) + - The JSON must be valid (no truncation) + - No exceptions raised + """ + target = tmp_path / "projects.json" + target.write_text(json.dumps({})) + + num_projects = 60 + errors: list[str] = [] + + def create_project(idx: int) -> None: + try: + project_id = f"proj-{idx:04d}" + + def _add(data): + if data is None: + data = {} + data[project_id] = { + "id": project_id, + "name": f"Project {idx}", + "directory": f"/tmp/proj_{idx}", + } + return data + + locked_json_read_write(target, _add) + except Exception as exc: + errors.append(f"Thread {idx}: {exc}") + + threads = [ + threading.Thread(target=create_project, args=(i,)) + for i in range(num_projects) + ] + for t in threads: + t.start() + for t in threads: + t.join() + + # No exceptions + assert not errors, f"Errors during concurrent creates: {errors}" + + # Valid JSON (no truncation) + raw = target.read_text() + registry = json.loads(raw) + + # All projects present + assert len(registry) == num_projects, ( + f"Expected {num_projects} projects, got {len(registry)} — data loss detected" + ) + for i in range(num_projects): + project_id = f"proj-{i:04d}" + assert project_id in registry, f"Missing project {project_id}" + + +# --------------------------------------------------------------------------- +# atomic_write_text +# --------------------------------------------------------------------------- + + +class TestAtomicWriteText: + """Verify atomic_write_text writes text atomically with UTF-8 encoding.""" + + def test_writes_content(self, tmp_path: Path) -> None: + target = tmp_path / "out.txt" + atomic_write_text(target, "hello world") + assert target.read_text(encoding="utf-8") == "hello world" + + def test_creates_parent_dirs(self, tmp_path: Path) -> None: + target = tmp_path / "a" / "b" / "deep.txt" + atomic_write_text(target, "nested") + assert target.read_text(encoding="utf-8") == "nested" + + def test_utf8_encoding(self, tmp_path: Path) -> None: + """Content with non-ASCII characters must be written as UTF-8.""" + target = tmp_path / "utf8.txt" + content = "cafÊ rÊsumÊ naïve æ—ĨæœŦčĒž" + atomic_write_text(target, content) + assert target.read_bytes().decode("utf-8") == content + + def test_overwrites_existing(self, tmp_path: Path) -> None: + target = tmp_path / "over.txt" + atomic_write_text(target, "original") + atomic_write_text(target, "replaced") + assert target.read_text(encoding="utf-8") == "replaced" + + def test_no_temp_file_on_success(self, tmp_path: Path) -> None: + target = tmp_path / "clean.txt" + atomic_write_text(target, "data") + assert not target.with_suffix(".tmp").exists() + + def test_accepts_string_path(self, tmp_path: Path) -> None: + target = str(tmp_path / "str_path.txt") + atomic_write_text(target, "works") + assert Path(target).read_text(encoding="utf-8") == "works" + + +# --------------------------------------------------------------------------- +# locked_json_read +# --------------------------------------------------------------------------- + + +class TestLockedJsonRead: + """Verify locked_json_read returns correct data under various conditions.""" + + def test_returns_none_for_missing_file(self, tmp_path: Path) -> None: + result = locked_json_read(tmp_path / "nonexistent.json") + assert result is None + + def test_returns_none_for_empty_file(self, tmp_path: Path) -> None: + target = tmp_path / "empty.json" + target.write_text("", encoding="utf-8") + assert locked_json_read(target) is None + + def test_returns_none_for_whitespace_only(self, tmp_path: Path) -> None: + target = tmp_path / "ws.json" + target.write_text(" \n \t ", encoding="utf-8") + assert locked_json_read(target) is None + + def test_reads_dict(self, tmp_path: Path) -> None: + target = tmp_path / "data.json" + data = {"key": "value", "count": 42} + target.write_text(json.dumps(data), encoding="utf-8") + assert locked_json_read(target) == data + + def test_reads_list(self, tmp_path: Path) -> None: + target = tmp_path / "list.json" + target.write_text("[1, 2, 3]", encoding="utf-8") + assert locked_json_read(target) == [1, 2, 3] + + def test_accepts_string_path(self, tmp_path: Path) -> None: + target = tmp_path / "str.json" + target.write_text('{"ok": true}', encoding="utf-8") + assert locked_json_read(str(target)) == {"ok": True} + + def test_roundtrip_with_atomic_write(self, tmp_path: Path) -> None: + """locked_json_read can read files produced by atomic_write.""" + target = tmp_path / "roundtrip.json" + data = {"roundtrip": True, "nested": {"a": 1}} + atomic_write(target, data) + assert locked_json_read(target) == data + + def test_concurrent_reads(self, tmp_path: Path) -> None: + """Multiple threads reading simultaneously should all succeed.""" + target = tmp_path / "concurrent.json" + data = {"thread_safe": True, "value": 12345} + target.write_text(json.dumps(data), encoding="utf-8") + + results: list[object] = [None] * 10 + errors: list[Exception | None] = [None] * 10 + + def reader(idx: int) -> None: + try: + results[idx] = locked_json_read(target) + except Exception as exc: + errors[idx] = exc + + threads = [threading.Thread(target=reader, args=(i,)) for i in range(10)] + for t in threads: + t.start() + for t in threads: + t.join() + + for i in range(10): + assert errors[i] is None, f"Thread {i} raised: {errors[i]}" + assert results[i] == data diff --git a/backend/utils/__init__.py b/backend/utils/__init__.py new file mode 100644 index 00000000..d007834b --- /dev/null +++ b/backend/utils/__init__.py @@ -0,0 +1 @@ +"""Backend utility modules.""" diff --git a/backend/utils/machine_config.py b/backend/utils/machine_config.py new file mode 100644 index 00000000..89930f48 --- /dev/null +++ b/backend/utils/machine_config.py @@ -0,0 +1,28 @@ +"""Machine-level configuration helpers. + +Reads configuration from environment variables. Can be expanded later +for additional machine-specific settings. +""" + +from __future__ import annotations + +import functools +import os +from pathlib import Path + + +@functools.cache +def get_registry_dir() -> Path: + """Return the registry directory path. + + Reads from ``EMCLARITY_REGISTRY_DIR`` environment variable, defaulting + to ``~/.emclarity`` if not set or empty. + + The result is cached so the path is resolved exactly once per process, + guaranteeing structural stability rather than relying on the environment + variable remaining unchanged between calls. + """ + env_value = os.environ.get("EMCLARITY_REGISTRY_DIR", "").strip() + if env_value: + return Path(env_value) + return Path.home() / ".emclarity" diff --git a/backend/utils/safe_json.py b/backend/utils/safe_json.py new file mode 100644 index 00000000..a6079868 --- /dev/null +++ b/backend/utils/safe_json.py @@ -0,0 +1,167 @@ +"""Thread-safe and process-safe JSON persistence utilities. + +Provides two core primitives: + +* :func:`atomic_write` -- write JSON to disk atomically via temp-file + ``os.replace()``. +* :func:`locked_json_read_write` -- read-modify-write a JSON file under dual locking + (``threading.Lock`` for in-process thread safety *and* ``fcntl.flock`` for + cross-process safety). + +Why dual locking +~~~~~~~~~~~~~~~~ +``fcntl.flock()`` is process-level only -- concurrent threads within a single FastAPI +worker are **not** protected by fcntl alone. The ``threading.Lock`` prevents in-process +races; ``fcntl.flock`` prevents cross-process races. +""" + +from __future__ import annotations + +import fcntl +import json +import logging +import os +import threading +from pathlib import Path +from typing import Any, Callable, TypeVar + +log = logging.getLogger(__name__) + +T = TypeVar("T") + +# Module-level thread lock -- one per import (i.e. one per process). +_lock = threading.Lock() + + +def atomic_write_text(path: Path | str, content: str) -> None: + """Write *content* as text to *path* atomically. + + Writes to a temporary file in the same directory, then uses + ``os.replace()`` which is atomic on POSIX filesystems. This avoids + partial/corrupt reads if the process is interrupted mid-write. + + Unlike :func:`atomic_write`, this accepts a raw string rather than + JSON-serialisable data. + + Args: + path: Destination file path. + content: Text content to write. + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + try: + tmp.write_text(content, encoding="utf-8") + os.replace(tmp, path) + except Exception: + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + raise + + +def atomic_write(path: Path | str, data: Any, *, indent: int = 2) -> None: + """Write *data* as JSON to *path* atomically. + + Writes to a temporary file in the same directory, then uses + ``os.replace()`` which is atomic on POSIX filesystems. This avoids + partial/corrupt reads if the process is interrupted mid-write. + + Args: + path: Destination file path. + data: JSON-serialisable data. + indent: JSON indentation level (default 2). + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_suffix(".tmp") + try: + tmp.write_text(json.dumps(data, indent=indent), encoding="utf-8") + os.replace(tmp, path) + except Exception: + # Clean up the temp file on failure if it exists + try: + tmp.unlink(missing_ok=True) + except OSError: + pass + raise + + +def locked_json_read(path: Path | str) -> Any: + """Thread-safe AND process-safe JSON read (no write-back). + + Acquires dual locks (``threading.Lock`` + ``fcntl.flock``) to ensure + a consistent read even if a concurrent writer is active, but does + **not** write the data back to disk afterwards. + + Use this instead of :func:`locked_json_read_write` when you only + need to read the file contents without modifying them. + + Args: + path: Path to the JSON file. + + Returns: + The parsed JSON data, or ``None`` if the file is empty or missing. + """ + path = Path(path) + + with _lock: + if not path.exists(): + return None + with open(path, "r", encoding="utf-8") as f: + fcntl.flock(f.fileno(), fcntl.LOCK_SH) + try: + content = f.read() + if content.strip(): + return json.loads(content) + return None + finally: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) + + +def locked_json_read_write( + path: Path | str, + transform_fn: Callable[[Any], T], +) -> T: + """Thread-safe AND process-safe JSON read-modify-write. + + Acquires a ``threading.Lock`` (for in-process thread safety among + FastAPI async workers) **and** an exclusive ``fcntl.flock`` (for + cross-process safety when multiple backend instances share a file). + + The *transform_fn* receives the parsed JSON data and must return the + new data to be written back. The return value of *transform_fn* is + also returned from this function. + + If the file does not exist, *transform_fn* receives ``None`` so the + caller can initialise the file. + + Args: + path: Path to the JSON file. + transform_fn: ``data_in -> data_out`` callback. + + Returns: + The value returned by *transform_fn* (i.e. the new file contents). + """ + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + + with _lock: + # Open in "a+" so the file is created if it doesn't exist, + # then seek back to read. We hold the flock for the entire + # read-modify-write cycle. + with open(path, "a+", encoding="utf-8") as f: + fcntl.flock(f.fileno(), fcntl.LOCK_EX) + try: + f.seek(0) + content = f.read() + if content.strip(): + data = json.loads(content) + else: + data = None + + result = transform_fn(data) + atomic_write(path, result) + return result + finally: + fcntl.flock(f.fileno(), fcntl.LOCK_UN) diff --git a/coordinates/BH_decomposeIMODxf.m b/coordinates/BH_decomposeIMODxf.m index 5fe5e65c..2ac687bd 100755 --- a/coordinates/BH_decomposeIMODxf.m +++ b/coordinates/BH_decomposeIMODxf.m @@ -7,21 +7,21 @@ a21 = IMOD_XF(3); a22 = IMOD_XF(4); -% * Converts a 2 by 2 transformation matrix into four "natural" parameters of +% * Converts a 2 by 2 transformation matrix into four "natural" parameters of % * image transformation. The transformation is specified by [a11], [a12], % * [a21], and [a22], where % * ^ x' = a11 * x + a12 * y % * ^ y' = a21 * x + a22 * y ^ -% * In the converted transformation, [theta] is overall rotation, [smag] is +% * In the converted transformation, [theta] is overall rotation, [smag] is % * overall magnification, [str] is a unidirectional stretch, and [phi] is the % * angle of the stretch axis. Two equivalent solutions are possible, with the -% * stretch axis in the first or fourth quadrant. The function returns the +% * stretch axis in the first or fourth quadrant. The function returns the % * solution that makes the magnification [smag] nearer to 1.0. - + % Just calc in degrees directly % ator = 0.0174532925; -% -% /* +% +% /* % To solve for the variables, the first step is to solve for THETA by % taking the arctangent of a function of the AMAT values. It is then % possible to compute F1, F2 and F3, intermediate factors whose @@ -34,118 +34,118 @@ % one of two different formulas, depending on whether PHI is near 45 % degrees or not, then SMAG is computed. % */ - + % /* first determine if there is an axis inversion: find angle from % transformed X axis to transformed Y axis and reduce to -180 to 180 % If difference is negative then invert Y components of matrix */ - - dtheta = atan2d(a22, a12) - atan2d(a21, a11); - - if (dtheta > 180.); dtheta = dtheta - 360; end - if (dtheta <= -180.); dtheta = dtheta + 360; end - if (dtheta < 0.) - a12 = -a12; - a22 = -a22; - end + +dtheta = atan2d(a22, a12) - atan2d(a21, a11); + +if (dtheta > 180.); dtheta = dtheta - 360; end +if (dtheta <= -180.); dtheta = dtheta + 360; end +if (dtheta < 0.) + a12 = -a12; + a22 = -a22; +end % /* next find the rotation angle theta that gives the same solution for % f2 when derived from a11 and a21 as when derived from a12 and a22 */ - theta = 0; - if (a21 ~= a12 || a22 ~= -1*a11) - theta = atan2d((a21-a12), (a22+a11)); - end - costh = cosd(theta); - sinth = sind(theta); - - f1 = a11*costh+a21*sinth; - f2 = a21*costh-a11*sinth; - f3 = a22*costh-a12*sinth; - +theta = 0; +if (a21 ~= a12 || a22 ~= -1*a11) + theta = atan2d((a21-a12), (a22+a11)); +end +costh = cosd(theta); +sinth = sind(theta); + +f1 = a11*costh+a21*sinth; +f2 = a21*costh-a11*sinth; +f3 = a22*costh-a12*sinth; + % /* Next solve for phi */ - - if (f2 < 1.e-10 && f2 > -1.e-10) - -% /* if f2 = 0, pick phi = 0., set cos phi to 1. */ - cosphisq = 1.; - else - -% /* otherwise, solve quadratic equation, pick the solution that is -% right for the first quadrant */ - afac = (f3-f1)*(f3-f1); - bfac = 4.*f2*f2; - cosphisq = 0.5*(1.+sqrt(1.-bfac/(bfac+afac))); - sinphisq = 1.-cosphisq; - fnum = f1*cosphisq-f3*sinphisq; - if (fnum < 0.) - fnum = -1*fnum; - end - fden = f3*cosphisq-f1*sinphisq; - if (fden < 0.) - fden = -fden; - end - if ((f2 > 0. && fnum < fden) || (f2 < 0. && fnum > fden)) - cosphisq = 1.-cosphisq; - end - end - phi = acosd(sqrt(cosphisq)); + +if (f2 < 1.e-10 && f2 > -1.e-10) + + % /* if f2 = 0, pick phi = 0., set cos phi to 1. */ + cosphisq = 1.; +else + + % /* otherwise, solve quadratic equation, pick the solution that is + % right for the first quadrant */ + afac = (f3-f1)*(f3-f1); + bfac = 4.*f2*f2; + cosphisq = 0.5*(1.+sqrt(1.-bfac/(bfac+afac))); sinphisq = 1.-cosphisq; - -% /* solve for str. */ - - if (cosphisq-0.5 > 0.25 || cosphisq - 0.5 < - 0.25) + fnum = f1*cosphisq-f3*sinphisq; + if (fnum < 0.) + fnum = -1*fnum; + end + fden = f3*cosphisq-f1*sinphisq; + if (fden < 0.) + fden = -fden; + end + if ((f2 > 0. && fnum < fden) || (f2 < 0. && fnum > fden)) + cosphisq = 1.-cosphisq; + end +end +phi = acosd(sqrt(cosphisq)); +sinphisq = 1.-cosphisq; -% /* for angles far from 45 deg, use an equation that is good at 0 -% or 90 deg but blows up at 45 deg. */ - str = (f1*cosphisq-f3*sinphisq)/(f3*cosphisq-f1*sinphisq); +% /* solve for str. */ - else +if (cosphisq-0.5 > 0.25 || cosphisq - 0.5 < - 0.25) + + % /* for angles far from 45 deg, use an equation that is good at 0 + % or 90 deg but blows up at 45 deg. */ + str = (f1*cosphisq-f3*sinphisq)/(f3*cosphisq-f1*sinphisq); + +else + + % /* for angles near 45 deg, use an equation that is good there but + % blows up at 0. */ + factmp = (f1+f3)*sqrt(cosphisq*sinphisq); + str = (factmp+f2)/(factmp-f2); +end -% /* for angles near 45 deg, use an equation that is good there but -% blows up at 0. */ - factmp = (f1+f3)*sqrt(cosphisq*sinphisq); - str = (factmp+f2)/(factmp-f2); - end - % /* solve for smag from the equation for f1, or f2 if that would fail % (which it does with stretch -1 along 45 degree line) */ - - dentmp = str * cosphisq + sinphisq; - if(dentmp > 1.e-5 || dentmp < -1.e-5) - smag = f1/dentmp; - else - smag = 1./((str-1.)*sqrt(cosphisq*sinphisq)); - end - + +dentmp = str * cosphisq + sinphisq; +if(dentmp > 1.e-5 || dentmp < -1.e-5) + smag = f1/dentmp; +else + smag = 1./((str-1.)*sqrt(cosphisq*sinphisq)); +end + % /* if it will make smag closer to 1.0, flip stretch axis 90 deg */ - - f1 = smag - 1; - f2 = str * smag - 1; - if (f1 < 0.) - f1 = -f1; - end - if (f2 < 0.) - f2 = -f2; - end - if(f1 > f2) - smag = smag * str; - str = 1 / str; - phi = phi-90; - end - +f1 = smag - 1; +f2 = str * smag - 1; +if (f1 < 0.) + f1 = -f1; +end +if (f2 < 0.) + f2 = -f2; +end +if(f1 > f2) + smag = smag * str; + str = 1 / str; + phi = phi-90; +end + + % /* Now if there is an inversion, then invert the stretch, mirror the % stretch axis, and add a rotation to bring inverted point along stretch % axis to a point mirrored around X */ -% - if (dtheta < 0) - str = -1*str; - phi = -1*phi; - theta = theta + 180 - 2 * phi; - if (theta > 180) - theta = theta - 180; - end +% +if (dtheta < 0) + str = -1*str; + phi = -1*phi; + theta = theta + 180 - 2 * phi; + if (theta > 180) + theta = theta - 180; end +end diff --git a/coordinates/BH_defineMatrix.m b/coordinates/BH_defineMatrix.m index b164a167..b76a9620 100755 --- a/coordinates/BH_defineMatrix.m +++ b/coordinates/BH_defineMatrix.m @@ -16,7 +16,7 @@ % % DIRECTION = forward : rotation from microscope frame to particle frame. % inverse : rotation from particle frame to microscope frame. -% +% % Output variables: % % ROTATION_MATRIX = 3d rotation matrix @@ -31,10 +31,10 @@ % % These are general, but in the scope of the BH_subTomo programs, they are % generally applied to an ndgrid which is transformed and used as the query to -% an interpolation. +% an interpolation. % % Regardless of how they are used, the angles are interpreted to reflect an -% active, intrinsic transformation on a particle, and the convention and +% active, intrinsic transformation on a particle, and the convention and % direction are taken into account in order for this to work. % % A good test is to create wedge masks of varying orientation because these @@ -57,8 +57,8 @@ % Normalize to unit sphere; randXYZ = randXYZ ./ sqrt(sum(randXYZ.^2,2)); angles = [atan2(randXYZ(2),randXYZ(1)), ... - acos(randXYZ(3)), ... - (2.*pi.*(rand(1) - 0.5))]; % between -pi/pi + acos(randXYZ(3)), ... + (2.*pi.*(rand(1) - 0.5))]; % between -pi/pi % Make sure to override conventions for consistency. CONVENTION = 'Bah'; @@ -72,113 +72,131 @@ % Rx = @(t)[ 1 0 0 ;... % 0 cos(t) -sin(t);... % 0 sin(t) cos(t) ]; -% +% % Ry = @(t)[ cos(t) 0 sin(t);... % 0 1 0;... % -sin(t) 0 cos(t) ]; -% +% % Rz = @(t)[ cos(t) -sin(t) 0;... % sin(t) cos(t) 0;... % 0 0 1 ]; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -if strcmpi(DIRECTION, 'forward') || strcmpi(DIRECTION, 'invVector') - angles = -1.*angles; -elseif strcmpi(DIRECTION, 'inv') || strcmpi(DIRECTION, 'forwardVector') - % For interpolation the vectors are applied to a grid, so the sense must - % be inverted to make the final transformation active. - - % In order to rotate the particle from a position defined by the input - % angles, back to the proper reference frame, the sense is already - % inverted, and just the order must be inverted. - % - % Think of this as taking an average in the proper frame, applying a given - % rotation with 'forward', then this undoes that action. - % - % IMPORTANT NOTE: because the order is flipped, successive rotations by - % multiple matrices must be right multplied for inverse operations. eg: - % R1(e1,e2,e3) & R2(e4,e5,e6) then Rtot = R1 * R2 = e1*e2*e3*e4*e5*e6*Mat - angles = flip(angles); -% angles = [angles(3), angles(2), angles(1)]; - else +if strcmpi(DIRECTION, 'forward') || strcmpi(DIRECTION, 'fwd') || strcmpi(DIRECTION, 'invVector') + helical_pre = false; + angles = -1.*angles; +elseif strcmpi(DIRECTION, 'inverse') || strcmpi(DIRECTION, 'inv') || strcmpi(DIRECTION, 'fwdVector') + helical_pre = true; + % For interpolation the vectors are applied to a grid, so the sense must + % be inverted to make the final transformation active. + + + % In order to rotate the particle from a position defined by the input + % angles, back to the proper reference frame, the sense is already + % inverted, and just the order must be inverted. + % + % Think of this as taking an average in the proper frame, applying a given + % rotation with 'forward', then this undoes that action. + % + % IMPORTANT NOTE: because the order is flipped, successive rotations by + % multiple matrices must be right multplied for inverse operations. eg: + % R1(e1,e2,e3) & R2(e4,e5,e6) then Rtot = R1 * R2 = e1*e2*e3*e4*e5*e6*Mat + % NOTE on flip: the flip is unflipped (which is very confusing) so the preceding line is what is implemented. + % NOTE on sense: this is also confusing, the definition used here rotates a vector anti-clockwise for a positive rotation. + % We are using this to mean, the particle is rotated in this manner and we want to rotate it by -1*angle. By rotating the interpolant grid + % by +angle we get the value there and bring it back (rotate by -angle) to the average frame. In that meaning, the sense/sign is "inverted" + % So to recap, angles passed in are considered active intrinsict rotations, e1, then e2', then e3'' (or extrinsic e3,e2,e1) + angles = flip(angles); + % angles = [angles(3), angles(2), angles(1)]; +else error('Direction must be forward or inv, not %s', DIRECTION) end % Reduce number of trig functions cosA = cos(angles); sinA = sin(angles); - - + + switch CONVENTION case 'Bah' - -% ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Rz(angles(1)); + + % ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Rz(angles(1)); ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [1,0,0; ... - 0,cosA(2),-sinA(2);... - 0,sinA(2),cosA(2)] * ... - [cosA(1),-sinA(1),0;... - sinA(1),cosA(1),0;... - 0,0,1] ; - + sinA(3),cosA(3),0;... + 0,0,1] * ... + [1,0,0; ... + 0,cosA(2),-sinA(2);... + 0,sinA(2),cosA(2)] * ... + [cosA(1),-sinA(1),0;... + sinA(1),cosA(1),0;... + 0,0,1] ; + case 'TILT' - + ROTATION_MATRIX = [cosA,0,sinA; ... - 0,1,0;... - -sinA,0,cosA]; - - + 0,1,0;... + -sinA,0,cosA]; + + case 'SPIDER' -% ROTATION_MATRIX = Rz(angles(3)) * Ry(angles(2)) * Rz(angles(1)); - + % ROTATION_MATRIX = Rz(angles(3)) * Ry(angles(2)) * Rz(angles(1)); + ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [cosA(2),0,sinA(2); ... - 0,1,0;... - -sinA(2),0,cosA(2)] * ... - [cosA(1),-sinA(1),0;... - sinA(1),cosA(1),0;... - 0,0,1] ; - + sinA(3),cosA(3),0;... + 0,0,1] * ... + [cosA(2),0,sinA(2); ... + 0,1,0;... + -sinA(2),0,cosA(2)] * ... + [cosA(1),-sinA(1),0;... + sinA(1),cosA(1),0;... + 0,0,1] ; + case 'Helical' + % ROTATION_MATRIX = Ry(angles(3)) * Rz(angles(2)) * Ry(angles(1)) ; -% ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Ry(angles(1)); - - ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [1,0,0; ... - 0,cosA(2),-sinA(2);... - 0,sinA(2),cosA(2)] * ... - [cosA(1),0,sinA(1); ... - 0,1,0;... - -sinA(1),0,cosA(1)]; + ROTATION_MATRIX = [cosA(3),0,sinA(3); ... + 0,1,0;... + -sinA(3),0,cosA(3)] * ... + [cosA(2),-sinA(2),0;... + sinA(2),cosA(2),0;... + 0,0,1] * ... + [cosA(1),0,sinA(1); ... + 0,1,0;... + -sinA(1),0,cosA(1)]; + % % ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Ry(angles(1)); + + % ROTATION_MATRIX = [cosA(3),-sinA(3),0;... + % sinA(3),cosA(3),0;... + % 0,0,1] * ... + % [1,0,0; ... + % 0,cosA(2),-sinA(2);... + % 0,sinA(2),cosA(2)] * ... + % [cosA(1),0,sinA(1); ... + % 0,1,0;... + % -sinA(1),0,cosA(1)]; + case 'IMOD' - + cosA = cos(angles); sinA = sin(angles); -% ROTATION_MATRIX = Rz(angles(3)) * Ry(angles(2)) * Rx(angles(1)); - + % ROTATION_MATRIX = Rz(angles(3)) * Ry(angles(2)) * Rx(angles(1)); + ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [cosA(2),0,sinA(2); ... - 0,1,0;... - -sinA(2),0,cosA(2)] * ... - [cosA(1),-sinA(1),0;... - sinA(1),cosA(1),0;... - 0,0,1] ; - + sinA(3),cosA(3),0;... + 0,0,1] * ... + [cosA(2),0,sinA(2); ... + 0,1,0;... + -sinA(2),0,cosA(2)] * ... + [cosA(1),-sinA(1),0;... + sinA(1),cosA(1),0;... + 0,0,1] ; + otherwise error('Convention must be Bah,SPI,Helical, not %s', CONVENTION) end @@ -188,55 +206,55 @@ % case 'Protomo' % % passive, intrinsic, Z X Z % % i3euler e1 e2 e3 -% +% % if strcmpi(dir,'forward') -% +% % elseif strcmpi(dir, 'inv') % ang = -1.* [ang(3), ang(2), ang(1)]; -% -% +% +% % elseif strcmpi(dir, 'i3') % ang = [ang(3), ang(2), ang(1)]; % end -% +% % RotMat = Rz(ang(3)) * Rx(ang(2)) * Rz(ang(1)); -% -% +% +% % case 'Imod' % % active, extrinsic, Z Y X -% +% % if strcmpi(dir, 'forward') % ang = -1 .* ang ; % end -% +% % RotMat = Rx(ang(3))*Ry(ang(2))*Rz(ang(1)) ; -% +% % case 'Spider' % % passive, extrinsic Z Y Z % % Note that in their documents they refer to the "object" % % rotating clockwise, which sounds active, but this is the -% % same as the CS anti-clockwise, which is just a passive +% % same as the CS anti-clockwise, which is just a passive % % (alias) rotation. I believe Frealign, and Relion also use. -% +% % % Spider puts the origin at top left, with first z on top -% +% % RotMat = Rz(-e3)*Ry(-e2)*Rz(-e1) ; -% +% % case '2d' % % active rotation, second two euler angles are dummy var -% +% % RotMat = Rz(e1); % RotMat = RotMat(1:2,1:2) ; -% +% % case 'NegProtomo' % % passive, intrinsic, Z X Z % % i3euler e1 e2 e3 -% -% +% +% % RotMat = Rz(-ang(1)) * Rx(-ang(2)) * Rz(-ang(3)) ; -% +% % end - - + + diff --git a/coordinates/BH_multi_angularSearch.m b/coordinates/BH_multi_angularSearch.m deleted file mode 100755 index fdd6f6d0..00000000 --- a/coordinates/BH_multi_angularSearch.m +++ /dev/null @@ -1,424 +0,0 @@ -function [ CCC_STORAGE] = BH_multi_angularSearch( ANGLE_STEP, ... - PEAK_LIST, ... - IN_PLANE_SEARCH, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padREF,... - peakMask, peakCOM, IDX, ... - refSym) -%UNTITLED Summary of this function goes her -% Detailed explanation goes here - -% Make a normalization factor for wedge weighting. This could be done in the -% begining, but to test I'll put it here so I don't have to change functino I/o - -peakBinary = (peakMask >= 0.01); -volBinary = (volMask >= 0.01); - - - -% If searching the full in plane range, limit based on symmetry. Important for -% wedge bias and also helps with speed. -if IN_PLANE_SEARCH(1) == -180 - limitSymmetry = 1; - fprintf('limiting to symmetry constrained in-plane search.\n') -else - limitSymmetry = 0; -end - - - -iClassTrim = iClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - -%[ iClassImg ] = BH_bandLimitCenterNormalize(unMaskedClassImage, bandpassFilt, volMask); - - - -if length(PEAK_LIST) > 1 - % Search angles based on previously found peaks. - peakList = PEAK_LIST; - - % Either a refinment around top three or top one peaks, or checking all - % inplane angles for top ten axially averaged. - if (size(peakList,2) == 10) - % Refinementd - % Get unique references for peakList wedge weight normalization - referenceList = unique(peakList(:,1)); - nRefs = length(referenceList); - iClassImg2 = cell(nRefs,1); - for iRef = 1:length(bandpassFilt) - iRef - size( iClassTrim) - size(bandpassFilt{iRef}) - [ iClassImg2{iRef} ] = BH_bandLimitCenterNormalize(iClassTrim.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - end - angCount=1; - - nAngles = 46.*size(peakList,1)+1 - pause(4) - cccStorage = zeros(nAngles, 13, 'double', 'gpuArray'); - - for iAngle = 1 : size(peakList,1) - - - iRef = peakList(iAngle, 1); - phi = peakList(iAngle, 2); phiInc = peakList(iAngle,5); - theta= peakList(iAngle, 3); thetaInc = peakList(iAngle,6); - psi = peakList(iAngle, 4); psiInc = peakList(iAngle,7); - - if size(peakList,1) == 1 % This is the final refinement -% superSample = 1; - inPlaneSearch = psi-psiInc : psiInc : psi+psiInc; - polarSearch = theta-thetaInc :thetaInc : theta+thetaInc; - azimuthalSearch= phi-phiInc : phiInc : phi + phiInc; - else - inPlaneSearch = psi-psiInc : psiInc : psi + psiInc; - polarSearch = theta-thetaInc: thetaInc : theta+ thetaInc; - azimuthalSearch= phi-2*phiInc : phiInc : phi + 2*phiInc; - end - - - - - - searchList = zeros(46,3); - nSearch = 1; - for iPhi = azimuthalSearch - for iTheta = polarSearch - for iPsi = inPlaneSearch - searchList(nSearch, :) = [iPhi, iTheta, iPsi]; - nSearch = nSearch + 1; - end - end - end - - - - for iRefine = 1:nSearch-1 - - - RotMat = BH_defineMatrix(searchList(iRefine,:),'Bah', 'forward'); - - - [ rotRef ] = BH_resample3d(ref_FT(:,:,:,iRef),RotMat, ... - peakList(iAngle,8:10), ... - 'Bah', 'GPU', 'forward'); - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - peakList(iAngle,8:10), ... - 'Bah', 'GPU', 'forward')); - else - rotWDG = refWDG; - end - - rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - - - rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef}, volBinary,padCalc,'single'); - rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - - rotRef_FT = conj(rotRef_FT); - rotRef_FT2= conj(rotRef_FT2); - - - % find translational shift using rotationally averaged tightly masked - % reference, to reduce chance of drift to alternate lattice sites. - try - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); - catch - iRef - - - error('sdfsd') - end - - % apply only a tranlational shift to the particle - [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - - rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - [ rotClassImg1 ] = BH_bandLimitCenterNormalize(... - rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,... - padCalc,'single'); -% [ rotClassImg2 ] = BH_bandLimitCenterNormalize(... -% rotClassImg.*peakMask, ... -% bandpassFilt, peakBinary, ... -% padCalc,'single'); - clear rotClassImg - - % now calc CCC, setting sampling shift to zero - [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg, rotWDG, ... - peakMask); - -% [ finalPeakCoord ] = BH_multi_xcf_Translational( ... -% rotClassImg2, rotRef_FT2, ... -% peakMask, peakCOM); - cccStorage(angCount,:) = [iRef, IDX, searchList(iRefine,:), iCCC, ... - iWeight, peakList(iAngle,8:10)+estPeakCoord, ... - phiInc,thetaInc,psiInc]; - - angCount = angCount + 1; - end % end inPlane - end % end search over best peaks - - else - % top ten - % Get unique references for peakList wedge weight normalization - -% referenceList = unique(peakList(:,1)); -% nRefs = length(referenceList); - angCount=1; - - nRefs = size(ref_FT,4); - iClassImg2 = cell(nRefs,1); - for iRef = 1:nRefs - [ iClassImg2{iRef} ] = BH_bandLimitCenterNormalize(iClassTrim.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - end - nAngles = nRefs.*size(peakList,1).*length(IN_PLANE_SEARCH); - cccStorage = zeros(nAngles, 10, 'double', 'gpuArray'); - - for iAngle = 1:size(peakList,1) - % The assumption is the best reference for the axially averaged is - % also the best ref otherwise. Maybe not true. - %iRef = peakList(iAngle, 1); - phi = peakList(iAngle, 2); - theta= peakList(iAngle, 3); - - for iInPlane = IN_PLANE_SEARCH - psi = iInPlane; - - evaluateRef = ones(1,nRefs); - if (limitSymmetry) - evaluateRef = evaluateRef.*((abs(psi).*evaluateRef) < 180 ./ refSym); - end - for iRef = 1:nRefs - if (evaluateRef(iRef)) - RotMat = BH_defineMatrix([phi, theta, psi - phi],'Bah', 'forward'); - - [ rotRef ] = BH_resample3d(ref_FT(:,:,:,iRef), ... - RotMat,peakList(iAngle,4:6), ... - 'Bah', 'GPU', 'forward'); - - - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - peakList(iAngle,4:6), ... - 'Bah', 'GPU', 'forward')); - else - rotWDG = refWDG; - end - - rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - - - rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef} , volBinary,padCalc,'single'); - rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef} , peakBinary,padCalc,'single'); - - rotRef_FT = conj(rotRef_FT); - rotRef_FT2= conj(rotRef_FT2); - - - - - - % find translational shift - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); - - % apply only a tranlational shift to the particle - [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - - rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - [ rotClassImg1 ] = BH_bandLimitCenterNormalize(rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,padCalc,'single'); - % [ rotClassImg2 ] = BH_bandLimitCenterNormalize(rotClassImg.*peakMask, ... - % bandpassFilt{iRef} , peakBinary,padCalc,'single'); - clear rotClassImg - - - % now calc CCC, setting sampling shift to zero - [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg, rotWDG,... - peakMask); - - % [ finalPeakCoord ] = BH_multi_xcf_Translational( ... - % rotClassImg2, rotRef_FT2, ... - % peakMask, peakCOM); - - - cccStorage(angCount,:) = [iRef, IDX, phi, theta, psi - phi, iCCC, ... - iWeight, estPeakCoord + ... - peakList(iAngle,4:6) ]; - angCount = angCount + 1; - end - end - end % end inPlane - end % end search over best peaks - end - -else % search angles based on grideSearchAngles - - referenceList = 1:size(ref_FT,4); - nRefs = length(referenceList); - iClassImg2 = cell(nRefs,1); - - - for iRef = 1:nRefs - - [ iClassImg2{iRef} ] = BH_bandLimitCenterNormalize(iClassTrim.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - end - angCount=1; - nAngles = nRefs.*sum(ANGLE_STEP(:,2)+1).*length(IN_PLANE_SEARCH); - cccStorage = zeros(nAngles, 10, 'double', 'gpuArray'); - for iAngle = 1:size(ANGLE_STEP,1) - - theta = ANGLE_STEP(iAngle,1); - - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. - - phiStep = ANGLE_STEP(iAngle,3); - - % To prevent only searching the same increments each time in a limited - % grid search, radomly offset the azimuthal angle by a random number - % between 0 and 1/2 the azimuthal increment. - azimuthalRandomizer = rand(1)*phiStep/2; - - for iAzimuth = 0:ANGLE_STEP(iAngle,2) - phi = rem(phiStep * (iAzimuth + azimuthalRandomizer),360) ; - - % For axially averaged this is always zero. - for iInPlane = IN_PLANE_SEARCH - psi = iInPlane; - [phi,theta,psi]; - - for iRef = 1:nRefs - - - - RotMat = BH_defineMatrix([phi, theta, psi ],'Bah', 'forward'); - - - [ rotRef ] = BH_resample3d(refRotAvg_FT(:,:,:,iRef),RotMat, ... - [0,0,0], 'Bah', 'GPU', 'forward'); - - - - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - [0,0,0], ... - 'Bah', 'GPU', 'forward')); - else - rotWDG = refWDG; - end - - rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - try - rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef} , volBinary,padCalc,'single'); - rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef} , peakBinary,padCalc,'single'); - catch - size(volMask) - size(peakMask) - size(bandpassFilt{iRef} ) - size(rotRef) - error('size mismatch in multi_angular search') - end - rotRef_FT = conj(rotRef_FT); - rotRef_FT2= conj(rotRef_FT2); - - - % find translational shift - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); - - % apply only a tranlational shift to the particle - [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - - rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - [ rotClassImg1 ] = BH_bandLimitCenterNormalize(rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,padCalc,'single'); -% [ rotClassImg2 ] = BH_bandLimitCenterNormalize(rotClassImg.*peakMask, ... -% bandpassFilt{iRef} , peakBinary,padCalc,'single'); - clear rotClassImg - - - % now calc CCC, setting sampling shift to zero - [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg,rotWDG, ... - peakMask); - -% [ finalPeakCoord ] = BH_multi_xcf_Translational( ... -% rotClassImg2, rotRef_FT2, ... -% peakMask, peakCOM); -% - - cccStorage(angCount,:) = [iRef, IDX, phi, theta, psi, iCCC, ... - iWeight, estPeakCoord]; - - angCount = angCount + 1; - end % peak search over refs - - - end % end inPlane - end % azimuthal - - end % polar - - % end of else clause, which is a search over axially averaged reference. -end - -%[ cccStorage ] = BH_multi_peakSearch(referenceList, cccStorage); - cccStorage = cccStorage(( ~(sum(isnan(cccStorage),2)) ),:); - cccStorage = cccStorage(( cccStorage(:,6) ~= 0 ),:); - - CCC_STORAGE = sortrows(gather(cccStorage),-6); - - clear rotRef rotWDG refWDG rotRef_FT rotRef_FT2 clear rotClassImg1 iClassImg2 -end % end angularSearch function - diff --git a/coordinates/BH_multi_calcBinShift.m b/coordinates/BH_multi_calcBinShift.m index a8026029..66feccdb 100644 --- a/coordinates/BH_multi_calcBinShift.m +++ b/coordinates/BH_multi_calcBinShift.m @@ -1,4 +1,4 @@ -function [ binSize, binShift ] = BH_multi_calcBinShift(coords, isTilt, samplingRate) +function [ binSize, binShift ] = BH_multi_calcBinShift(coords, samplingRate, force_odd_dimension) % Address fractional shifts on binning % Coordinates are stored relative to the lower left corner of the full % tilt/tomo. On binning a shift is needed to keep that origin in the same @@ -7,24 +7,28 @@ % way to do this is to shift the data on binning in 2d, and leave the % coordinates alone. -if (isTilt) % Expecting just the x,y,z for a tilt series and the binning. Also may - % shift to have an odd dimension so that Imod origin is always the same. - binSize = floor(coords./samplingRate); + % shift to have an odd dimension so that Imod origin is always the same. + binSize = floor(coords./samplingRate); + if (force_odd_dimension) binSize = binSize - (1-mod(binSize,2)); - - originFull = floor(coords ./2) + 1; - originBin = floor(binSize./2) + 1; - % This is the shift we need to apply to the binned image to make sure - % that the origin is in the same place. - binShift = -1.*(samplingRate.*originBin - originFull) ./ samplingRate; + end + + originFull = emc_get_origin_index(coords); + originBin = emc_get_origin_index(binSize); -end + % 1 ++++++^+++^ + % 2 _ _ _ _ _ _ + % 3 ___ ___ ___ + % If the continuous specimen in on line 1 and the unbinned image is sampling that specimen as in line 2 + % Our goal is to have the feature that is on origin 1 (pixel 4 = ^) to be on origin 2 (pixel 2 = ^) + % You know, typing this out makes me think it is unneeded. + + % This is the shift we need to apply to the binned image to make sure + % that the origin is in the same place. + binShift = -1.*(samplingRate.*originBin - originFull) ./ samplingRate; -% tmpShift = (coords-fix(coords./samplingRate).*samplingRate); -% binShiftTemplateSearch = [tmpShift(1), tmpShift(3) + tmpShift(2),tmpShift(4)]; -% binShift = binShiftTemplateSearch ./ samplingRate; end diff --git a/coordinates/BH_multi_calcBinShift.mrc b/coordinates/BH_multi_calcBinShift.mrc deleted file mode 100644 index 01e6d6e8..00000000 --- a/coordinates/BH_multi_calcBinShift.mrc +++ /dev/null @@ -1,10 +0,0 @@ -function [ binShift, binShiftTemplateSearch ] = BH_multi_calcBinShift(coords, samplingRate) -%UNTITLED2 Summary of this function goes here -% Detailed explanation goes here - -tmpShift = (coords-fix(coords./samplingRate).*samplingRate); -binShiftTemplateSearch = [tmpShift(1), tmpShift(3) + tmpShift(2),tmpShift(4)]; -binShift = binShiftTemplateSearch ./ samplingRate; - -end - diff --git a/coordinates/BH_multi_gridCoordinates.m b/coordinates/BH_multi_gridCoordinates.m index f1fa6aaa..d966394c 100755 --- a/coordinates/BH_multi_gridCoordinates.m +++ b/coordinates/BH_multi_gridCoordinates.m @@ -1,12 +1,12 @@ function [ Gc1,Gc2,Gc3,g1,g2,g3 ] = BH_multi_gridCoordinates( SIZE, SYSTEM, METHOD, ... - TRANSFORMATION, ... - flgFreqSpace, ... - flgShiftOrigin, flgRad, ... - varargin) + TRANSFORMATION, ... + flgFreqSpace, ... + flgShiftOrigin, flgRad, ... + varargin) %Return grid vectors in R3 for various coordinate systems. % Create grid vectors of dimension SIZE, that are either Cartesian, % Cylindrical, or Spherical. Optionally only return a matrix with radial -% values. Grids are centered with the origin at ceil((N+1)/2). +% values. Grids are centered with the origin at ceil((N+1)/2). if strcmpi(METHOD,'GPU') SIZE = gpuArray(single(SIZE)); else @@ -34,19 +34,19 @@ end end else - if strcmpi(varargin{1}{1},'halfGrid') - doFullGrid = 0; - else - error('1st varargin to grid coords is not numeric or (halfGrid bool) not understood'); - end + if strcmpi(varargin{1}{1},'halfGrid') + doFullGrid = 0; + else + error('1st varargin to grid coords is not numeric or (halfGrid bool) not understood'); + end end end - + if numel(SIZE) == 3 - sX = SIZE(1) ; sY = SIZE(2) ; sZ = SIZE(3); + sX = SIZE(1) ; sY = SIZE(2) ; sZ = SIZE(3); flg3D = 1; elseif numel(SIZE) == 2 - sX = SIZE(1) ; sY = SIZE(2) ; + sX = SIZE(1) ; sY = SIZE(2) ; if strcmpi(METHOD,'GPU'); sZ = gpuArray(single(1)); else sZ = single(1);end flg3D = 0; else @@ -61,7 +61,7 @@ % the origin in IMODs case for an even image is -0.5 relative to mine. % Switching to force odd size - 20171201 conventionShift = [0,0,0]; -% conventionShift = flgShiftOrigin(2:4) .* (1-mod([sX,sY,sZ],2)); + % conventionShift = flgShiftOrigin(2:4) .* (1-mod([sX,sY,sZ],2)); flgShiftOrigin = flgShiftOrigin(1); else conventionShift = [0,0,0]; @@ -75,7 +75,7 @@ symIDX = 0; flgMask = 0; -if iscell(TRANSFORMATION) +if iscell(TRANSFORMATION) switch TRANSFORMATION{1} @@ -83,7 +83,7 @@ flgTrans = 0; R = [1,0,0;0,1,0;0,0,1]; dXYZ = [0,0,0]'; - DIR = 'forwardVector'; + DIR = 'fwdVector'; MAG = {1}; % The majority of function calls that are not in a resample/rescale % program call this case, and don't expect a cell output. @@ -93,16 +93,16 @@ flgGridVectors = 1; R = [1,0,0;0,1,0;0,0,1]; dXYZ = TRANSFORMATION{3}; - DIR = 'forwardVector'; + DIR = 'fwdVector'; MAG = {TRANSFORMATION{6}}; - case 'single' + case 'single' if numel(TRANSFORMATION{2}) == 9 R = reshape(TRANSFORMATION{2},3,3); else R = reshape(TRANSFORMATION{2},2,2); end - + dXYZ = TRANSFORMATION{3}; if length(dXYZ) == 2 dXYZ = [dXYZ;0]; @@ -113,14 +113,14 @@ symInc = 360 / flgSymmetry; symIDX = 0:flgSymmetry-1; Gc1 = cell(flgSymmetry,1); - Gc2 = cell(flgSymmetry,1); + Gc2 = cell(flgSymmetry,1); Gc3 = cell(flgSymmetry,1); else symIDX = 1; symInc = 0; end - if strcmpi(DIR, 'inv') || strcmpi(DIR,'forwardVector') + if strcmpi(DIR, 'inv') || strcmpi(DIR,'fwdVector') MAG = {TRANSFORMATION{6}}; else MAG = {1./TRANSFORMATION{6}}; % faster to just do A(I) but left as {{}} for clarity @@ -133,8 +133,8 @@ end end - - + + case 'sequential' flgSequential = 1; @@ -151,83 +151,83 @@ for iTrans = 1:nTrans R_seq{iTrans} = reshape(TRANSFORMATION{iTrans,2},3,3); - + dXYZ_seq{iTrans} = TRANSFORMATION{iTrans,3}; - - + + % Convention is only forward so np need to consider flipping MAG_seq{iTrans} = TRANSFORMATION{iTrans,6}; - + end % For now assuming no symmetry operation on sequential transformations flgSymmetry = TRANSFORMATION{1,5}; DIR = TRANSFORMATION{1,4}; - + symInc = 360 / flgSymmetry; symIDX = 0:flgSymmetry-1; Gc1 = {}; Gc2 = {}; Gc3 ={}; - + otherwise error(['TRANSFORMATION must be a cell,',... - '(none,gridVectors,single,sequential),',... - 'Rotmat, dXYZ, forward|inv, symmetry\n']); + '(none,gridVectors,single,sequential),',... + 'Rotmat, dXYZ, forward|inv, symmetry\n']); end end if ( makeVectors ) - % sX = gpuArray(sX) ; sY = gpuArray(sY) ; sZ = gpuArray(sZ); - if flgShiftOrigin == 1 - if (doFullGrid) - x1 = [-1*floor((sX)/2):floor((sX-1)/2)]; - else - x1 = [0:floor((sX)/2)]; - end - y1 = [-1*floor((sY)/2):floor((sY-1)/2)]; - if (flg3D); z1 = [-1*floor((sZ)/2):floor((sZ-1)/2)]; end - - elseif flgShiftOrigin == -1 - if (doFullGrid) - x1 = [1:sX]; - else - x1 = 1:ceil((sX+1)/2); - end - y1 = [1:sY]; - if (flg3D); z1 = [1:sZ]; end - - elseif flgShiftOrigin == -2 - if (doFullGrid) - x1 = fftshift([1:sX]); - else - x1 = fftshift([1:ceil((sX+1)/2);]); - end - y1 = fftshift([1:sY]); - if (flg3D); z1 = fftshift([1:sZ]); end + % sX = gpuArray(sX) ; sY = gpuArray(sY) ; sZ = gpuArray(sZ); + if flgShiftOrigin == 1 + if (doFullGrid) + x1 = [-1*floor((sX)/2):floor((sX-1)/2)]; else - if (doFullGrid) - x1 = [0:floor(sX/2),-1*floor((sX-1)/2):-1]; - else - x1 = [0:floor(sX/2)]; - end - y1 = [0:floor(sY/2),-1*floor((sY-1)/2):-1]; - if (flg3D); z1 = [0:floor(sZ/2),-1*floor((sZ-1)/2):-1]; end - end - + x1 = [0:floor((sX)/2)]; + end + y1 = [-1*floor((sY)/2):floor((sY-1)/2)]; + if (flg3D); z1 = [-1*floor((sZ)/2):floor((sZ-1)/2)]; end + + elseif flgShiftOrigin == -1 + if (doFullGrid) + x1 = [1:sX]; + else + x1 = 1:ceil((sX+1)/2); + end + y1 = [1:sY]; + if (flg3D); z1 = [1:sZ]; end + + elseif flgShiftOrigin == -2 + if (doFullGrid) + x1 = fftshift([1:sX]); + else + x1 = fftshift([1:ceil((sX+1)/2);]); + end + y1 = fftshift([1:sY]); + if (flg3D); z1 = fftshift([1:sZ]); end + else + if (doFullGrid) + x1 = [0:floor(sX/2),-1*floor((sX-1)/2):-1]; + else + x1 = [0:floor(sX/2)]; + end + y1 = [0:floor(sY/2),-1*floor((sY-1)/2):-1]; + if (flg3D); z1 = [0:floor(sZ/2),-1*floor((sZ-1)/2):-1]; end + end + if strcmpi(METHOD, 'GPU') x1 = gpuArray(x1); y1 = gpuArray(y1); if (flg3D); z1 = gpuArray(z1); end end end - + % Make any needed shifts for convention x1 = x1 - conventionShift(1); y1 = y1 - conventionShift(2); if (flg3D); z1 = z1 - conventionShift(3); end -if strcmpi(DIR, 'inv') || strcmpi(DIR, 'forwardVector') +if strcmpi(DIR, 'inv') || strcmpi(DIR, 'fwdVector') x1 = x1 - dXYZ(1); y1 = y1 - dXYZ(2); if (flg3D); z1 = z1 - dXYZ(3); end @@ -255,7 +255,7 @@ return end -% Rescale the vectors prior to making gridVectors +% Rescale the vectors prior to making gridVectors if (flgSequential) x1 = x1.*MAG_seq{1}; y1 = y1.*MAG_seq{1}; @@ -263,7 +263,7 @@ else x1 = x1.*MAG{1}; y1 = y1.*MAG{1}; - z1 = z1.*MAG{1}; + z1 = z1.*MAG{1}; end % No matter the case, the cartesian grids are needed @@ -271,7 +271,7 @@ % Optionally evaluate only a smaller masked region -if (flgMask) +if (flgMask) X = X(binaryVol); Y = Y(binaryVol); Z = Z(binaryVol); @@ -281,19 +281,19 @@ for iTrans = 1:1+(flgSequential) if (flgSequential) - rAsym = R_seq{iTrans}; + rAsym = R_seq{iTrans}; if (iTrans == 1) dXyzAsym = 0; % Instead of shifting then shifting back, just note the original shift else - % adding the R2' because the first term doesn't need to be multiplied by - % R2 (in the first action under symmetry loop) but making a change here - % which involves extra multiplications is okay, since this function is - % used much less than 'single' style resampling. - dXyzAsym = ( R_seq{iTrans}'*R_seq{iTrans-1} * ... - dXYZ_seq{iTrans-1}.*MAG_seq{iTrans-1} + ... - R_seq{iTrans-1} * dXYZ_seq{iTrans}.*MAG_seq{iTrans} ); - + % adding the R2' because the first term doesn't need to be multiplied by + % R2 (in the first action under symmetry loop) but making a change here + % which involves extra multiplications is okay, since this function is + % used much less than 'single' style resampling. + dXyzAsym = ( R_seq{iTrans}'*R_seq{iTrans-1} * ... + dXYZ_seq{iTrans-1}.*MAG_seq{iTrans-1} + ... + R_seq{iTrans-1} * dXYZ_seq{iTrans}.*MAG_seq{iTrans} ); + X = Gc1{1}.*MAG_seq{iTrans}; Y = Gc2{1}.*MAG_seq{iTrans}; Z = Gc3{1}.*MAG_seq{iTrans}; @@ -303,18 +303,18 @@ % Note that if symmetric, dXYZ changes each loop after this point dXyzAsym = dXYZ.*MAG{1}; end - + for iSym = symIDX - + % Only in plane symmetries considered anywhere so inv|forward shouldn't % matter. - + R = rAsym * BH_defineMatrix([iSym.*symInc,0,0],'Bah','inv'); - + % Any forward transformations of the grids if (flgTrans) dXYZ = shiftDir .* R*dXyzAsym; - + Xnew = X.*R(1) + Y.*R(4) + Z.*R(7) - dXYZ(1); Ynew = X.*R(2) + Y.*R(5) + Z.*R(8) - dXYZ(2); if (flg3D) @@ -325,13 +325,13 @@ else Xnew = X; Ynew = Y ; Znew = Z; end - + % Only return the radial grid if requested if (flgRad) G1 = sqrt(Xnew.^2 + Ynew.^2 + Znew.^2); G2 = ''; G3 = ''; - + else switch SYSTEM case 'Cartesian' @@ -343,22 +343,22 @@ G2(G2 < 0) = G2(G2 < 0) + 2.*pi; % [0,pi] G3 = acos(Z./G1); - + case 'Cylindrical' - + G1 = sqrt(Xnew.^2 + Ynew.^2); G2 = atan2(Ynew,Xnew); % set from [-pi,pi] --> [0,2pi] G2(G2 < 0) = G2(G2 < 0) + 2.*pi; G3 = Znew; - + otherwise error('SYSTEM must be Cartesian, Spherical, Cylindrical') - end + end end % Only use as cell if symmetry is requested if (flgSymmetry) - % + % Gc1{iSym+1} = G1; Gc2{iSym+1} = G2; @@ -371,7 +371,7 @@ end % loop over symmetric transformations end % loop over sequential transformations - + clear X Y Z Xnew Ynew Znew x1 y1 z1 diff --git a/coordinates/BH_multi_gridSearchAngles.m b/coordinates/BH_multi_gridSearchAngles.m index da635215..1603a578 100755 --- a/coordinates/BH_multi_gridSearchAngles.m +++ b/coordinates/BH_multi_gridSearchAngles.m @@ -1,7 +1,7 @@ function [ nIN_PLANE, IN_PLANE_SEARCH, angleStep, nAngles ] = ... - BH_multi_gridSearchAngles( ANGLE_SEARCH) + BH_multi_gridSearchAngles( ANGLE_SEARCH) %Consolodating function, calculate angular sampling. -% +% % % Called by: % @@ -14,7 +14,7 @@ % BH_alignRaw3d - % doesn't use: nIN_PLANE, ANGLE_INCREMENT % -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % @@ -24,7 +24,7 @@ % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % -% TODO: +% TODO: % - handle helical grid search % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -35,7 +35,7 @@ % For monolayer which can generally be at 0 or 180 +/- create search option % around these two, indicated by a negative value for the out of plane range -if (OUT_OF_PLANE(1) < 0) +if (OUT_OF_PLANE(1) < 0) OUT_OF_PLANE(1)= abs(OUT_OF_PLANE(1)); biPolarSearch = 1; else @@ -49,7 +49,7 @@ symmetryConstrainedSearch = 0; end - + if all(IN_PLANE) @@ -68,14 +68,14 @@ nIN_PLANE = length(IN_PLANE_SEARCH); angleStep(1,:) = [0,0,0,0,psiStep]; else -IN_PLANE_SEARCH = 0; + IN_PLANE_SEARCH = 0; if (symmetryConstrainedSearch) for iSym = 1:symmetryConstrainedSearch-1 IN_PLANE_SEARCH = [IN_PLANE_SEARCH,IN_PLANE_SEARCH + iSym.*(360/symmetryConstrainedSearch)]; end - + end - + psiStep = 0; nIN_PLANE = 1; % Always search the unrotated sample angleStep(1,:) = [0,0,0,0,0]; @@ -89,7 +89,7 @@ else topPolar = OUT_OF_PLANE(1)/OUT_OF_PLANE(2); thetaStep = OUT_OF_PLANE(2); - end + end if (biPolarSearch) polarAngles = 0:OUT_OF_PLANE(2):OUT_OF_PLANE(1);%(0:topPolar).*thetaStep; @@ -109,23 +109,23 @@ if (theta - thetaStep == 0) || (theta + thetaStep == 180) % strict even spacing leaves the first out of plane undersampled phiStep = 0.5 * phiStep; - + end - + nAzimuthal = floor(360/phiStep); - - + + end % first position is psiStep independent of this. if (iPolarAngle) angleStep(iPolarAngle,:) = [theta, nAzimuthal, ... - phiStep, thetaStep, psiStep]; + phiStep, thetaStep, psiStep]; else angleStep(iPolarAngle,:) = [theta, nAzimuthal, ... - phiStep, 0, psiStep]; + phiStep, 0, psiStep]; end - end + end else % Setting top polar limits the angular search to at most the in plane angles, % as the azimuth is also zero for iPolarAngle = 1. @@ -136,28 +136,28 @@ % full search, rotationally averaged, first in plane, refinement1, refinement 2. nAngles = zeros(5,1); if sum(any(angleStep)) - + for i = 1:size(angleStep,1) if angleStep(i,1) == 0 % theta = 0 in plane only - nAngles(1) = nAngles(1) + nIN_PLANE; + nAngles(1) = nAngles(1) + nIN_PLANE; else nAngles(1) = nAngles(1) + (angleStep(i,2) .* nIN_PLANE); end - nAngles(2) = nAngles(2) + angleStep(i,2); + nAngles(2) = nAngles(2) + angleStep(i,2); end else -% Translational only search -nAngles = nAngles + 1; + % Translational only search + nAngles = nAngles + 1; end nAngles(3) = 10 .* nIN_PLANE; nAngles(4) = 175; nAngles(5) = 630; - + end % end of gridSearchAngles function diff --git a/coordinates/BH_multi_iterator.m b/coordinates/BH_multi_iterator.m index c300851e..bb8434c3 100755 --- a/coordinates/BH_multi_iterator.m +++ b/coordinates/BH_multi_iterator.m @@ -8,11 +8,11 @@ % nextBest = [64,72,96,108,128,144,160,168,180,192,216,224,256,... % 270,288,300,320,336,360,384,400,432,448,480,512]; nextBest = [64,72,80,84,90,96,108,112,120,126,128,144,160,162, ... - 168,180,192,216,224,240,256,270,288,320,324,336,... - 360,378,384,400,416,432,448,480,486,504,512]; + 168,180,192,216,224,240,256,270,288,320,324,336,... + 360,378,384,400,416,432,448,480,486,504,512]; % fall off between 1 and 0 in masking function APODIZATION = 2.*6; - + switch OPERATION case 'fourier' @@ -27,23 +27,23 @@ sizeTarget= SIZES(1,:); if ( flgDescend ) for i = 1:3 - try - nB(i) = nextBest(find(nextBest <= sizeTarget(i), 1, 'last')); - catch - nB(i) = nextBest(1); - end + try + nB(i) = nextBest(find(nextBest <= sizeTarget(i), 1, 'last')); + catch + nB(i) = nextBest(1); + end end else for i = 1:3 try - nB(i) = nextBest(find(nextBest >= sizeTarget(i), 1, 'first')); + nB(i) = nextBest(find(nextBest >= sizeTarget(i), 1, 'first')); catch nB(i) = nextBest(end); end end end - - + + if ~all(nB) error('next best not found in range 128-512 for [%d,%d,%d]', sizeTarget); end @@ -51,21 +51,21 @@ case 'fourier2d' % Found by -% % % for i = 64:2:3838*2 -% % % if (sum(factor(i).*(factor(i) >= 5))< 10) -% % % f = [f,i]; -% % % end -% % % end + % % % for i = 64:2:3838*2 + % % % if (sum(factor(i).*(factor(i) >= 5))< 10) + % % % f = [f,i]; + % % % end + % % % end nextBest = [64,72,80,84,90,96,108,112,120,126,128,144,160,162, ... - 168,180,192,216,224,240,252,256,270,288,320,324,336,... - 360,378,384,432,448,480,486,504,512,540,576,640,648,... - 720,756,768,810,864,896,960,972,1008,1024,1080,... - 1134,1152,1280,1296,1344,1440,1458,1512,1536,1620,1728,... - 1792,1920,1944,2016,2048,2160,2268,2304,2430,2560,... - 2592,2688,2880,2916,3024,3072,3240,3402,3456,3584,... - 3840,3888,4032,4096,4320,4374,4536,4608,4860,5120,... - 5184,5376,5760,5832,6048,6144,6480,6804,6912,7168,7290,... - 7488,7560,7840,7920,8192]; + 168,180,192,216,224,240,252,256,270,288,320,324,336,... + 360,378,384,432,448,480,486,504,512,540,576,640,648,... + 720,756,768,810,864,896,960,972,1008,1024,1080,... + 1134,1152,1280,1296,1344,1440,1458,1512,1536,1620,1728,... + 1792,1920,1944,2016,2048,2160,2268,2304,2430,2560,... + 2592,2688,2880,2916,3024,3072,3240,3402,3456,3584,... + 3840,3888,4032,4096,4320,4374,4536,4608,4860,5120,... + 5184,5376,5760,5832,6048,6144,6480,6804,6912,7168,7290,... + 7488,7560,7840,7920,8192]; sizeTarget= SIZES(1,:); nB = [0,0]; for i = 1:2 @@ -75,7 +75,7 @@ nB(i) = 0; end end - + if ~all(nB) % For some reason "error" only takes scalar args for print formating. fprintf('next best not found in range 64-7290 for [%d,%d,%d]\n',target); @@ -86,77 +86,99 @@ case 'convolution' % This could be optimized automatically to balance increased target size vs % number of iterations/post padding. - + % the target size, to get the most out of calcs, spend as much time on the % gpu as possible. With finer angular searches, the number of references % needs more memory, so a smaller size here means more transfers, but this % should be balanced by the finer angles (more comp) - if all(SIZES(1,:) == 256) - nextBest = [128,144,160,168,192,216,224,256]; - elseif all(SIZES(1,:) == 384) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384]; - elseif all(SIZES(1,:) == 432) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432]; - elseif all(SIZES(1,:) == 512) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432,480,512]; - elseif all(SIZES(1,:) > 512) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432,480,512,... - 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]; - end + % if all(SIZES(1,:) <= 256) + % nextBest = [128,144,160,168,192,216,224,256]; + % elseif all(SIZES(1,:) <= 384) + % nextBest = [128,144,160,168,192,216,224,256,... + % 288,300,320,336,360,384]; + % elseif all(SIZES(1,:) <= 432) + % nextBest = [128,144,160,168,192,216,224,256,... + % 288,300,320,336,360,384,400,432]; + % elseif all(SIZES(1,:) <= 512) + % nextBest = [128,144,160,168,192,216,224,256,... + % 288,300,320,336,360,384,400,432,480,512]; + % else + % nextBest = [128,144,160,168,192,216,224,256,... + % 288,300,320,336,360,384,400,432,480,512,... + % 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]; + % end + + nextBest = repmat([128,144,160,168,192,216,224,256,... + 288,300,320,336,360,384,400,432,480,512,... + 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]',1,3); + + nextBest(nextBest(:,1) > SIZES(1,1),1) = 16; + nextBest(nextBest(:,2) > SIZES(1,2),2) = 16; + nextBest(nextBest(:,3) > SIZES(1,3),3) = 16; sizeImage = SIZES(2,:); sizeTemplate = SIZES(3,:); % the mask or kernel - sizeParticle = SIZES(4,:) ;% a subregion of sizeTemplate - - + sizeParticle = SIZES(4,:);% a subregion of sizeTemplate + + borderSizeCalc = floor((sizeTemplate + APODIZATION)./2); borderSizeKeep = borderSizeCalc + 2.*sizeParticle; + - abs(sum(sizeImage - sizeTemplate)) - sum(0.1.*sizeImage) if abs(sum(sizeImage - sizeTemplate)) < sum(0.1.*sizeImage) OUTPUT = [[0,0,0];[0,0,0] ;sizeImage; ... - sizeImage; sizeImage ; [1,1,1]]; + sizeImage; sizeImage ; [1,1,1]]; return end - score = zeros(length(nextBest),6); + score = zeros(size(nextBest,1),10); - - validCalc = repmat(nextBest',1,3) - 2.*repmat(borderSizeCalc,length(nextBest),1); - validKeep = repmat(nextBest',1,3) - 2.*repmat(borderSizeKeep,length(nextBest),1); - minIter= floor(repmat(sizeImage,length(nextBest),1)./validKeep); - postPad= repmat(nextBest',1,3)- ... - (repmat(sizeImage+borderSizeKeep,length(nextBest),1) -minIter.*(validKeep+1)); - + + validCalc = nextBest - 2.*repmat(borderSizeCalc,size(nextBest,1),1); + validKeep = nextBest - 2.*repmat(borderSizeKeep,size(nextBest,1),1); + minIter= floor(repmat(sizeImage,size(nextBest,1),1)./validKeep); + postPad= nextBest- ... + (repmat(sizeImage+borderSizeKeep,size(nextBest,1),1) -minIter.*(validKeep+1)); + + % Penalize large Z % Added this so I can work with test cases where the volume to be % searched is the same size as the reference + score(:,5:7) = minIter; + score(:,2:4) = nextBest ./ postPad .* (minIter >= 0); + adjustForIterations = minIter + 1; + adjustForIterations = (adjustForIterations); + adjustForIterations(~isfinite(adjustForIterations)) = 1; + score(:,2:4) = score(:,2:4) ./ adjustForIterations; + score(:,1) = sum(score(:,2:4),2); + score + % Testing out the best overall score now + [~, cX] = max(score(:,2)); + [~, cY] = max(score(:,3)); + [~, cZ] = max(score(:,4)); + + chunkSize = [nextBest(cX,1), nextBest(cY,2), nextBest(cZ,3)]; + + % [ ~, best_score ] = max(score(:,1)); + % chunkSize = nextBest(best_score.*[1,1,1]); + - score(:,2:4) = repmat(nextBest',1,3) ./ postPad .* (minIter >= 0) - - [~, cX] = max(score(:,2)) ; - [~, cY] = max(score(:,3)) ; - [~, cZ] = max(score(:,4)) ; - - chunkSize = nextBest([cX,cY,cZ]); - - cY = cY + length(nextBest); - cZ = cZ + 2.*length(nextBest); + % cX = best_score; + % cY = best_score; + % cZ = best_score; + % TODO: review what is going on here + cY = cY + size(nextBest,1); + cZ = cZ + 2.*size(nextBest,1); validAreaKeep = validKeep([cX,cY,cZ]); validAreaCalc = validCalc([cX,cY,cZ]); nIters = minIter([cX,cY,cZ])+1; postPAD= postPad([cX,cY,cZ]); - + OUTPUT = [borderSizeKeep;postPAD ;chunkSize; ... - validAreaKeep; validAreaCalc ; nIters]; + validAreaKeep; validAreaCalc ; nIters]; case 'binning' - + binFactor = SIZES(1,:); sizeImage = SIZES(2,:); @@ -185,7 +207,7 @@ end end end - + OUTPUT(i) = nextBest(fftVal); end @@ -210,12 +232,12 @@ end end end - + OUTPUT = [finalVal; padVal]; otherwise error('OPERATION is case-sensitive fourier, convolution or binning, not %s', OPERATION); end - + end diff --git a/coordinates/BH_multi_recGeom.m b/coordinates/BH_multi_recGeom.m index a77fb0e9..c7665a01 100755 --- a/coordinates/BH_multi_recGeom.m +++ b/coordinates/BH_multi_recGeom.m @@ -1,31 +1,114 @@ -function [ recGeom, tiltName, nTomos] = BH_multi_recGeom( reconCoordName ) +function [ recGeom, tiltName, tomoList, tilt_geometry ] = BH_multi_recGeom( reconCoordName, mapBackIter ) %UNTITLED Summary of this function goes here % Detailed explanation goes here -% could use import data in later versions of matlab (it wasn't working in -% the compiled binaries with < 15a for some reason.) -recFile = fopen(reconCoordName,'r'); -tiltName = textscan(recFile,'%s',1) ; -tiltName = tiltName{1}{1}; -nTomos = textscan(recFile,'%d',1); nTomos = nTomos{1}; -recCoords = textscan(recFile,'%f'); -fclose(recFile); - -%%% Some sanity checks -% First line should be the name of the tilt-series the tomo is -% reconstructed from. +% File format: +% 1. Name of the tilt-series the tomo is reconstructed from +% 2. Number of tomograms +% 3. For each tomogram, numbered sequentially: +% 1. NX +% 2. NY start +% 3. NY end +% 4. NZ +% 5. X shift +% 6. Z shift +recFile = importdata(reconCoordName); +tiltName = recFile.textdata{1}; +nTomos = recFile.data(1); +recCoords = recFile.data(2:end); + +if (nTomos == 0) + fprintf("WARNING: No tomograms found in %s\n", reconCoordName); + error('The number of tomograms is zero'); +end + +tilt_geometry_name = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', tiltName, mapBackIter+1); +try + tilt_geometry = load(tilt_geometry_name); +catch + error('Could not load the tilt geometry file: %s', tilt_geometry_name); +end + +% Check that we have a multiple of 6 entries. This should have been caught in recSCript2.sh +if mod(numel(recCoords),6) ~= 0 + error('The number of entries in the reconCoord file is not a multiple of 6'); +end + + +%%% Some sanity checks +% First line should be the name of the tilt-series the tomo is reconstructed from. [~,tiltNameFromTomo,~] = fileparts(reconCoordName); tiltStr = strsplit(tiltNameFromTomo,'_'); tiltNameFromTomo = strjoin(tiltStr(1:end-1),'_'); if ~strcmp(tiltNameFromTomo, tiltName) error('the tomo base name (%s) does not match the tiltName in the coords file (%s)',tiltNameFromTomo,tiltName); end +% The tilt_geometry should have rows that are a multiple of 26 (> 26 means more than one orientation per peak) +if mod(size(tilt_geometry,2),26) ~= 0 && mod(size(tilt_geometry,2),23) ~= 0 + tilt_geometry + error('The tilt geometry file does not have a multiple of 26 entries/row'); +end + +% This includes all possible tomos from a given tilt-series when defined. +% Tomos may be ignored when cleaning template matching results, or later if set to be ignored +% in geometryAnalysis or if there are zero sub-tomos left. +recGeom = cell(nTomos,1); +tiltList = cell(nTomos,1); +for iTomo = 1:nTomos + read_in_Coords = recCoords(1 + (iTomo-1)*6: 6 + (iTomo-1)*6); + % This is not necessarily correct, e.,g you could have 4 bin10 tomos from one tilt, and not keep any model points + % from tomo 3, then you would have tomos 1,2,4 but here we are using 1:3. + tomoName = sprintf('%s_%d',tiltName, iTomo); + + % Check to make sure no out of bounds conditions were created in X Y + % when the user created the model or point file + % TODO: only checking Y b/c that results in a crash. Checking X would probably make sense too. + if read_in_Coords(2) < -75 + error(sprintf('Out of bounds condition for %s yMin at %f, please change recon.txt recon.coords', tomoName, read_in_Coords(2))) + elseif read_in_Coords(2) < 1 + % If not too extreme, just clamp it to 1 + read_in_Coords(2) = 1; + end + yMax = tilt_geometry(1,21); + if read_in_Coords(3) > yMax + 75 + error(sprintf('Out of bounds condition for %s ymin at %f ymax at %f, please change recon.txt recon.coords', tomoName, read_in_Coords(3), yMax)) + elseif read_in_Coords(3) > yMax + read_in_Coords(3) = yMax; + end + -recGeom = zeros(nTomos,6); -for iSt = 1:nTomos - recGeom(iSt,:) = recCoords{1}(1 + (iSt-1)*6: 6 + (iSt-1)*6); + tomoCoords = struct(); + tomoCoords.('is_active') = true; + tomoCoords.('NX') = read_in_Coords(1); + tomoCoords.('NY') = read_in_Coords(2); + tomoCoords.('NZ') = read_in_Coords(3); + tomoCoords.('dX_specimen_to_tomo') = read_in_Coords(4); + tomoCoords.('dY_specimen_to_tomo') = read_in_Coords(5); + tomoCoords.('dZ_specimen_to_tomo') = read_in_Coords(6); + tomoCoords.('tilt_NX') = tilt_geometry(1,20); + tomoCoords.('tilt_NY') = tilt_geometry(1,21); + y_i = floor(emc_get_origin_index(tilt_geometry(1,21)) + tomoCoords.('dY_specimen_to_tomo') - tomoCoords.('NY') ./ 2); + y_f = y_i + floor(tomoCoords.('NY')) - 1; + tomoCoords.('y_i') = y_i; + tomoCoords.('y_f') = y_f; + + % Check that NX, NY, NZ are all positive + if tomoCoords.('NX') <= 0 + error('NX is not positive for %s', tomoName); + end + if tomoCoords.('NY') <= 0 + error('NY is not positive for %s', tomoName); + end + if tomoCoords.('NZ') <= 0 + error('NZ is not positive for %s', tomoName); + end + + recGeom{iTomo} = tomoCoords; + tomoList{iTomo} = tomoName; end - + + + % Note that the x/z shifts (col 5,6) are shifts given to IMOD, which are the opposite of the location of the origin (relative to the center) % To make it more confusing, since the reconstruction is done in a ref frame rotated about X, the Z shift is flipped so it matches the origin in Z end diff --git a/coordinates/EMC_coordGrids.m b/coordinates/EMC_coordGrids.m index 8e40ac64..0bc8a468 100644 --- a/coordinates/EMC_coordGrids.m +++ b/coordinates/EMC_coordGrids.m @@ -104,7 +104,7 @@ gY(gY < 0) = gY(gY < 0) + 2.*pi; % set from [-pi,pi] to [0,2pi] gZ = nan; else - error('EMC:SYSTEM', "SYSTEM should be 'cartesian', 'spherical', 'cylindrical' or 'radial'") + error('EMC:SYSTEM', "SYSTEM should be 'cartesian', 'spherical', 'cylindrical' or 'radial'") end end diff --git a/coordinates/EMC_coordTransform.m b/coordinates/EMC_coordTransform.m index 8cf1d364..92cb6902 100755 --- a/coordinates/EMC_coordTransform.m +++ b/coordinates/EMC_coordTransform.m @@ -46,7 +46,7 @@ % default = 1 % % -> 'offset' (vector): [x, y, z] or [x, y] offset to apply (should correspond to SIZE). -% Offsets are used to adjust the center of rotation defined by 'origin'. +% Offsets are used to adjust the center of rotation defined by 'origin'. % NOTE: this effectively apply a shift on both the vectors and the grids. % NOTE: if there is no rotation or scaling to apply, this has no effect % on the final interpolated image. @@ -118,7 +118,7 @@ if ~isempty(varargin) if length(varargin) ~= 3 error('EMC:varargin', ... - 'varargin should contain 3 row vectors, got %s elements', length(varargin)) + 'varargin should contain 3 row vectors, got %s elements', length(varargin)) elseif ~flg.is3d if ~isscalar(varargin{3}) && ~isnan(varargin{3}) error('EMC:varargin', 'For a 2d case, vZ should be NaN') @@ -128,10 +128,10 @@ end else vX = EMC_setMethod(cast(varargin{1}, OPTION.precision), METHOD); - vY = EMC_setMethod(cast(varargin{2}, OPTION.precision), METHOD); + vY = EMC_setMethod(cast(varargin{2}, OPTION.precision), METHOD); vZ = EMC_setMethod(cast(varargin{3}, OPTION.precision), METHOD); end - + if ~isnumeric(vX) || ~isrow(vX) || SIZE(1) ~= length(vX) error('EMC:varargin', 'varargin{1} (vX) should be a numeric row vector of %d elements', SIZE(1)) elseif ~isnumeric(vY) || ~isrow(vY) || SIZE(2) ~= length(vY) @@ -139,19 +139,19 @@ elseif flg.is3d && ~isnumeric(vZ) || ~isrow(vZ) || SIZE(3) ~= length(vZ) error('EMC:varargin', 'varargin{3} (vZ) should be a numeric row vector of %d elements', SIZE(3)) end - + % Apply offsets and|or normalize if whished. Note: shifts are not applied to vectors. if (flg.offset) - vX = vX - OPTION.offset(1); - vY = vY - OPTION.offset(2); - if flg.is3d; vZ = vZ - OPTION.offset(3); end + vX = vX - OPTION.offset(1); + vY = vY - OPTION.offset(2); + if flg.is3d; vZ = vZ - OPTION.offset(3); end end if (OPTION.normalize) vX = vX ./ SIZE(1); - vY = vY ./ SIZE(2); + vY = vY ./ SIZE(2); if flg.is3d; vZ = vZ ./ SIZE(3); end end - + else % varargin is empty OPTION = EMC_getOption(OPTION, {'offset', 'origin', 'normalize', 'precision'}, true); [vX, vY, vZ] = EMC_coordVectors(SIZE, METHOD, OPTION, false); @@ -163,7 +163,7 @@ % Compute the grids; Optionally evaluate only a smaller masked region. if flg.is3d [X, Y, Z] = ndgrid(vX, vY, vZ); - if flg.binary + if flg.binary X = X(binaryVol); Y = Y(binaryVol); Z = Z(binaryVol); @@ -196,9 +196,9 @@ % Only in plane symmetries considered anywhere % so inverse|forward shouldn't matter. if iSym > 1 - R = OPTION.rotm * BH_defineMatrix([iSym.*symInc,0,0],'Bah','inverse'); + R = OPTION.rotm * BH_defineMatrix([iSym.*symInc,0,0],'Bah','inverse'); else - R = OPTION.rotm; + R = OPTION.rotm; end if flg.transform || iSym > 1 @@ -218,7 +218,7 @@ YTrans = Y; if flg.is3d; ZTrans = Z; else; ZTrans = nan; end end - + % Only use as cell if symmetry is requested if flg.sym gX{iSym+1} = XTrans; % I [TF] hope this doesn't generate a copy. @@ -240,7 +240,7 @@ [flg.is3d, SIZE, ndim] = EMC_is3d(SIZE); if ~(strcmpi(METHOD, 'gpu') || strcmpi(METHOD, 'cpu')) - if isstring(METHOD) || ischar(METHOD) + if isstring(METHOD) || ischar(METHOD) error('EMC:METHOD', "SYSTEM should be 'gpu' or 'cpu', got %s", METHOD) else error('EMC:METHOD', "SYSTEM should be 'gpu' or 'cpu', got %s", clas(METHOD)) @@ -255,16 +255,16 @@ % Extract optional parameters OPTION = EMC_getOption(OPTION, {'rotm', 'shift', 'mag', 'sym', 'direction', ... - 'origin', 'offset', 'binary', 'normalize', 'precision'}, false); + 'origin', 'offset', 'binary', 'normalize', 'precision'}, false); % rotm if isfield(OPTION, 'rotm') - if ~isnumeric(OPTION.rotm) || ~ismatrix(OPTION.rotm) + if ~isnumeric(OPTION.rotm) || ~ismatrix(OPTION.rotm) error('EMC:rotm', 'rotm should be a %dx%d numeric matrix, got %s', ... - ndim, ndim, class(OPTION.rotm)) + ndim, ndim, class(OPTION.rotm)) elseif numel(OPTION.rotm) == ndim^2 error('EMC:rotm', 'rotm should be a %dx%d numeric matrix, got size:%s', ... - ndim, ndim, mat2str(size(OPTION.rotm))) + ndim, ndim, mat2str(size(OPTION.rotm))) end % Most of the time, it will not be an identity matrix, so don't check and do transformation anyway. flg.transform = true; @@ -276,14 +276,14 @@ if isfield(OPTION, 'shift') if ~isnumeric(OPTION.shift) || ~isvector(OPTION.shift) error('EMC:shift', ... - 'shift should be a vector of float|int, got %s', class(OPTION.shift)) + 'shift should be a vector of float|int, got %s', class(OPTION.shift)) elseif any(isnan(OPTION.shift)) || any(isinf(OPTION.shift)) error('EMC:shift', ... - 'shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) + 'shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) elseif numel(OPTION.shift) ~= ndim error('EMC:shift', ... - 'For a %dd SIZE, shift should be a vector of %d float|int, got %s', ... - ndim, ndim, mat2str(OPTION.shift, 2)) + 'For a %dd SIZE, shift should be a vector of %d float|int, got %s', ... + ndim, ndim, mat2str(OPTION.shift, 2)) elseif any(OPTION.shift) flg.shift = true; end @@ -295,22 +295,22 @@ if isfield(OPTION, 'mag') if ~isnumeric(OPTION.mag) error('EMC:mag', ... - 'mag should be a numeric scalar or vector, got %s', class(OPTION.mag)) + 'mag should be a numeric scalar or vector, got %s', class(OPTION.mag)) elseif isvector(OPTION.mag) if length(OPTION.mag) ~= ndim error('EMC:mag', ... - 'mag should be a vector of %d elements, got %d elements', ndim, length(OPTION.mag)) - elseif any(isnan(OPTION.mag)) || any(isinf(OPTION.mag)) + 'mag should be a vector of %d elements, got %d elements', ndim, length(OPTION.mag)) + elseif any(isnan(OPTION.mag)) || any(isinf(OPTION.mag)) error('EMC:mag', ... - 'mag should not have any nan nor inf, got:%s', mat2str(OPTION.mag)) + 'mag should not have any nan nor inf, got:%s', mat2str(OPTION.mag)) end flg.transform = true; elseif isscalar(OPTION.mag) && ~any(isnan(OPTION.mag)) || ~any(isinf(OPTION.mag)) OPTION.mag = zeros(1, ndim) + OPTION.mag; % isotropic scaling flg.transform = true; else - error('EMC:mag', ... - 'mag should be a numeric scalar or a numeric vector of %d elements', ndim) + error('EMC:mag', ... + 'mag should be a numeric scalar or a numeric vector of %d elements', ndim) end else OPTION.mag = ones(1, ndim); % default @@ -320,7 +320,7 @@ if isfield(OPTION, 'sym') if ~isnumeric(OPTION.sym) || ~isscalar(OPTION.sym) || OPTION.sym < 1 || rem(OPTION.sym, 1) error('EMC:sym', ... - 'sym should be a positive integer') + 'sym should be a positive integer') elseif OPTION.sym ~= 1 flg.sym = true; end @@ -344,7 +344,7 @@ % origin if isfield(OPTION, 'origin') if ~isnumeric(OPTION.origin) || ~isscalar(OPTION.origin) || ... - ~(OPTION.origin == 1 || OPTION.origin == -1 || OPTION.origin == 0 || OPTION.origin == 2) + ~(OPTION.origin == 1 || OPTION.origin == -1 || OPTION.origin == 0 || OPTION.origin == 2) error('EMC:origin', 'origin should be 0, 1, 2, or -1, got %d', OPTION.origin) end else @@ -355,14 +355,14 @@ if isfield(OPTION, 'offset') if ~isnumeric(OPTION.offset) || ~isvector(OPTION.offset) error('EMC:offset', ... - 'offset should be a vector of float|int, got %s', class(OPTION.offset)) + 'offset should be a vector of float|int, got %s', class(OPTION.offset)) elseif any(isnan(OPTION.offset)) || any(isinf(OPTION.offset)) error('EMC:offset', ... - 'offset should not contain NaNs or Inf, got %s', mat2str(OPTION.offset, 2)) + 'offset should not contain NaNs or Inf, got %s', mat2str(OPTION.offset, 2)) elseif numel(OPTION.offset) ~= ndim error('EMC:offset', ... - 'For a %dd SIZE, offset should be a vector of %d float|int, got %s', ... - ndim, ndim, mat2str(OPTION.offset, 2)) + 'For a %dd SIZE, offset should be a vector of %d float|int, got %s', ... + ndim, ndim, mat2str(OPTION.offset, 2)) end else OPTION.offset = zeros(1, ndim); % default diff --git a/coordinates/EMC_coordVectors.m b/coordinates/EMC_coordVectors.m index eca2ce51..5f540173 100644 --- a/coordinates/EMC_coordVectors.m +++ b/coordinates/EMC_coordVectors.m @@ -78,7 +78,7 @@ if isfield(OPTION, 'origin') if ~isscalar(OPTION.origin) || ~isnumeric(OPTION.origin) error('EMC:origin', 'OPTION.origin should be an integer, got %s of size: %s', ... - class(OPTION.origin), mat2str(size(OPTION.origin))) + class(OPTION.origin), mat2str(size(OPTION.origin))) elseif OPTION.origin ~= 1 && OPTION.origin ~= -1 && OPTION.origin ~= 0 && OPTION.origin ~= 2 error('EMC:origin', 'OPTION.origin should be 0, 1, 2, or -1, got %d', OPTION.origin) end @@ -97,17 +97,17 @@ if isfield(OPTION, 'shift') if ~isnumeric(OPTION.shift) || ~isrow(OPTION.shift) error('EMC:shift', ... - 'OPTION.shift should be a row vector of float|int, got %s', class(OPTION.shift)) + 'OPTION.shift should be a row vector of float|int, got %s', class(OPTION.shift)) elseif any(isnan(OPTION.shift)) || any(isinf(OPTION.shift)) error('EMC:shift', ... - 'OPTION.shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) + 'OPTION.shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) elseif numel(OPTION.shift) ~= ndim error('EMC:shift', ... - 'For a %dd SIZE, OPTION.shift should be a vector of %d float|int, got %s', ... - ndim, ndim, mat2str(OPTION.shift, 2)) + 'For a %dd SIZE, OPTION.shift should be a vector of %d float|int, got %s', ... + ndim, ndim, mat2str(OPTION.shift, 2)) elseif (OPTION.half || OPTION.origin == -1) && any(OPTION.shift) error('EMC:shift', ... - 'OPTION.shifts are not allowed with half=true or origin=-1 , got %s', mat2str(OPTION.shift, 2)) + 'OPTION.shifts are not allowed with half=true or origin=-1 , got %s', mat2str(OPTION.shift, 2)) end else OPTION.shift = zeros(1, ndim); % default @@ -131,7 +131,7 @@ if isfield(OPTION, 'precision') if ~(ischar(OPTION.precision) || isstring(OPTION.precision)) || ... - ~strcmpi(OPTION.precision, 'single') && ~strcmpi(OPTION.precision, 'double') + ~strcmpi(OPTION.precision, 'single') && ~strcmpi(OPTION.precision, 'double') error('EMC:precision', "OPTION.precision should be 'single' or 'double'") end else @@ -188,7 +188,7 @@ if isDim(2); vY = (limits(1, 2) - OPTION.shift(2)):(limits(2, 2) - OPTION.shift(2)); else; vY = nan; end if is3d; vZ = (limits(1, 3) - OPTION.shift(3)):(limits(2, 3) - OPTION.shift(3)); else; vZ = nan; end -% reciprocal space + % reciprocal space else if isDim(1) if (OPTION.half) diff --git a/coordinates/eulerSearch.m b/coordinates/eulerSearch.m index 17f1e0ee..42887588 100644 --- a/coordinates/eulerSearch.m +++ b/coordinates/eulerSearch.m @@ -8,9 +8,10 @@ number_of_search_dimensions = 0; number_of_search_positions = 0; number_of_out_of_plane_angles = 1; % poorly named. Theta of zero is still searched but not "outofplane" + active_theta_positions = []; number_of_angles_at_each_theta = []; best_parameters_to_keep = 0; - list_of_search_parameters = {}; + list_of_search_parameters = {}; list_of_best_parameters = {}; symmetry_symbol = 'C1'; number_of_asymmetric_units = 1; @@ -26,6 +27,7 @@ max_search_x = 0.0; max_search_y = 0.0; bipolar_search = false; + initialized = false; parameter_map = struct( ... 'phi', [] , ... 'theta', [], ... @@ -36,13 +38,13 @@ methods function [obj] = eulerSearch(wanted_symmetry_symbol, ... - wanted_theta_max,... - wanted_theta_step,... - wanted_psi_max,... - wanted_psi_step,... - wanted_resolution_limit,... - wanted_parameters_to_keep,... - wanted_random_start_angle) + wanted_theta_max,... + wanted_theta_step,... + wanted_psi_max,... + wanted_psi_step,... + wanted_resolution_limit,... + wanted_parameters_to_keep,... + wanted_random_start_angle) if (wanted_theta_max < 0) wanted_theta_max = abs(wanted_theta_max); @@ -50,7 +52,7 @@ end obj.random_start_angle = wanted_random_start_angle; - + obj.theta_max = wanted_theta_max; obj.theta_step = wanted_theta_step; % emClarity takes -angle:step:angle. This max is based on 0:360 @@ -58,34 +60,35 @@ obj.psi_max = 2.*wanted_psi_max; obj.psi_step = wanted_psi_step; obj.symmetry_symbol = wanted_symmetry_symbol; - + SetSymmetryLimits(obj); CalculateGridSearchPositions(obj); - + end + - function [] = CalculateGridSearchPositions(obj) - - + + obj.initialized = true; theta_max_local = obj.theta_max; obj.parameter_map.psi = -obj.psi_max./2 : obj.psi_step : obj.psi_max/2; - obj.number_of_search_positions = 0; - + obj.number_of_search_positions = 0; + theta_search = [ 0 : obj.theta_step : theta_max_local ]; if isempty(theta_search) - theta_search = 0; + theta_search = 0; end if (obj.bipolar_search) theta_search = [theta_search, flip(180-theta_search)]; end obj.parameter_map.theta = theta_search; obj.number_of_out_of_plane_angles = length(theta_search); + obj.active_theta_positions = 1:length(theta_search); obj.number_of_angles_at_each_theta = zeros(obj.number_of_out_of_plane_angles,1); obj.parameter_map.phi = cell(obj.number_of_out_of_plane_angles,1); - + obj.number_of_search_positions = 0; % Change this to include inplane angles explicitly. @@ -97,21 +100,21 @@ else % angular sampling was adapted from Spider subroutine VOEA (Paul Penczek) phi_step = 1.*abs(obj.theta_step / sind(theta)); - if (phi_step > obj.phi_max) + if (phi_step > obj.phi_max) phi_step = obj.phi_max; else phi_step = obj.phi_max / floor(obj.phi_max / phi_step + 0.5); end end - - if (obj.random_start_angle == true) + + if (obj.random_start_angle == true) phi_start_local = phi_step / 2.0 * (rand(1) - 0.5); else phi_start_local = 0.0; end - - obj.parameter_map.phi{iT} = [0:phi_step:obj.phi_max - 1] + phi_start_local; - + + obj.parameter_map.phi{iT} = [0:phi_step:obj.phi_max - 1] + phi_start_local; + obj.number_of_angles_at_each_theta(iT) = length(obj.parameter_map.phi{iT}) .* length(obj.parameter_map.psi); end @@ -122,43 +125,59 @@ end end + + function [] = HelicalRestriction(obj, max_deviation_from_xy_plane) + if ~obj.initialized + error('Must call Init before calling HelicalRestrictions'); + end + if (max_deviation_from_xy_plane ~= 0.0) + included_angles = abs(obj.parameter_map.theta - 90) <= max_deviation_from_xy_plane; + if (sum(included_angles) == 0) + error('No angles are within the helical restriction'); + end + obj.active_theta_positions = find(included_angles); + obj.number_of_out_of_plane_angles = length(obj.active_theta_positions); + obj.number_of_search_positions = sum(obj.number_of_angles_at_each_theta(obj.active_theta_positions)); + + end + end function [] = SetSymmetryLimits(obj) - + switch obj.symmetry_symbol(1) - case 'C' - if (length(obj.symmetry_symbol) < 2) - error('Cyclic symmetry requires an int specifying CX'); - end - + case 'C' + if (length(obj.symmetry_symbol) < 2) + error('Cyclic symmetry requires an int specifying CX'); + end + obj.psi_max = min(obj.psi_max,360.0 / EMC_str2double(obj.symmetry_symbol(2:end))); obj.theta_max = min(180.0, obj.theta_max); obj.phi_max = 360.0; % This will be incompatible with "symmetry_constrained_search" in BH_mutli_gridAngleSEarch (or whatever) obj.number_of_asymmetric_units = EMC_str2double(obj.symmetry_symbol(2:end)); case 'D' % FIXME is this right? - if ((length(obj.symmetry_symbol) < 2)) - error('D symmetry requires an int specifying DX'); - end + if ((length(obj.symmetry_symbol) < 2)) + error('D symmetry requires an int specifying DX'); + end obj.psi_max = min(360.0 / EMC_str2double(obj.symmetry_symbol(2:end))); obj.theta_max = min(obj.theta_max,90.0); obj.phi_max = 360.0; obj.number_of_asymmetric_units = EMC_str2double(obj.symmetry_symbol(2:end)*2); - - case 'O' - if ((length(obj.symmetry_symbol) > 1)) - error('Octahedral symmetry requires no int'); - end - + + case 'O' + if ((length(obj.symmetry_symbol) > 1)) + error('Octahedral symmetry requires no int'); + end + obj.psi_max = min(obj.psi_max,90.0); - obj.theta_max = min(obj.theta_max,54.7); + obj.theta_max = min(obj.theta_max,54.7); obj.phi_max = 90.0; obj.number_of_asymmetric_units = 24; - - case 'I' - % Double check convention: TODO - % 2 fold on Z, 5 fold 31.17 deg around X on Y axis, 3 fold 20.91 - % deg around Y on X. For I2 the X/Y axes are flipped + + case 'I' + % Double check convention: TODO + % 2 fold on Z, 5 fold 31.17 deg around X on Y axis, 3 fold 20.91 + % deg around Y on X. For I2 the X/Y axes are flipped if ((length(obj.symmetry_symbol) < 2)) obj.psi_max = min(obj.psi_max,180.0); obj.theta_max = 31.7; @@ -166,7 +185,7 @@ elseif strcmp(obj.symmetry_symbol,'2') obj.psi_max = min(obj.psi_max,180.0); obj.theta_max = 31.7; - obj.phi_max = 180.0; + obj.phi_max = 180.0; else error('Icosohedral can be I or I2, not (%s)',obj.symmetry_symbol); end @@ -175,8 +194,8 @@ error('symmetry symbol (%s) not recognized', obj.symmetry_symbol); end end - - + + end end diff --git a/ctf/BH_ctfCalc.m b/ctf/BH_ctfCalc.m index 21715c4d..06bd164b 100755 --- a/ctf/BH_ctfCalc.m +++ b/ctf/BH_ctfCalc.m @@ -1,8 +1,11 @@ function [ ctfMask, Hqz] = BH_ctfCalc(PixelSize, Cs, Lamda, Defocus, ... - CTFSIZE, AMPCONT, Phase_Only, varargin) + CTFSIZE, AMPCONT, Phase_Only, varargin) % Calculate a ctf and with the given modification to information at resolution % lower than the first peak. Also return the unmodified ctf. +% FIXME: I think the need for double was for overflow in the phase +% we use single in cisTEM, so it must just be scaling issue. Or better yet, switch over to mexCTF + precision = 'single'; flgComplex = 0; calcOneD = 0; @@ -14,9 +17,9 @@ precision = 'double'; calcOneD = 0; else - maxZ = varargin{1}; - precision = 'double'; - calcOneD = 1; + maxZ = varargin{1}; + precision = 'double'; + calcOneD = 1; end else precision = 'double'; @@ -45,8 +48,8 @@ preShiftedOrigin = PixelSize{2}; phi = PixelSize{3}; calcRad = 0; - - + + % While switching to the default behavior of using ang, check any input radial grid % and make sure it is in Angstrom. Check the middle of the frequency % range that way the input could be shifted or not. @@ -64,7 +67,7 @@ calcRad = 1; end -CS = Cs; +CS = Cs; WL = Lamda; if numel(Defocus) == 1 @@ -80,15 +83,15 @@ - CS = CS * 10^10; - WL = WL * 10^10; - df1 = df1 * 10^10; - df2 = df2 * 10^10; - +CS = CS * 10^10; +WL = WL * 10^10; +df1 = df1 * 10^10; +df2 = df2 * 10^10; + -if numel(CTFSIZE) == 1 +if numel(CTFSIZE) == 1 CTFSIZE(2) = CTFSIZE(1); end @@ -97,27 +100,27 @@ end if ( calcRad ) - + if strcmpi(precision, 'single') [radialGrid,phi,~,~,~,~] = ... - BH_multi_gridCoordinates(CTFSIZE(1:2),'Cylindrical','GPU',{'none'},1,0,0); + BH_multi_gridCoordinates(CTFSIZE(1:2),'Cylindrical','GPU',{'none'},1,0,0); else - + [radialGrid,phi,~,~,~,~] = ... - BH_multi_gridCoordinates(CTFSIZE(1:2),'Cylindrical','GPU',{'none'},1,0,0); + BH_multi_gridCoordinates(CTFSIZE(1:2),'Cylindrical','GPU',{'none'},1,0,0); radialGrid = double(radialGrid); phi = double(phi); end - - radialGrid = radialGrid ./ PIXEL_SIZE; + radialGrid = radialGrid ./ PIXEL_SIZE; + end % Any additional phase shift due to the phase plate is stored with the % amplitude contrast. This will produce very small errors (< 1% for .07,0.1) % for older versions that expect just the amplitude contrast ratio, rather % than the phase shift. -% % % if (abs(AMPCONT - 1.0) < 1e-3) +% % % if (abs(AMPCONT - 1.0) < 1e-3) % % % precomputed_amplitude_contrast_term = pi / 2.0; % % % else % % % precomputed_amplitude_contrast_term = atan2(AMPCONT,sqrt(1.0 - AMPCONT^2)); @@ -126,85 +129,81 @@ % df1 should be defocus of greater mag and phi0 -90/90 % phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 + DF.*WL.*(radialGrid).^2); -dfTerm = 0.5.*( (df1+df2) + (df1-df2)*cos(2.*(phi-phi0)) ); -phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 + ... - WL.*(radialGrid).^2 .* dfTerm); -% dPdQ = 2*pi*CS*WL^3.*radialGrid.^3 + 2*WL.*radialGrid.*dfTerm; +if (df1 - df2 < 0) + error('df1 must be greater than df2') +end + +dfTerm = 0.5.*( (df1+df2) + (df1-df2)*cos(2.*(phi0-phi)) ); + +phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 - WL.*(radialGrid).^2 .* dfTerm); +% dPdQ = 2*pi*CS*WL^3.*radialGrid.^3 + 2*WL.*radialGrid.*dfTerm; if ( flgComplex ) ctfMask = exp(-1i.*(phasePerturbation-atan2(AMPCONT,sqrt(1+AMPCONT)))); Hqz = exp(+1i.*(phasePerturbation-atan2(AMPCONT,sqrt(1+AMPCONT)))); return else -% Hqz = (sqrt(1-AMPCONT^2).*sin(phasePerturbation) - AMPCONT.*cos(phasePerturbation)); + % Hqz = (sqrt(1-AMPCONT^2).*sin(phasePerturbation) - AMPCONT.*cos(phasePerturbation)); end Hqz = sin(phasePerturbation - AMPCONT); nanCheck = isnan(Hqz); if gather(sum(nanCheck(:))) - Hqz(nanCheck) = 0; + Hqz(nanCheck) = 0; end - if Phase_Only < 0 - - + % FIXME I need to know if I am a half grid or els this fails! + oX = floor(CTFSIZE(1)/2)+1; + oY = floor(CTFSIZE(2)/2)+1; - - oX = floor(CTFSIZE(1)/2)+1; - oY = floor(CTFSIZE(2)/2)+1; - - - if (calcOneD) - if (preShiftedOrigin) - rV = Hqz(oX:end); - else - rV = Hqz(1:oX); % should this be oX-1? - end - elseif (doHalfGrid) - if ( preShiftedOrigin) - rV = Hqz(1:end,oY); - else - rV = Hqz(1:end,1); - end + if (calcOneD) + if (preShiftedOrigin) + rV = Hqz(oX:end); else - if ( preShiftedOrigin) - rV = Hqz(oX:end,oY); - else - rV = Hqz(1:oX,1); - end + rV = Hqz(1:oX); % should this be oX-1? end - - firstZero = find(rV > 0, 1,'first'); - if isempty(firstZero) || firstZero < floor(0.1.*CTFSIZE(1)) - firstMin = floor(CTFSIZE(1)/2)-6; + elseif (doHalfGrid) + if ( preShiftedOrigin) + rV = Hqz(1:end,oY); else - [~,firstMin]=min(abs(rV(7:firstZero-1)-rV(8:firstZero))); - firstMin = firstMin + 6; + rV = Hqz(1:end,1); end - - - if ( preShiftedOrigin && ~calcOneD) - if doHalfGrid - freqMin = radialGrid(firstMin,ceil((CTFSIZE(1)+1)./2)); - maxRes = 0.5./radialGrid(ceil((CTFSIZE(1)+1)/2),1); - else - try - freqMin = radialGrid(ceil((CTFSIZE(1)+1)./2)+firstMin,ceil((CTFSIZE(2)+1)./2)); - catch - ceil((CTFSIZE(1)+1)./2) - end - maxRes = 0.5./radialGrid(1,ceil((CTFSIZE(2)+1)/2)); - end - + else + if ( preShiftedOrigin) + rV = Hqz(oX:end,oY); else - freqMin = radialGrid(firstMin,1); - freqZero = radialGrid(firstZero,1); + rV = Hqz(1:oX,1); + end + end + + firstZero = find(rV > 0, 1,'first'); + if isempty(firstZero) || firstZero < floor(0.1.*CTFSIZE(1)) + firstMin = floor(CTFSIZE(1)/2)-6; + else + [~,firstMin]=min(abs(rV(7:firstZero-1)-rV(8:firstZero))); + firstMin = firstMin + 6; + end + + if ( preShiftedOrigin && ~calcOneD) + if doHalfGrid + freqMin = radialGrid(firstMin,ceil((CTFSIZE(1)+1)./2)); maxRes = 0.5./radialGrid(ceil((CTFSIZE(1)+1)/2),1); + else + try + freqMin = radialGrid(ceil((CTFSIZE(1)+1)./2)+firstMin,ceil((CTFSIZE(2)+1)./2)); + catch + ceil((CTFSIZE(1)+1)./2); + end + maxRes = 0.5./radialGrid(1,ceil((CTFSIZE(2)+1)/2)); end - - + else + freqMin = radialGrid(firstMin,1); + freqZero = radialGrid(firstZero,1); + maxRes = 0.5./radialGrid(ceil((CTFSIZE(1)+1)/2),1); + end + if (thisZero > 0) lowCut = 1./(0.1*freqMin+0.9*freqZero); if isempty(lowCut) @@ -215,20 +214,22 @@ else bFactor = 100; end - ctfMask = BH_bandpass3d(size(Hqz),0,800,lowCut,'GPU',maxRes); + + + ctfMask = BH_bandpass3d(size(Hqz),0.01,800,lowCut,'GPU',maxRes); % This term is straight from dTegunov's deconv snr = 10.^3.*exp((-2.2.*bFactor).*radialGrid); ctfMask = ctfMask .* Hqz ./ (Hqz.^2 + 1./snr); snr = []; else - if (flgDampen) + if (flgDampen) envelope = (exp(-20.*(radialGrid.*(0.5/max(radialGrid(:)))).^1.25) +0.1)./1.1; else envelope = 1; end - + try - ctfMask = (envelope).*(sign(Hqz).*(radialGrid <= freqMin ).*abs(Hqz).^abs(Phase_Only) + (radialGrid > freqMin).*Hqz); + ctfMask = (envelope).*(sign(Hqz).*(radialGrid <= freqMin ).*abs(Hqz).^abs(Phase_Only) + (radialGrid > freqMin).*Hqz); catch size(rV) size(radialGrid) @@ -239,12 +240,11 @@ maxRes error('sfd') end - end - - -elseif Phase_Only == 1 - - ctfMask = sign(Hqz); + end + + +elseif Phase_Only == 1 + ctfMask = sign(Hqz); else ctfMask = Hqz; diff --git a/ctf/BH_ctfCalcError.m b/ctf/BH_ctfCalcError.m index 241c09a8..3dc84bb1 100644 --- a/ctf/BH_ctfCalcError.m +++ b/ctf/BH_ctfCalcError.m @@ -1,6 +1,6 @@ - function [ ctfDepth ] = BH_ctfCalcError( pixelSize, Cs, Lambda, Defocus, ... - CTFSIZE, AMPCONT, ... - resCutOff,thicknessAng,dampeningMax,cycleNumber) +function [ ctfDepth ] = BH_ctfCalcError( pixelSize, Cs, Lambda, Defocus, ... + CTFSIZE, AMPCONT, ... + resCutOff,thicknessAng,dampeningMax,cycleNumber) %For a given resolution wanted, calculate the point where destructive %interference drops the CTF amplitude to some given threshold. % Detailed explanation goes here @@ -32,7 +32,7 @@ length([-maxDiff:0.1*maxDiff:maxDiff].*10^-10) for jDelDef = [-maxDiff:0.1*maxDiff:maxDiff].*10^-10 if isempty(ctf1) - ctf1 = BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1,1); + ctf1 = BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1,1); else ctf1 = ctf1 + BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1,1); end @@ -64,7 +64,7 @@ ctf1 = []; for jDelDef = [-maxDiff:0.1*maxDiff:maxDiff].*10^-10 if isempty(ctf1) - ctf1 = BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1); + ctf1 = BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1); else ctf1 = ctf1 + BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1); end @@ -86,7 +86,7 @@ if ctfDepth < 0 fprintf('The optimal ctfDepth was not found\n'); fprintf('Inputs %3.3e pix %3.3e cs %3.3e wl %3.3e def %3.3e resTarget %3.3e tomoDepth\n',... - pixelSize*10^-10, Cs, Lambda, Defocus,resCutOff,thicknessAng); + pixelSize*10^-10, Cs, Lambda, Defocus,resCutOff,thicknessAng); ctfDepth = min(thicknessAng/30 * 10^-9,resCutOff(1) * 10 ^-8) elseif ctfDepth < 0.5*10e-9 fprintf('\n\nCapping ctfDepth to 5 nm from a calc %3.3f nm\n\n',ctfDepth*10^9); diff --git a/ctf/BH_ctf_Correct.m b/ctf/BH_ctf_Correct.m deleted file mode 100755 index a9235640..00000000 --- a/ctf/BH_ctf_Correct.m +++ /dev/null @@ -1,215 +0,0 @@ -function [ ] = BH_ctf_Correct( PARAMETER_FILE, STACK_PRFX ) -%CTF correction for tilt series using general geometry. -% Correct for the CTF using a local approach, similar to strip based -% periodogram, but with tiles that are smaller allowing for arbitrary -% defocus gradients. -% -% The full stack is corrected, st if only a small region is to be used, -% it would be faster to have trimmed the stack. This should be done -% before ctf estimation though, st the correct origin is included in the -% tilt information. -% - -pBH = BH_parseParameterFile(PARAMETER_FILE); - -try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR; -catch - mapBackIter = 0; -end - -if isnan(str2double(STACK_PRFX)) - % It is a name, run here. - nGPUs = 1; - flgParallel = 0; - STACK_LIST = {STACK_PRFX}; - ITER_LIST = {STACK_LIST}; -else - flgParallel = 1; - nGPUs = pBH.('nGPUs'); - - STACK_LIST_tmp = fieldnames(subTomoMeta.mapBackGeometry); - STACK_LIST_tmp = STACK_LIST_tmp(~ismember(STACK_LIST_tmp,'tomoName')); - ITER_LIST = cell(nGPUs,1); - nST = 1; STACK_LIST = {}; - for iStack = 1:length(STACK_LIST_tmp) - if subTomoMeta.mapBackGeometry.(STACK_LIST_tmp{iStack}).nTomos - STACK_LIST{nST} = STACK_LIST_tmp{iStack}; - nST = nST +1; - end - end - clear STACK_LIST_tmp - for iGPU = 1:nGPUs - ITER_LIST{iGPU} = STACK_LIST(iGPU:nGPUs:length(STACK_LIST)); - end -end - -pixelSize = pBH.('PIXEL_SIZE'); -!mkdir -p ctfStacks - -try - EMC_parpool(nGPUs); -catch - delete(gcp('nocreate')); - EMC_parpool(nGPUs); -end - - -parfor iGPU = 1:nGPUs - - if ( flgParallel ) - useGPU = iGPU; - gpuDevice(useGPU); - else - useGPU = BH_multi_checkGPU(-1); - gpuDevice(useGPU); - end - - for iTilt = 1:length(ITER_LIST{iGPU}) - - - STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - - try - % make sure there isn't a refined version first. - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',STACK_PRFX,mapBackIter+1); - TLT = load(TLTNAME) - fprintf('using refined TLT %s\n', TLTNAME); - catch - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - end - - inputStack = sprintf('aliStacks/%s_ali%d.fixed',STACK_PRFX,mapBackIter+1); - outputStack = sprintf('ctfStacks/%s_ali%d_ctf.fixed',STACK_PRFX,mapBackIter+1); - iMrcObj = MRCImage(inputStack,0); - - - - iHeader = getHeader(iMrcObj); - iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... - iHeader.cellDimensionY/iHeader.nY, ... - iHeader.cellDimensionZ/iHeader.nZ]; - - d1 = iHeader.nX; - d2 = iHeader.nY; - nPrjs = iHeader.nZ; - - correctedStack = zeros(d1,d2,nPrjs,'single'); - - for iPrj = 1:nPrjs - - CS = TLT(iPrj,17); - WL = TLT(iPrj,18); - AMPCONT = TLT(iPrj,19); - ddF = TLT(iPrj,12); - dPhi = TLT(iPrj,13); - D0 = TLT(iPrj,15); - - - fastFTSize = BH_multi_iterator([d1,d2],'fourier2d'); - padVal = BH_multi_padVal([d1,d2],fastFTSize); - trimVal = BH_multi_padVal(fastFTSize,[d1,d2]); - - - initImg = randn(fastFTSize,'single','gpuArray'); - f = FFT(initImg); - - ctf = CTF(fastFTSize,pixelSize*10^10,'GPU'); - maxZ = 500; - maxEval = cosd(TLT(iPrj,4)).*(d1/2) + maxZ./2*abs(sind(TLT(iPrj,4))); - oX = ceil((d1+1)./2); - oY = ceil((d2+1)./2); - iEvalMask = floor(oX-maxEval):ceil(oX+maxEval); - - STRIPWIDTH = 512; - STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); - % take at least 1200 Ang & include the taper if equal to STRIPWIDTH - tileSize = floor(max(600./pixelSize, STRIPWIDTH + 28)); - tileSize = tileSize + mod(tileSize,2); - %fprintf('stripwidth tilesize %d %d\n',STRIPWIDTH,tileSize); - incLow = ceil(tileSize./2); - incTop = tileSize - incLow; - - - iProjection = BH_padZeros3d(getVolume(iMrcObj,[-1],[-1],TLT(iPrj,23),'keep'), ... - padVal(1,:),padVal(2,:),'GPU','singleTaper'); - - correctedPrj = zeros([d1,d2],'single','gpuArray'); - iProjectionFT = f.fwdFFT(iProjection); - - - stripDefocusOffset = floor(STRIPWIDTH/2); - for i = 1: STRIPWIDTH : d1 - - if (i+tileSize-1) < d1 - endIDX = (i+tileSize-1); - endCUT = i + STRIPWIDTH - 1 + 7; - trimmedSIZE = STRIPWIDTH; - elseif any(ismember(i:d1,iEvalMask)) - endIDX = d1; - endCUT = d1; - trimmedSIZE = endCUT-i+1 -7; - end - - % The eval mask condition can be replaced once the per tomo condition - % is trusted. - if any(ismember(i:endIDX,iEvalMask)) - - - DF = D0 +(i + stripDefocusOffset - oX)*pixelSize*-1.*tand(TLT(iPrj,4)); - - - if ~( isempty(DF) ) - - iDefocus = [DF - ddF, DF + ddF, dPhi]; - - if pixelSize < 2.0e-10 - % use double precision - this is not enabled, but needs to be - - % requires changes to radial grid as well. - ctf.new_img(iDefocus,CS,WL,AMPCONT,-1,-1); - else - ctf.new_img(iDefocus,CS,WL,AMPCONT,-1); - end - - - tile = ctf.multiply(iProjectionFT); - - tile = BH_padZeros3d(real(f.invFFT(tile,2)), ... - trimVal(1,:),trimVal(2,:),'GPU','single'); - - % trim prior to pulling off gpu to minimize xfer - else - - % No particles in this strip, so just replace with simple inversion - % to keep the global image statistics ~ correct. - - tile = -1.*BH_padZeros3d(iProjection, trimVal(1,:),trimVal(2,:),'GPU','single'); - - end - - %correctedStack(i + 7 : endCUT,:,TLT(iPrj,1)) = ... - % gather(tile(8:trimmedSIZE+7,:)); - - correctedPrj(i:endIDX,:) = tile(i:endIDX,:); - - else - %fprintf('ignoring strip centered on %d for prj %d',i,TLT(iPrj,1)); - end - end % end loop over strips - correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj); - - end % end loop over prjs - - SAVE_IMG(MRCImage(correctedStack), outputStack,iPixelHeader); - - end % end loop over tilt-series -end % end parfor - -delete(gcp('nocreate')); - - -end - diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 2fbfd4ad..b914618d 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -9,215 +9,157 @@ % emClarity ctf 3d paramN.m 'templateMatching' = use this to make recs % for higher res template matching (in the works) -% Read in 2dCtf stacks to trouble shoot -PosControl2d=0; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); -% Apply a Wiener filter with this many zeros during Ctf multiplication -global bh_global_turn_on_phase_plate -masterTM = struct(); +% phase_plate_mode is now handled in BH_parseParameterFile +phase_plate_mode = emc.phase_plate_mode; +subTomoMeta = struct(); resTarget = 15; + % TODO remove thise params tiltWeight = [0.2,0]; -shiftDefocusOrigin = 1; +shiftDefocusOrigin = emc.set_defocus_origin_using_subtomos; tiltStart = 1; -try - flgEraseBeads_aferCTF = pBH.('erase_beads_after_ctf'); -catch - flgEraseBeads_aferCTF = false; % If false they SHOULD be erased in ctf estimate/update, but since the user could change parameter, include here. -end +% erase_beads_after_ctf is now handled in BH_parseParameterFile +flgEraseBeads_aferCTF = emc.erase_beads_after_ctf; % Test David's new super sampling in reconstruction. No check that this % version (currently 4.10.40) is properly sourced. -try - super_sample = pBH.('super_sample'); - if (super_sample > 0) - [~,v] = system('cat $IMOD_DIR/VERSION'); - v = split(v,'.'); - if (EMC_str2double(v{1}) < 4 || (EMC_str2double(v{2}) <= 10 && EMC_str2double(v{3}) < 42)) - fprintf('Warning: imod version is too old for supersampling\n'); - super_sample = ''; - else - super_sample = sprintf(' -SuperSampleFactor %d',super_sample); - end - else + +super_sample = emc.('super_sample'); +if (super_sample > 0) + [~,v] = system('cat $IMOD_DIR/VERSION'); + v = split(v,'.'); + if (EMC_str2double(v{1}) < 4 || (EMC_str2double(v{2}) <= 10 && EMC_str2double(v{3}) < 42)) + fprintf('Warning: imod version is too old for supersampling\n'); super_sample = ''; + else + super_sample = sprintf(' -SuperSampleFactor %d',super_sample); end - -catch +else super_sample = ''; end + -try - expand_lines = pBH.('expand_lines'); - if isempty(super_sample) || expand_lines == false - expand_lines = ''; - else - expand_lines = ' -ExpandInputLines'; - end -catch +flip_defocus_offset = emc.test_flip_defocus_offset +flip_tilt_offset = emc.test_flip_tilt_offset + +expand_lines = emc.('expand_lines'); +if isempty(super_sample) || expand_lines == false expand_lines = ''; +else + expand_lines = ' -ExpandInputLines'; end fprintf('\n Superampling in imod is [%s] with expandLines [%s]\n',super_sample ,expand_lines); %default to cycle number zero for %determining mean z height of particles -recWithoutMat = false; reconstructionParameters = 0; filterProjectionsForTomoCPRBackground=0; -loadSubTomoMeta = true; -if nargin > 2 - if ~isempty(EMC_str2double(varargin{1})) - reconstructionParameters = EMC_str2double(varargin{1}); - recWithoutMat = true; - if length(varargin) > 2 - % Full recon for tomoCPR - bh_global_turn_on_phase_plate = 0 - filterProjectionsForTomoCPRBackground = 28 +flgWhitenPS = [0,0,0.0]; +use_existing_tmpCache=''; +recon_for_templateMatching = false; +recon_for_subTomo = false; + +recon_subset=[1,-1]; + +if nargin > 1 + if strcmpi(varargin{1},'templateSearch') + recon_for_templateMatching = true; + if (phase_plate_mode) + fprintf('WARNING: the filtered tomogram should only be used for viz, not template matching.'); + end + else + if strcmpi(varargin{1},'split') + % Default to zero for normal use + recon_for_subTomo = true; + % phase_plate_mode is already set from parameters else - loadSubTomoMeta = false; - % Default to on for subregion picking - % If user has specified phakePhasePlate, don;t use ...otherwise - if isempty(bh_global_turn_on_phase_plate(1)) || bh_global_turn_on_phase_plate(1) == 0 - bh_global_turn_on_phase_plate = [1,2] - end + error('Extra argument to ctf 3d either templateSearch/split and optionally a vector [iProjcess, nProcesses (from 1)]'); end end -elseif nargin > 1 - if strcmpi(varargin{1},'templateSearch') - recWithoutMat = true - loadSubTomoMeta = false - if (bh_global_turn_on_phase_plate(1)) - fprintf('WARNING: the filtered tomogram should only be used for viz, not template matching.'); + + if nargin > 2 + if isempty(EMC_str2double(varargin{2})) + error('Extra argument to ctf 3d either templateSearch/split and optionally a vector [iProjcess, nProcesses (from 1)]'); + else + recon_subset = EMC_str2double(varargin{2}); + if numel(recon_subset) ~= 2 + error('Extra argument to ctf 3d either templateSearch/split and optionally a vector [iProjcess, nProcesses (from 1)]'); + end + end - else - error('Extra argument to ctf 3d should be a vector [THICKNESS, BINNING] tiltN, or a string templateSearch'); end else % Default to zero for normal use - if isempty(bh_global_turn_on_phase_plate) - bh_global_turn_on_phase_plate = 0; - end + recon_for_subTomo = true; + % phase_plate_mode is already set from parameters end -try +fprintf('recon_subset is [%d,%d]\n',recon_subset(1),recon_subset(2)); + +if ( recon_for_templateMatching + recon_for_subTomo ~= 1) + error('Only one of the two modes can be used at a time'); +end + +try % -1, whiten before ctf, 1 whiten after - test both. - flgWhitenPS = [pBH.('whitenPS')(1),0,pBH.('whitenPS')(2)]; + usr_flgWhitenPS = emc.('whitenPS'); + if (numel(usr_flgWhitenPS) == 3) + flgWhitenPS = usr_flgWhitenPS; + else + error('flgWhitenPS should be a 3 element vector'); + end catch - flgWhitenPS = [0,0,0]; end -if (bh_global_turn_on_phase_plate(1) && flgWhitenPS(1)) - fprintf('WARNING: phakePhasePlate and whitening are conflicting preocesses. Turning off whitening.\n') - flgWhitenPS(1) = 0; +if (phase_plate_mode && any(emc.whitenPS)) + fprintf('WARNING: phakePhasePlate and whitening are conflicting preocesses. Turning off whitening.\n'); + emc.whitenPS = [0,0,0]; end try - applyExposureFilter = pBH.('applyExposureFilter') + applyExposureFilter = emc.('applyExposureFilter') catch applyExposureFilter = 1; end % This will be set false if the reconstruction is for template matching or % for tomoCPR -try - useSurfaceFit = pBH.('useSurfaceFit') -catch - useSurfaceFit = 1 -end +useSurfaceFit = emc.('useSurfaceFit') -try - % Not for normal use, pass the total dose less first frame to flip values. - invertDose = pBH.('invertDose') -catch - invertDose = 0; -end -%cycleNumber = sprintf('cycle%0.3d',CYCLE); -%fprintf('cycle is %d\n',CYCLE); fprintf('tiltweight is %f %f\n',tiltWeight); +[tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(emc.fastScratchDisk, use_existing_tmpCache, 'ctf3d', false); - -tmpCache= pBH.('fastScratchDisk'); - -if strcmpi(tmpCache, 'ram') - if isempty(getenv('EMC_CACHE_MEM')) - fprintf('Did not find a variable for EMC_CACHE_MEM\nSkipping ram\n'); - tmpCache= ''; - else - % I have no ideah how much is needed - if EMC_str2double(getenv('EMC_CACHE_MEM')) < 64 - fprintf('There is only 64 Gb of cache on ramdisk, not using'); - tmpCache = ''; - else - tmpCache=getenv('MCR_CACHE_ROOT'); - fprintf('Using the tmp EMC cache in ram at %s\n',tmpCache); - end - end -end - -% Check to make sure it even exists -if isempty(dir(tmpCache)) - fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); - tmpCache = ''; -end - -reconScaling = 1; - -if isempty(tmpCache) - tmpCache='cache'; - flgCleanCache = 0; - CWD=''; -else - flgCleanCache = 1; - CWD = sprintf('%s/',pwd); - % Check for a trailing slash - slashCheck = strsplit(tmpCache,'/'); - if isempty(slashCheck{end}) - % This means the final character was a slash, strip it - tmpCache = sprintf('%scache',tmpCache); %strjoin(slashCheck(1:end-1),'/'); - else - tmpCache = sprintf('%s/cache',tmpCache); - end +if ( recon_for_templateMatching) + useSurfaceFit = false; end -% Incase this is launched form another process (synthetic mapback for example, make one level lower in the cache -tmpCache=sprintf('%s/ctf3d',tmpCache); -fprintf('tmpCache is %s\n',tmpCache); -system(sprintf('mkdir -p %s',tmpCache)); -system(sprintf('mkdir -p %s','cache')); % This should exist, but to be safe. -if (recWithoutMat) - useSurfaceFit = false; - if (loadSubTomoMeta) - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR; - masterTM = subTomoMeta; clear subTomoMeta - CYCLE = masterTM.currentCycle; - else - mapBackIter = 0; - CYCLE = 0; - end +if (recon_for_templateMatching) + mapBackIter = 0; + CYCLE = 0; else - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + % Load using wrapper + subTomoMeta = BH_loadSubTomoMeta(emc.('subTomoMeta'), emc.('metadata_format')); mapBackIter = subTomoMeta.currentTomoCPR; - masterTM = subTomoMeta; clear subTomoMeta - CYCLE = masterTM.currentCycle; + CYCLE = subTomoMeta.currentCycle; end cycleNumber = sprintf('cycle%0.3d',CYCLE); % This should be run after raw alignment and after cycle 0 if (CYCLE) - if isfield(masterTM.(cycleNumber),'RawAlign') + if isfield(subTomoMeta.(cycleNumber),'RawAlign') fprintf(' %s \n',cycleNumber); - elseif isfield(masterTM.(sprintf('cycle%0.3d',CYCLE-1)),'RawAlign') + elseif isfield(subTomoMeta.(sprintf('cycle%0.3d',CYCLE-1)),'RawAlign') cycleNumber = sprintf('cycle%0.3d',CYCLE-1); fprintf('Falling back the previous alignment cycle\n'); else @@ -227,74 +169,42 @@ try - flgDampenAliasedFrequencies = pBH.('flgDampenAliasedFrequencies') -catch - flgDampenAliasedFrequencies = 0 -end - -try - flg2dCTF = pBH.('flg2dCTF'); -catch - flg2dCTF = 0; -end - -try - % Part of the experiment with template matching using higher res info, also - % allow for a median filter post CTF correction, pre reconstruction to + % Part of the experiment with template matching using higher res info, also + % allow for a median filter post CTF correction, pre reconstruction to % further denoise prior to template matching. - flgMedianFilter = pBH.('ctfMedianFilter'); + flgMedianFilter = emc.('ctfMedianFilter'); catch flgMedianFilter = 0; end - -% ctf3dDepth=pBH.('defocusErrorEst') -%mean in case cones. %%%%% Take these from param file later. -if (reconstructionParameters(1)) - samplingRate = reconstructionParameters(2); +if (recon_for_subTomo) + samplingRate = emc.('Ali_samplingRate'); + % This number is used to roughly balance the trade off between + % achievable resolution, and run time during reconstruction as + % determined by the thickness of each 3d slab reconstructed. Given that + % we expect the resolution to improve beyond our current value, we + % multiply by 1/2, which gives a (only loosely optimized) resTarget. + resTarget = mean(subTomoMeta.('currentResForDefocusError')*0.5); + if (emc.whitenPS(1)) + emc.whitenPS(2) = resTarget; + end else - if (loadSubTomoMeta) - samplingRate = pBH.('Ali_samplingRate'); - % This number is used to roughly balance the trade off between - % achievable resolution, and run time during reconstruction as - % determined by the thickness of each 3d slab reconstructed. Given that - % we expect the resolution to improve beyond our current value, we - % multiply by 1/2, which gives a (only loosely optimized) resTarget. - resTarget = mean(masterTM.('currentResForDefocusError')*0.5); - if (flgWhitenPS(1)) - flgWhitenPS(2) = resTarget; - end - else - % For template search - samplingRate = pBH.('Tmp_samplingRate'); - try - resTarget = pBH.('lowResCut'); - catch - resTarget = 12; - end - + % For template search + samplingRate = emc.('Tmp_samplingRate'); + try + resTarget = emc.('lowResCut'); + catch + resTarget = 12; end - end -try - max_ctf3dDepth = pBH.('max_ctf3dDepth'); -catch - max_ctf3dDepth = 500*10^-9; -end - -if (max_ctf3dDepth < 1 * 10^-9 || max_ctf3dDepth > 1000 * 10^-9) - error('max_ctf3dDepth should be between 1 and 1000 nm'); -else - fprintf('Using a max_ctfDepth of %2.2f nm\n',max_ctf3dDepth*10^9); -end fprintf('Using a target resolution of %2.2f Angstroms\n',resTarget); -nGPUs = pBH.('nGPUs'); +nGPUs = emc.('nGPUs'); % Optionally specify gpu idxs if numel(nGPUs) == 1 gpuList = 1:nGPUs; @@ -303,325 +213,252 @@ nGPUs = length(gpuList); end -pixelSize = pBH.('PIXEL_SIZE').*10^10 .* samplingRate; +emc.pixel_size_angstroms = emc.pixel_size_angstroms .* samplingRate; -% if (recWithoutMat) -% reconstructionParameters(1) = ')(i) =(1) ./ pixelSize; -% end -if pBH.('SuperResolution') - pixelSize = pixelSize * 2; -end - -eraseRadius = ceil(1.5.*(pBH.('beadDiameter')./pBH.('PIXEL_SIZE').*0.5) / samplingRate); +eraseRadius = ceil(1.5.*(emc.('beadDiameter')./emc.pixel_size_si.*0.5) / samplingRate); nTomosPerTilt = 0; recGeom = 0; - -if (recWithoutMat) - if reconstructionParameters(1) && loadSubTomoMeta - tiltList{1} = varargin{2}; - nTilts = 1; - % We just need one valid subtomot - iTry = 1; - tomoList{1} = ''; - while iTry < 25 - if (isfield(masterTM.mapBackGeometry.tomoName,sprintf('%s_%d',tiltList{1},iTry))) - tomoList{1} = sprintf('%s_%d',tiltList{1},iTry); - break; - end - iTry = iTry + 1; - end - if isempty(tomoList{1}) - error('Did not find a valid tomogram in the searchspace ->25'); - end - else - % TODO set up a check on the recon folder to get what is needed for - % templateSearch - getCoords = dir('recon/*.coords'); - nTilts = length(getCoords); - if (nTilts == 0) - error('Did not find any tomogram coordinates in recon/TS*.coords'); - end - tiltList = cell(nTilts,1); - nTomosTotal = 0; - nTomosPerTilt = cell(nTilts,1); - recGeom = cell(nTilts,1); - for iStack = 1:nTilts - [ recGeom{iStack}, tiltName, nTomosPossible] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name) ); - nTomosTotal = nTomosTotal + nTomosPossible; - nTomosPerTilt{iStack} = nTomosPossible; - tiltList{iStack} = tiltName; - end - - tomoList = cell(nTomosTotal,1); - nTomosAdd = 0; - for iStack = 1:nTilts - for iTomo = 1:nTomosPerTilt{iStack} - tomoList{nTomosAdd+1} = sprintf('%s_%d',tiltName,iTomo); - nTomosAdd = nTomosAdd +1; - end - end - +tiltRecGeom = 0; +tomoList = {}; +nTomos= 0; +tiltRecGeom = {}; +tiltTomoList = {}; +if (recon_for_subTomo) + [tiltList, nTilts] = BH_returnIncludedTilts(subTomoMeta.mapBackGeometry); +else + % TODO set up a check on the recon folder to get what is needed for + % templateSearch + getCoords = dir('recon/*.coords'); + nTilts = length(getCoords); + if (nTilts == 0) + error('Did not find any tomogram coordinates in recon/TS*.coords'); + end + tiltList = cell(nTilts,1); + tiltRecGeom = cell(nTilts,1); + tiltTomoList = cell(nTilts,1); + for iStack = 1:nTilts + % Since we are calling this for templateSearch nTomosPossible == nTomos + % After template matching, there may be inactive tomos, but we'll have the same amount + [ tiltRecGeom{iStack}, tiltName, tiltTomoList{iStack}, tilt_geometry] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); + tiltList{iStack} = tiltName; end -else - [tiltList,nTilts] = BH_returnIncludedTilts(masterTM.mapBackGeometry); - tomoList = fieldnames(masterTM.mapBackGeometry.tomoName); end % Divide the tilt series up over each gpu iterList = cell(nGPUs,1); % If there is only one tilt, things break in a weird way -nGPUs -nTilts -nGPUs = min(nGPUs, nTilts) -for iGPU = 1:nGPUs - iterList{gpuList(iGPU)} = iGPU+(tiltStart-1):nGPUs:nTilts; - iterList{gpuList(iGPU)}; +nGPUs = min(nGPUs, nTilts); + +if (recon_subset(2) > 0) + [ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores, recon_subset); +else + [ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores); end + + try - EMC_parpool(nGPUs) + EMC_parpool(nParProcesses) catch delete(gcp('nocreate')) - EMC_parpool(nGPUs) -end + EMC_parpool(nParProcesses) +end - -parfor iGPU = 1:nGPUs - for iTilt = iterList{gpuList(iGPU)} - - iTomoList = {}; + +parfor iParProc = 1:nParProcesses +% for iParProc = 1:nParProcesses % revert + % iGPU = mod(iParProc,nGPUs); + for iTilt = iterList{iParProc} + nTomos = 0; + alreadyMade = 0; + % For now, since the tilt geometry is not necessarily updated (it is manual) - % in the masterTM, check that newer (possible perTilt refined) data is + % in the subTomoMeta, check that newer (possible perTilt refined) data is % not present. - try - % make sure there isn't a refined version first. - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',tiltList{iTilt},mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using refined TLT %s\n', TLTNAME); - catch - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - end - - - % Get all the tomogram names that belong to a given tilt-series. - if (~recWithoutMat) - nTomos = 0; - alreadyMade = 0; - for iTomo = 1:length(tomoList) - if strcmp(tiltList{iTilt},masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) - iTomoList{nTomos+1} = tomoList{iTomo}; - nTomos = nTomos + 1; - end - % The order of tomo num could be off but only if all are present do we - % skip. - if (bh_global_turn_on_phase_plate(1)) - filtered = '_filtered'; - else - filtered = ''; - end - checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', ... - tiltList{iTilt},iTomo,samplingRate,filtered); - if exist(checkRecon, 'file') - fprintf('found %s to already exits\n',checkRecon); - alreadyMade = alreadyMade +1; - end + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', tiltList{iTilt}, mapBackIter+1); + TLT = load(TLTNAME); + % Get all the tomogram names that belong to a given tilt-series. + % FIXME: I'm not sure it makes sense to restrict this block to for_subTomo + if (recon_for_subTomo || recon_for_templateMatching) + if (recon_for_subTomo) + % List of all possible tomos, some may be "in-active" since this is post-template matching + tomoList = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).tomoList; + else + % List of all possible tomos, all are "active" since this is pre-template matching + tomoList = tiltTomoList{iTilt}; + end + nTomos = length(tomoList); + if (phase_plate_mode) + filtered = '_filtered'; + else + filtered = ''; end + for iTomo = 1:nTomos + % The order of tomo num could be off but only if all are present do we + % skip. + alt_cache = emc.alt_cache; + checkRecon = EMC_checkCacheForFile(alt_cache, sprintf('cache/%s_bin%d%s.rec', tomoList{iTomo}, samplingRate, filtered)); + if isfile(checkRecon) + try + % Could have a corrupt file + testread = MRCImage(checkRecon,0); + fprintf('found %s to already exits\n',checkRecon); + alreadyMade = alreadyMade + 1 + catch + fprintf('found %s to already exits but it is corrupt\n',checkRecon); + system(sprintf('rm %s',checkRecon)); + end + end + + end + if alreadyMade == nTomos fprintf('All tomos 1-%d found to exist for tilt-series %s\n',nTomos,tiltList{iTilt}); + % remove the value form the iter list + iterList{iParProc} = iterList{iParProc}(iterList{iParProc} ~= iTilt); continue end end - - - - preBinStacks(TLT, tiltList{iTilt}, mapBackIter,1,... - samplingRate,... - PosControl2d,... - tiltWeight,... - flgMedianFilter); - end + + preBinStacks(TLT, ... + tiltList{iTilt}, ... + mapBackIter,... + 1,... + samplingRate,... + tiltWeight,... + flgMedianFilter,... + emc); + + end end -% All data is handled through disk i/o so everything unique created in the -% parfor is also destroyed there as well. -parfor iGPU = 1:nGPUs% -%for iGPU = 1:nGPUs - gpuDevice(gpuList(iGPU)); - % Loop over each tilt - for iTilt = iterList{gpuList(iGPU)} - +% All data is handled through disk i/o so everything unique created in the +parfor iParProc = 1:nParProcesses + % for iParProc = 1:nParProcesses % + iGPU = mod(iParProc,nGPUs); +% for iGPU = 1:nGPUs % + + % for iGPU = 1:nGPUs + gpuDevice(iGPU+1); + % Loop over each tilt + for iTilt = iterList{iParProc} + slab_list = {}; - - if (recWithoutMat) - if (loadSubTomoMeta) - nTomos = 1; + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', tiltList{iTilt}, mapBackIter + 1 ); + TLT = load(TLTNAME); + fprintf('iParProc %d and iTilt %d using TLT %s\n', iParProc, iTilt, TLTNAME); + + if (recon_for_subTomo || recon_for_templateMatching) + if (recon_for_subTomo) + % List of all possible tomos, some may be "in-active" since this is post-template matching + tomoList = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).tomoList; else - % templaterch - nTomos = nTomosPerTilt{iTilt}; - iCoords = recGeom{iTilt}; + % List of all possible tomos, all are "active" since this is pre-template matching + tomoList = tiltTomoList{iTilt}; end - else - nTomos = masterTM.mapBackGeometry.(tiltList{iTilt}).nTomos; - iCoords = masterTM.mapBackGeometry.(tiltList{iTilt}).coords; - % FIXME + % else tomoCPR, nTilts = 1 and tomoList is already set end - - if (recWithoutMat && ~loadSubTomoMeta) || ~recWithoutMat - targetSizeY = diff(floor(iCoords(:,2:3)),1,2)+1; - iCoords = iCoords ./ samplingRate; - iCoords(:,1:4) = floor(iCoords(:,1:4)); - iCoords(:,3) = iCoords(:,3) - (diff(floor(iCoords(:,2:3)),1,2)+1 - floor(targetSizeY./samplingRate)); + nTomos = length(tomoList); + + iCoords = cell(nTomos,1); + if (recon_for_templateMatching) + % tiltRecGeom is a cell with each value being a cell returned by multi_recGeom + % each element of this cell is a struct tomoCoords, we effectively create an anonymous struct + % accessed through iCoords. + for iCoordIdx = 1:nTomos + % each element of this cell is a struct tomoCoords + iCoords{iCoordIdx} = tiltRecGeom{iTilt}{iCoordIdx}; + end + else + for iCoordIdx = 1:nTomos + % each element of this cell is a struct tomoCoords + iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iCoordIdx}); + end end - iTomoList = cell(nTomos,1); - % For now, since the tilt geometry is not necessarily updated (it is manual) - % in the masterTM, check that newer (possible perTilt refined) data is - % not present. - try - % make sure there isn't a refined version first. - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',tiltList{iTilt},mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using refined TLT %s\n', TLTNAME); - catch - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - end - - - if (~recWithoutMat) - % Get all the tomogram names that belong to a given tilt-series. - nTomos = 0; - alreadyMade = 0; - for iTomo = 1:length(tomoList) - if strcmp(tiltList{iTilt},masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) - iTomoList{nTomos+1} = tomoList{iTomo}; - nTomos = nTomos + 1; + if samplingRate > 1 + % For now, we are only using the alt_cache for the tomos + fullStack = sprintf('%aliStacks/%s_ali%d.fixed', tiltList{iTilt}, mapBackIter + 1); + alt_cache = emc.alt_cache; + inputStack = EMC_setCacheForFile(alt_cache, sprintf('cache/%s_ali%d_bin%d.fixed', tiltList{iTilt}, mapBackIter + 1, samplingRate)); + % Check if the file exists using the cache selection logic + if ~emc_check_for_valid_image_file(inputStack) + % BH_multi_loadOrBin will create the file in cache/, but we need to move it to the selected cache + default_cache_path = sprintf('cache/%s_ali%d_bin%d.fixed', tiltList{iTilt}, mapBackIter + 1, samplingRate); + BH_multi_loadOrBin(fullStack, samplingRate, 2, false); + % If the selected cache is different from default, move the file + if ~strcmp(inputStack, default_cache_path) + % Ensure the target directory exists + inputDir = fileparts(inputStack); + if ~exist(inputDir, 'dir') + system(sprintf('mkdir -p %s', inputDir)); + end + % Move the file to the selected cache location + system(sprintf('mv %s %s', default_cache_path, inputStack)); end - % The order of tomo num could be off but only if all are present do we - % skip. - if (bh_global_turn_on_phase_plate(1)) - filtered = '_filtered'; - else - filtered = ''; - end - checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', ... - tiltList{iTilt},iTomo,samplingRate,filtered); - if exist(checkRecon, 'file') - fprintf('found %s to already exits\n',checkRecon); - alreadyMade = alreadyMade +1; - end - - end - - if alreadyMade == nTomos - fprintf('All tomos 1-%d found to exist for tilt-series %s\n',nTomos,tiltList{iTilt}); - continue end + else + inputStack = sprintf('aliStacks/%s_ali%d.fixed', tiltList{iTilt}, mapBackIter + 1); end - - - if samplingRate > 1 - fullStack = sprintf('%aliStacks/%s_ali%d.fixed', ... - tiltList{iTilt},mapBackIter+1); - inputStack = sprintf('cache/%s_ali%d_bin%d.fixed',... - tiltList{iTilt},mapBackIter+1,samplingRate); - if ~exist(inputStack, 'file') - % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s > /dev/null',samplingRate,fullStack,inputStack); - % % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s ',samplingRate,fullStack,inputStack); - % - % system(binCMD); - BH_multi_loadOrBin(fullStack,-1.*samplingRate,2); - - end - else - inputStack = sprintf('aliStacks/%s_ali%d.fixed',... - tiltList{iTilt},mapBackIter+1); - end - -% system(sprintf('header %s',inputStack)); - % iHeader = MRCImage(inputStack,0); - % STACK = gpuArray(single(getVolume(iHeader))); - - maskedStack = single(getVolume(MRCImage(inputStack))); - - if (recWithoutMat) - if (reconstructionParameters(1) && loadSubTomoMeta) - NX = size(maskedStack,1); - NY = size(maskedStack,2); - -% NY = size(maskedStack,2)-1; - NZ = floor(reconstructionParameters(1)) - maxZ = NZ; - iCoords = [NX,0,NY-1,NZ,0,0]; - tomoNumber = 1; - else - [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... - samplingRate, cycleNumber,... - 0,1); - end - else - [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... - samplingRate, cycleNumber,... - 0,1); - end - - if ( flg2dCTF || recWithoutMat && loadSubTomoMeta) - nSections = 1; - ctf3dDepth = maxZ * 10 ^ -9; + maskedStack = OPEN_IMG('single', inputStack); + + if (recon_for_subTomo) + [ ~, specimen_NZ_nm, ~ ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... + tomoList, nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + 0,1); else - dampeningMax = 0.90; - [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... - TLT(1,17),TLT(1,18),TLT(1,15), ... - 2048, TLT(1,19), ... - resTarget,maxZ*10, ... - dampeningMax,CYCLE); - fprintf('\n\nCalculated a ctfDepth of %2.2f nm for %s\n\n',ctf3dDepth*10^9,tiltList{iTilt}); - if (ctf3dDepth > max_ctf3dDepth) - ctf3dDepth = max_ctf3dDepth; - fprintf('Calculated ctfDepth exceeds user specified max, so actually using a max_ctfDepth of %2.2f nm\n',ctf3dDepth*10^9); - end - % sections centered at 0, which for now is also supposed to coincide with - % the mean defocus determination, although this could be corrected using - % knowledge of particle positions given assurance that particles are the - % primary source of signal (and not carbon for example). - nSections = ceil(maxZ/(ctf3dDepth*10^9)); - % max odd number - nSections = nSections + ~mod(nSections,2); + [ ~, specimen_NZ_nm, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... + tomoList,nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + 0,1); + end + + % TODO: for very thick specimen, this may be preventing the avg from getting to high enough + % resolution to be useful. So far, this is only optimized on in vitro samples. + dampeningMax = 0.90; + + [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... + TLT(1,17),TLT(1,18),abs(TLT(1,15)), ... + 2048, TLT(1,19), ... + resTarget,specimen_NZ_nm*10, ... + dampeningMax,CYCLE); + fprintf('\n\nCalculated a ctfDepth of %2.2f nm for %s\n\n',ctf3dDepth*10^9,tiltList{iTilt}); + if (ctf3dDepth > emc.max_ctf3dDepth) + ctf3dDepth = emc.max_ctf3dDepth; + fprintf('Calculated ctfDepth exceeds user specified max, so actually using a max_ctfDepth of %2.2f nm\n',ctf3dDepth*10^9); end - fprintf('with %3.3f nm sections, correcting %d tilt-series\n',ctf3dDepth*10^9,nSections); + % sections centered at 0, which for now is also supposed to coincide with + % the mean defocus determination, although this could be corrected using + % knowledge of particle positions given assurance that particles are the + % primary source of signal (and not carbon for example). + n_slabs_to_reconstruct = ceil(specimen_NZ_nm/(ctf3dDepth*10^9)); + % max odd number + n_slabs_to_reconstruct = n_slabs_to_reconstruct + ~mod(n_slabs_to_reconstruct,2); + fprintf('with %3.3f nm sections, correcting %d tilt-series\n', ctf3dDepth*10^9, n_slabs_to_reconstruct); - % For each tomo create a list of slices that are to be reconstructed + % For each tomo create a list of slices that are to be reconstructed % for every section section. - [ sectionList ] = calcTomoSections(iCoords, tomoNumber,pixelSize, ... - nSections,tiltList{iTilt}, ctf3dDepth); - + [ slab_list ] = calc_slab_boundaries(iCoords, tomoList, emc.pixel_size_angstroms, n_slabs_to_reconstruct, tiltList{iTilt}, ctf3dDepth, samplingRate); + - if (recWithoutMat) - avgZ = 0; - surfaceFit = 0; - + if (recon_for_subTomo) + [ avgZ, specimen_NZ_nm, surfaceFit ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... + tomoList,nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + slab_list, 0); else - - [ avgZ, maxZ, tomoNumber, surfaceFit ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... - samplingRate, cycleNumber,... - sectionList,0); - - + avgZ = 0; + surfaceFit = 0; end if ( shiftDefocusOrigin ) @@ -631,289 +468,266 @@ fprintf('Using sample origin as the defocus origin\n'); fprintf('If you want to use the COM of subTomos, set shiftDefocusToSubTomoCOM=1\n'); end - - - - - - % Correct a tilt series for earch section which requires writing each to - % disk for use of IMOD. - if (mapBackIter) - tiltErrorFile = sprintf('mapBack%d/%s_ali%d_ctf.beamTiltError', ... - mapBackIter,tiltList{iTilt}, mapBackIter); - try - tiltError = load(tiltErrorFile); - if numel(tiltError) ~= 1 - error('tiltError should be a single number in degrees.\n'); - else - fprintf('\nUsing %f degrees for beam tilt error.\n',tiltError) - end - - catch - fprintf('\nTiltErrorFile %s not found.\n', tiltErrorFile); - fprintf('\nUsing 0 degrees for beam tilt error.\n',tiltErrorFile) - tiltError = 0; - end - - - else - tiltError = 0; - end - for iSection = 1:nSections - + first_slab = true(nTomos,1); + for iSection = 1:n_slabs_to_reconstruct defFitFull = ''; preCombDefocus = 0; if (mapBackIter) - defFitFull = sprintf('mapBack%d/%s_ali%d_ctf.defFidFull',mapBackIter, ... - tiltList{iTilt},mapBackIter); + defFitFull = sprintf('mapBack%d/%s_ali%d_ctf.defFidFull',mapBackIter, tiltList{iTilt}, mapBackIter); if exist(defFitFull,'file') preCombDefocus = load(defFitFull); - fprintf('3dCTF using pre calc combined per tilt defocus %s\n',defFitFull); - else - fprintf('Did not find %s\n!!',defFitFull); + fprintf('3dCTF using pre calc combined per tilt defocus %s\n', defFitFull); end end - - if (PosControl2d) - correctedStack = maskedStack; - else - - - % I would have thought the global would be recognized, but it looks - % like there is something odd about its use with a parfor loop - % FIXME, when setting up the iterator, make clean copies for each - % worker that are local in scope.e - -% This is slated to be deleted, just leave pre erasure to happen in ctf estimate/update -% if ~(flgEraseBeads_aferCTF) -% scalePixelsBy = samplingRate; -% maskedStack = BH_eraseBeads(maskedStack,eraseRadius, tiltList{iTilt}, scalePixelsBy,mapBackIter,sortrows(TLT,1)); -% end - - [ correctedStack ] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,pixelSize,maskedStack,... - maxZ*10/pixelSize,flgDampenAliasedFrequencies,... + + % I would have thought the global would be recognized, but it looks + % like there is something odd about its use with a parfor loop + % FIXME, when setting up the iterator, make clean copies for each + % worker that are local in scope.e + + [ correctedStack ] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... + avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... + specimen_NZ_nm*10/emc.pixel_size_angstroms,... preCombDefocus,samplingRate,... applyExposureFilter,surfaceFit,... - useSurfaceFit,invertDose,... - bh_global_turn_on_phase_plate,... + useSurfaceFit,... + phase_plate_mode,... filterProjectionsForTomoCPRBackground,... - flgWhitenPS); - end + emc.whitenPS, ... + flip_defocus_offset, ... + flip_tilt_offset); % Write out the stack to the cache directory as a tmp file - + if (flgEraseBeads_aferCTF) scalePixelsBy = samplingRate; correctedStack = BH_eraseBeads(correctedStack,eraseRadius, tiltList{iTilt}, scalePixelsBy,mapBackIter,sortrows(TLT,1)); end - - - outputStack = sprintf('%s/%s_ali%d_%d.fixed', ... - tmpCache,tiltList{iTilt},mapBackIter+1,iSection) - SAVE_IMG(correctedStack,outputStack,pixelSize); + + + outputStack = sprintf('%s/%s_ali%d_%d.fixed', tmpCache, tiltList{iTilt}, mapBackIter+1, iSection); + SAVE_IMG(correctedStack, {outputStack, 'half'}, emc.pixel_size_angstroms); correctedStack = []; - - % Loop over tomos reconstructing section and appending a file to - for iT = 1:nTomos - - thisTomo = tomoNumber(iT); - - if any(sectionList{iT}(iSection,:)+9999) - - + + % Loop over tomos reconstructing section and appending a file to + for iTomo = 1:nTomos + + if (slab_list{iTomo}(iSection,1)) + if (recon_for_templateMatching) + this_tomo_idx = iTomo; + else + this_tomo_idx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + end - reconName = sprintf('%s/%s_ali%d_%d_%d.rec', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); + reconName = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, this_tomo_idx, iSection); + - if (loadSubTomoMeta) - if (recWithoutMat) - TA = sortrows(masterTM.tiltGeometry.(tomoList{1}),1); - else - TA = sortrows(masterTM.tiltGeometry.(sprintf('%s_%d',tiltList{iTilt},thisTomo)),1); - end + if (recon_for_subTomo) + TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{iTomo}),1); TA = TA(:,4); - else + end + + + if (recon_for_templateMatching) if (mapBackIter) - TA = load(sprintf('%smapBack%d/%s_ali%d_ctf.tlt',CWD,mapBackIter,tiltList{iTilt},... - mapBackIter)); + % FIXME: I don't think this block should work, it should only be the tilt angles! + error('THis block should not be reached.') + TA = load(sprintf('%smapBack%d/%s_ali%d_ctf.tlt', CWD, mapBackIter, tiltList{iTilt}, mapBackIter)); else - TA = load(sprintf('%sfixedStacks/%s.tlt',CWD,tiltList{iTilt})); - end + TA = load(sprintf('%sfixedStacks/%s.tlt', CWD, tiltList{iTilt})); + end end - - rawTLT = sprintf('cache/%s_%d.rawtlt',tiltList{iTilt},thisTomo); + + alt_cache = emc.alt_cache; + rawTLT = EMC_setCacheForFile(alt_cache, sprintf('cache/%s.rawtlt', tomoList{iTomo})); rawTLT_file = fopen(rawTLT, 'w'); fprintf(rawTLT_file,'%f\n', TA'); fclose(rawTLT_file); if (mapBackIter) - - LOCAL = sprintf('%smapBack%d/%s_ali%d_ctf.local',CWD,mapBackIter,tiltList{iTilt}, ... - mapBackIter); - else - LOCAL = sprintf('%sfixedStacks/%s.local',CWD,tiltList{iTilt}); + LOCAL = sprintf('%smapBack%d/%s_ali%d_ctf.local', CWD, mapBackIter, tiltList{iTilt}, mapBackIter); + else + LOCAL = sprintf('%sfixedStacks/%s.local', CWD, tiltList{iTilt}); end - - % Put a local copy if using a nondefault cache -% if ( flgCleanCache ) -% sprintf('cp %s/%s %s/%s',CWD,rawTLT,tmpCache,rawTLT) -% sprintf('cp %s/%s %s/%s',CWD,LOCAL,tmpCache,LOCAL) -% system(sprintf('cp %s/%s %s/%s',CWD,rawTLT,tmpCache,rawTLT)); -% system(sprintf('cp %s/%s %s/%s',CWD,LOCAL,tmpCache,LOCAL)); -% rawTLT = sprintf('%s/%s',tmpCache,rawTLT) -% LOCAL = sprintf('%s/%s',tmpCache,LOCAL) -% -% end - - fprintf('Local file %s\n',LOCAL); - if exist(LOCAL,'file') flgLocal = 1; else fprintf('Did not find local alignment information at %s\n',LOCAL); flgLocal = 0; end + + % round down and then we'll add any extra needed to the final chunk + tiltChunkSize = floor(iCoords{iTomo}.NY ./ samplingRate ./ emc.n_tilt_workers); + % This shoulid never happen, but to be safe + if (emc.n_tilt_workers > floor(iCoords{iTomo}.NY ./ samplingRate)) + error('n_tilt_workers is greater than the number of slices in the tilt series'); + end - % hangover from slab padding, remove later. - padRec = 0; + y_i = floor(iCoords{iTomo}.y_i ./ samplingRate); + y_f = floor(iCoords{iTomo}.y_f ./ samplingRate); - nTiltWorkers = 2; - nTotalSlices = (iCoords(thisTomo,3)-iCoords(thisTomo,2)+1); - tiltChunkSize = ceil(nTotalSlices/nTiltWorkers); - tiltChunks = iCoords(thisTomo,2):tiltChunkSize:iCoords(thisTomo,3); - tiltChunks(end) = iCoords(thisTomo,3); - totalSlices = [tiltChunks(1),tiltChunks(end)]; - - - rCMD = sprintf(['tilt %s %s -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... - '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... - super_sample, expand_lines, ... - outputStack, reconName, rawTLT, gpuList(iGPU), ... - iCoords(thisTomo,1),floor(sectionList{iT}(iSection,5))+2*padRec,... - iCoords(thisTomo,5),sectionList{iT}(iSection,6)); - + tiltChunks = y_i:tiltChunkSize:y_f; + tiltChunks(end) = y_f; + % Imod expects zero indexed slices + tiltChunks = tiltChunks - 1; + totalSlices = [tiltChunks(1),tiltChunks(end)]; + % Make sure we didn't go OOB on the first chunk + if (tiltChunks(1) < 0) + tiltChunks(1) = 0; + n_slices_in_Y = tiltChunks(end) - tiltChunks(1) + 1; + end - % Explicitly set Radial to Nyquist + rCMD = sprintf(['tilt %s %s -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... + '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... + super_sample, ... + expand_lines, ... + outputStack, ... + reconName, ... + rawTLT, ... + iGPU, ... + floor(iCoords{iTomo}.NX ./ samplingRate),... % WIDTH = NX + floor(round(slab_list{iTomo}(iSection,5))), ... % THICKNESS = NZ + -iCoords{iTomo}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X is negative dX which describes the vector from the specimen origin to the tomo origin + slab_list{iTomo}(iSection,6)); + + + reconScaling = 1; + % Explicitly set Radial to Nyquist if (flgLocal) - rCMD = [rCMD sprintf('-LOCALFILE %s -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d',LOCAL,reconScaling)]; + rCMD = [rCMD sprintf('-LOCALFILE %s -RADIAL 0.5,.05 -MODE 12 -SCALE 0,%d', LOCAL, reconScaling)]; else - rCMD = [rCMD sprintf('-RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d',reconScaling)]; - end + rCMD = [rCMD sprintf('-RADIAL 0.5,.05 -MODE 12 -SCALE 0,%d', reconScaling)]; + end - system(sprintf('rm -f %s.sh',reconName)); + if isfile(sprintf('%s.sh',reconName)) + system(sprintf('rm %s.sh',reconName)); + end recScript = fopen(sprintf('%s.sh',reconName),'w'); fprintf(recScript,'#!/bin/bash\n\n'); - fprintf(recScript,'%s -SLICE -1,-1 -TOTALSLICES %d,%d\n',rCMD,totalSlices); - for iRecSec = 1:nTiltWorkers-1 - if iRecSec < nTiltWorkers -1 - iShift = 1; - else + fprintf(recScript,'%s -SLICE -1,-1 -TOTALSLICES %d,%d\n', rCMD, totalSlices); + + iShift = 1; + for iChunk = 1:length(tiltChunks)-1 + if (iChunk == length(tiltChunks)-1) iShift = 0; - end % /dev/null - fprintf(recScript,'%s -SLICE %d,%d -TOTALSLICES %d,%d > /dev/null &\n',rCMD, ... - tiltChunks(iRecSec),... - tiltChunks(iRecSec+1)-iShift,... - totalSlices); + end + fprintf(recScript,'%s -SLICE %d,%d -TOTALSLICES %d,%d > /dev/null &\n', ... + rCMD, ... + tiltChunks(iChunk),... + tiltChunks(iChunk+1) - iShift,... + totalSlices); end fprintf(recScript,'\n\nwait\n\n'); fclose(recScript); - system(sprintf('chmod a=wrx %s.sh',reconName)); - - [recError,~] = system(sprintf('%s.sh > /dev/null ',reconName)); % /dev/null + pause(1); + system(sprintf('chmod a=wrx %s.sh', reconName)); + + [recError,~] = system(sprintf('%s.sh > /dev/null', reconName)); % /dev/null if (recError) system(sprintf('%s.sh',reconName)); - error('\n\nerror during reconstruction %s\n\n',reconName); + error('\n\nerror during reconstruction %s\n\n', reconName); end % Z coords (y in this orientation) are decreasing into the % monitor. For symmetrical padding this doesn't matter, but keep % in mind. /dev/null - trimCMD = sprintf('trimvol -rx -y %d,%d %s.TMPPAD %s > /dev/null ' , ... - padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName); -% % % trimCMD = sprintf('newstack -fromone -secs %d-%d %s.TMPPAD %s > /dev/null', ... -% % % padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) + trimCMD = sprintf('trimvol -mode 12 -rx -y %d,%d %s.TMPPAD %s > /dev/null ' , ... + 1,floor(round(slab_list{iTomo}(iSection,5))), reconName, reconName); [msg,~]= system(trimCMD); if (msg) - fprintf('%d from trimCMD\n',msg) - trimCMDPrintError = sprintf('trimvol -rx -y %d,%d %s.TMPPAD %s', ... - padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) -% % % trimCMDPrintError = sprintf('newstack -fromone -secs %d-%d %s.TMPPAD %s', ... -% % % padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) + fprintf('%d from trimCMD\n',msg) + trimCMDPrintError = sprintf('trimvol -mode 12 -rx -y %d,%d %s.TMPPAD %s', ... + 1+slab_list{iTomo}(iSection,3),floor(round(slab_list{iTomo}(iSection,5)))-slab_list{iTomo}(iSection,4), reconName, reconName); system(trimCMDPrintError); - end - system(sprintf('rm %s.TMPPAD', reconName)); - % fprintf([trimCMD ' \n']) - - + error('error during trimvol'); + end + system(sprintf('rm %s.TMPPAD', reconName)); end - - end % end loop over tomos for this section - system(sprintf('rm %s',outputStack)); - - end % end loop over sections - - deltaZ = []; - evalMask = []; - maskedStack = []; - - for iT = 1:nTomos - thisTomo = tomoNumber(iT); + end % end loop over tomos for this section - if (bh_global_turn_on_phase_plate(1)) - reconNameFull = sprintf('cache/%s_%d_bin%d_filtered.rec', ... - tiltList{iTilt},thisTomo,samplingRate); - elseif reconstructionParameters(1) - - reconNameFull = sprintf('cache/%s_%d_bin%d_backgroundEst.rec', ... - tiltList{iTilt},thisTomo,samplingRate); + if isfile(outputStack) + system(sprintf('rm %s',outputStack)); + end + end % end loop over sectionsF() + + deltaZ = []; + evalMask = []; + maskedStack = []; + + for iTomo = 1:nTomos + % Note that phase_plate_mode could be true for any of the recon_for_stage bools, so it must + % be checked first. + alt_cache = emc.alt_cache; + if (phase_plate_mode) + reconNameFull = EMC_setCacheForFile(alt_cache, sprintf('cache/%s_bin%d_filtered.rec', tomoList{iTomo}, samplingRate)); else - reconNameFull = sprintf('cache/%s_%d_bin%d.rec', ... - tiltList{iTilt},thisTomo,samplingRate); + reconNameFull = EMC_setCacheForFile(alt_cache, sprintf('cache/%s_bin%d.rec', tomoList{iTomo},samplingRate)); end - - recCMD = 'newstack -fromone'; - for iSection = 1:nSections - reconName = sprintf('%s/%s_ali%d_%d_%d.rec', ... - tmpCache, tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - - if any(sectionList{iT}(iSection,:)+9999) - recCMD = [recCMD,sprintf(' -secs 1-%d %s', ... - floor(sectionList{iT}(iSection,5)), ... - reconName)]; - else + fprintf('in ctf3d reconNameFull is %s\n\n',reconNameFull); + - fprintf('no info for section %d for tomo %d\n',iSection,thisTomo); + % Get the total number of sections for this tomo + n_total_sections = 0; + for iSection = 1:n_slabs_to_reconstruct + if(slab_list{iTomo}(iSection,1)) + n_total_sections = n_total_sections + 1; end end - - system([recCMD, sprintf(' %s > /dev/null ',reconNameFull)]); %/dev/null - - - for iSection = 1:nSections - cleanUp3 = sprintf('rm %s/%s_ali%d_%d_%d.rec', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - system(cleanUp3); - cleanUp4 = sprintf('rm %s/%s_ali%d_%d_%d.rec.sh', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - system(cleanUp4); - + + if (n_total_sections == 0) + fprintf('no sections for tomo %d\n',iTomo); + continue end - + file_of_outputs = sprintf('%s.filelist',reconNameFull); + recombineCMD = fopen(file_of_outputs,'w'); + fprintf(recombineCMD,'%d\n', n_total_sections); + + cleanup3 = sprintf('rm -f %s',file_of_outputs); + % if (use_inverted_newstack) + % slab_order = n_slabs_to_reconstruct:-1:1; + % else + slab_order = 1:n_slabs_to_reconstruct; + % end + % for iSection = 1:n_slabs_to_reconstruct + for iSection = slab_order + if (slab_list{iTomo}(iSection,1)) + if (recon_for_templateMatching) + this_tomo_idx = iTomo; + else + this_tomo_idx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + end + this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, this_tomo_idx, iSection); + cleanup3 = sprintf('%s %s',cleanup3,this_slab); + fprintf(recombineCMD, '%s\n', this_slab); + fprintf(recombineCMD, '%d-%d\n',1+slab_list{iTomo}(iSection,3),floor(round(slab_list{iTomo}(iSection,5)))-slab_list{iTomo}(iSection,4)); + end + end + fclose(recombineCMD); + pause(1); + recCMD = sprintf('newstack -mode 12 -fromone -FileOfInputs %s -output %s\n', file_of_outputs, reconNameFull); + + [err_msg, ~] = system(sprintf('%s > /dev/null ',recCMD)); %/dev/null + if (err_msg) + fprintf('error during recombination %s\n',reconNameFull); + system(recCMD); + error('error during recombination'); + end + + system(cleanup3); end % end of recombination loop - - maskedStack = []; - - end % end of loop over tilt-series + + maskedStack = []; + + end % end of loop over tilt-series end % end of parfor over gpus if (flgCleanCache) @@ -921,7 +735,7 @@ checkDir = dir(tmpCache); if isempty(checkDir) fprintf('not removing the temp cache because it did not eval with dir\n'); - else + else cleanItUp = sprintf('rm %s/*',tmpCache); system(cleanItUp); end @@ -929,264 +743,165 @@ end -% % % function [STACK, evalMask, deltaZ] = loadAndMaskStack(TLT, STACK_PRFX, ... -% % % mapBackIter,maxZpix,... -% % % samplingRate,... -% % % PosControl2d,... -% % % tiltWeight,flgWhitenPS,pixelSize) -% % % -% % % if (PosControl2d) -% % % prefix = 'ctf'; -% % % suffix = '_ctf' -% % % else -% % % prefix = 'ali'; -% % % suffix = ''; -% % % end -% % % -% % % if samplingRate > 1 -% % % fullStack = sprintf('%sStacks/%s_ali%d%s.fixed', ... -% % % prefix,STACK_PRFX,mapBackIter+1,suffix); -% % % inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed',... -% % % STACK_PRFX,mapBackIter+1,suffix,samplingRate); -% % % if ~exist(inputStack, 'file') -% % % % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s > /dev/null',samplingRate,fullStack,inputStack); -% % % %% binCMD = sprintf('newstack -bin %d -antialias 6 %s %s ',samplingRate,fullStack,inputStack); -% % % % -% % % % system(binCMD); -% % % BH_multi_loadOrBin(fullStack,-1.*samplingRate,2); -% % % -% % % end -% % % else -% % % inputStack = sprintf('%sStacks/%s_ali%d%s.fixed',... -% % % prefix,STACK_PRFX,mapBackIter+1,suffix) -% % % end -% % % -% % % system(sprintf('header %s',inputStack)); -% % % % iHeader = MRCImage(inputStack,0); -% % % % STACK = gpuArray(single(getVolume(iHeader))); -% % % -% % % STACK = single(getVolume(MRCImage(inputStack))); -% % % -% % % % iHeader = getHeader(iHeader); -% % % % iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... -% % % % iHeader.cellDimensionY/iHeader.nY, ... -% % % % iHeader.cellDimensionZ/iHeader.nZ]; -% % % -% % % -% % % [d1,d2,d3] = size(STACK); -% % % nPrjs = d3; -% % % -% % % useableArea = [d1-128,d2-128,maxZpix]; -% % % -% % % -% % % -% % % [evalMask, deltaZ ] = BH_multi_projectionMask( [d1,d2,d3;useableArea], TLT, 'cpu' ); -% % % -% % % -% % % -% % % -% % % % Local normalization doesn't address any large scale gradients in the -% % % % images. Do a simple high pass over the lowest 7 frequencyBinns -% % % bandNyquist = BH_bandpass3d([d1,d2,1],0,0,1,'GPU','nyquistHigh'); -% % % -% % % taperMask = gpuArray(fspecial('gaussian',[9,9],1.5)); -% % % -% % % -% % % for iPrj = 1:nPrjs -% % % -% % % -% % % iEvalMask = gpuArray(evalMask(:,:,TLT(iPrj,1))); -% % % fprintf('iPrj %d size %d, %d\n',iPrj,size(STACK,3),TLT(iPrj,1)); -% % % iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); -% % % -% % % % iMask = convn(single(iEvalMask),taperMask,'same'); -% % % -% % % -% % % iProjection = iProjection - mean(iProjection(iEvalMask)); -% % % if ( flgWhitenPS(1) ) -% % % %fprintf('confirm whitening PS\n.'); -% % % flgWhitenPS -% % % [iProjection,~] = BH_whitenNoiseSpectrum(iProjection,'',[600,14,pixelSize,160],flgWhitenPS); -% % % mean2(iProjection) -% % % else -% % % iProjection = real(ifftn(fftn(iProjection).*bandNyquist)); -% % % end -% % % % iProjection = real(ifftn(fftn(iProjection.*iMask).*bandNyquist)); -% % % -% % % -% % % inFin = ~(isfinite(iProjection)); nInf = sum(inFin(:)); -% % % if (nInf) -% % % % fprintf('Removing %d (%2.4f) inf from prj %d\n',nInf,100*nInf/numel(iProjection),TLT(iPrj,1)); -% % % iProjection(inFin) = 0; -% % % end -% % % -% % % iRms = rms(iProjection(iEvalMask)); -% % % outliers = (iProjection > 6 * iRms); nOutliers = sum(outliers(:)); -% % % tiltScale = 1- ( abs(sind(TLT(iPrj,4))).* tiltWeight(1)); -% % % if (nOutliers) -% % % -% % % % iProjection(outliers) = sign(iProjection(outliers)).*3.*iRms.*((rand(size(iProjection(outliers)))./2)+0.5); -% % % iProjection(outliers) = 6.*iRms.* (rand(size(iProjection(outliers)))-0.5); -% % % iProjection = iProjection ./ ( rms(iProjection(iEvalMask)) ./ tiltScale); -% % % else -% % % iProjection = iProjection ./ ( iRms ./ tiltScale); -% % % end -% % % -% % % % if ( flgWhitenPS ) -% % % % %fprintf('confirm whitening PS\n.'); -% % % % [iProjection,~] = BH_whitenNoiseSpectrum(iProjection,'',pixelSize,1); -% % % % end -% % % -% % % if tiltWeight(2) -% % % % I don't think this makes sense, but test keeping the power constant -% % % % after application of the exposure filter. -% % % iProjection = fftn(iProjection); -% % % iPower = sum(abs(iProjection(:))); -% % % iProjection = iProjection .* iExpFilter; -% % % STACK(:,:,TLT(iPrj,1)) = gather(single(real(ifftn(iProjection.* ... -% % % (iPower./sum(abs(iProjection(:)))))))); -% % % else -% % % STACK(:,:,TLT(iPrj,1)) = gather(iProjection);%gather(single(real(ifftn(fftn(iProjection) .* iExpFilter)))); -% % % end -% % % % STACK(:,:,TLT(iPrj,1)) = gather(single(iMask.*real(ifftn(fftn(iProjection) .* iExpFilter)))); -% % % % STACK(:,:,TLT(iPrj,1)) = gather(single(iProjection.*iMask)); -% % % clear iProjection iMask iExpFilter iEvalMask -% % % end -% % % -% % % -% % % clear bandNyquist iMask exposureFilter iProjection lowRMSMAsk -% % % -% % % -% % % end - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -function [] = preBinStacks(TLT, STACK_PRFX, mapBackIter,usableArea,... - samplingRate,... - PosControl2d,... - tiltWeight,... - flgMedianFilter) - - - -if (PosControl2d) - prefix = 'ctf'; - suffix = '_ctf'; -else - prefix = 'ali'; - suffix = ''; -end - - -fullStack = sprintf('%sStacks/%s_ali%d%s.fixed', ... - prefix,STACK_PRFX,mapBackIter+1,suffix); -inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed',... - STACK_PRFX,mapBackIter+1,suffix,samplingRate); -if ~exist(inputStack, 'file') - BH_multi_loadOrBin(fullStack,-1.*samplingRate,2,flgMedianFilter); +function [] = preBinStacks(TLT, ... + STACK_PRFX, ... + mapBackIter, ... + usableArea,... + samplingRate,... + tiltWeight,... + flgMedianFilter,... + emc) + + + + +prefix = 'ali'; +suffix = ''; + +% TODO: this could all be in loadOrBin +fullStack = sprintf('%sStacks/%s_ali%d%s.fixed', prefix,STACK_PRFX, mapBackIter+1, suffix); +alt_cache = emc.alt_cache; +inputStack = EMC_setCacheForFile(alt_cache, sprintf('cache/%s_ali%d%s_bin%d.fixed', STACK_PRFX, mapBackIter+1, suffix, samplingRate)); +% Check if the file exists using the cache selection logic +if ~emc_check_for_valid_image_file(inputStack) + % BH_multi_loadOrBin will create the file in cache/, but we need to move it to the selected cache + default_cache_path = sprintf('cache/%s_ali%d%s_bin%d.fixed', STACK_PRFX, mapBackIter+1, suffix, samplingRate); + BH_multi_loadOrBin(fullStack, samplingRate, 2, false); + % If the selected cache is different from default, move the file + if ~strcmp(inputStack, default_cache_path) + % Ensure the target directory exists + inputDir = fileparts(inputStack); + if ~exist(inputDir, 'dir') + system(sprintf('mkdir -p %s', inputDir)); + end + % Move the file to the selected cache location + system(sprintf('mv %s %s', default_cache_path, inputStack)); + end end end -function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, pixelSize,... - nSections,tiltName, ctf3Depth) +function [ slab_list ] = calc_slab_boundaries(iCoords, tomoList, pixel_size_angstroms, n_slabs_to_reconstruct, tiltName, ctf_3d_depth_si, samplingRate, use_inverted_newstack) -nTomos = length(tomoNumber); -sectionList = cell(nTomos,1); + %%% This function is to produce a list of z indices, starting from 1, to pass to imod for tilt based reconstruction +nTomos = length(tomoList); +slab_list = cell(nTomos,1); for iTomo = 1:nTomos % min and max in absolute pixels min and max from 1:nZrecon - sectionList{iTomo} = zeros(nSections,6); + slab_list{iTomo} = zeros(n_slabs_to_reconstruct,6); end % With rounding this could end up a bit short except the top and bottom are both % half a section larger than minimally needed. -nSec = floor(ctf3Depth*10^10/pixelSize) ; -nSec = nSec + ~mod(nSec,2); -halfSec = (nSec-1)/2; +slab_size_pixels = floor(ctf_3d_depth_si * 10^10 / pixel_size_angstroms); +slab_size_pixels = slab_size_pixels + ~mod(slab_size_pixels,2); +oS = emc_get_origin_index(slab_size_pixels); -for iT = 1:length(tomoNumber) - iTomo = tomoNumber(iT); +for iTomo = 1:nTomos % Origin + originshift - -1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6), - reconRange = floor([-1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6),0]); - reconRange(2) = reconRange(1) + iCoords(iTomo,4) - 1; - nZ = 1; - flgFirstSec = 1; + if ~(iCoords{iTomo}.is_active) + % Nothing to do, the first column in this row of slab_list is already zero, but we can set it + % explicitly in case the code changes in the future + slab_list{iTomo}(:,1) = 0; + continue; + end - for iSection = 1:nSections - - sectionCenter = ((nSections-1)/-2+(iSection-1))*(nSec-1); + tomo_origin_wrt_tilt_origin = iCoords{iTomo}.dZ_specimen_to_tomo ./ samplingRate; + tomo_origin_in_tomo_frame = emc_get_origin_index(iCoords{iTomo}.NZ ./ samplingRate); + + fraction_origin_shift = tomo_origin_wrt_tilt_origin - round(tomo_origin_wrt_tilt_origin); + wanted_NZ = ceil(iCoords{iTomo}.NZ./samplingRate); + + tomogram_lower_bound = floor((tomo_origin_wrt_tilt_origin - tomo_origin_in_tomo_frame)) + 1; + recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + wanted_NZ - 1; + % For each slab see if this tomogram has any sections in it + for iSlab = 1:n_slabs_to_reconstruct + slab_idx = ((n_slabs_to_reconstruct-1)/-2+(iSlab-1)); + slab_origin_in_specimen_frame = slab_idx * slab_size_pixels; - % Check that sectionCenter is within range - if sectionCenter + halfSec < reconRange(1) || ... - sectionCenter - halfSec > reconRange(2) - sectionList{iT}(iSection,:) = -9999; + slab_lower_bound = slab_origin_in_specimen_frame - oS + 1; + slab_upper_bound = slab_origin_in_specimen_frame + oS - 1; + slab_range = slab_lower_bound:slab_upper_bound; + + is_in_range = ismember(recon_range_z_in_specimen_frame, slab_range); + valid_indices = recon_range_z_in_specimen_frame(is_in_range); + + slab_list{iTomo}(iSlab,5) = length(valid_indices); + + if (slab_list{iTomo}(iSlab,5) > 0) + slab_list{iTomo}(iSlab,1) = 1; else - - if (sectionCenter - halfSec > 0) - sectionList{iT}(iSection,1) = max(sectionCenter - halfSec ,reconRange(1)); - if sectionList{iT}(iSection,1) ~= reconRange(1) - sectionList{iT}(iSection,1) = sectionList{iT}(iSection,1) +1; - end - elseif (sectionCenter - halfSec < 0) - sectionList{iT}(iSection,1) = max(sectionCenter - halfSec,reconRange(1)); - if sectionList{iT}(iSection,1) ~= reconRange(1) - sectionList{iT}(iSection,1) = sectionList{iT}(iSection,1) +1; - end - else - sectionList{iT}(iSection,1) = max(-halfSec,reconRange(1)); - end - - if (sectionCenter - halfSec > 0) - sectionList{iT}(iSection,2) = min(sectionCenter + halfSec,reconRange(2)); - elseif (sectionCenter - halfSec < 0) - sectionList{iT}(iSection,2) = min(sectionCenter + halfSec ,reconRange(2)); - else - sectionList{iT}(iSection,2) = min(halfSec,reconRange(2)); - end - - - % Check that first section starts in the correct place. If a small error, - % just shift the results, otherwise complain. - if (flgFirstSec) - if sectionList{iT}(iSection,1) ~= reconRange(1) - if abs(sectionList{iT}(iSection,1) - reconRange(1)) < 10 - sectionList{iT}(iSection,1) = reconRange(1); - fprintf('\n\nShifting first section %s_n%d\n\n',tiltName,iTomo); - else - error('section start %d is too far off from expected %d\n', ... - sectionList{iT}(iSection,1), reconRange(1)) - end - end - flgFirstSec = 0; + continue; + end + valid_region_origin = emc_get_origin_index(slab_list{iTomo}(iSlab,5)); + % This is a vector from the origin of the sample to the origin of the slab. + % The shift passed to imod-tilt moves the reconstructed area in the opposite sense. + % All slabs need to be shifted to the specimen origin (0) from tilts perspective, and then the are assembled into the final volume. + % This means a slab at Z > 0 needs to be shifted in the negative direction, which means supplying + % a shift that is also > 0, moving the volume "up" in the rotated coordinate system (imod -Z) + % I know ... this is a shit show. + slab_list{iTomo}(iSlab,2) = fraction_origin_shift; + + dZ_for_reconstructed_slab = (valid_indices(valid_region_origin) - fraction_origin_shift); + slab_list{iTomo}(iSlab,6) = dZ_for_reconstructed_slab; %dZ + end + + % Check to ensure we don't have any tiny slabs leftover, if so, merge them into a neighboring slab + biggest_slab = max(slab_list{iTomo}(:,5)); + for iSlab = 1:n_slabs_to_reconstruct + if (slab_list{iTomo}(iSlab,1) && (slab_list{iTomo}(iSlab, 5) / biggest_slab < 0.2)) + if (iSlab > 1 && slab_list{iTomo}(iSlab-1,1)) + delta = slab_list{iTomo}(iSlab,5); + slab_list{iTomo}(iSlab-1,5) = slab_list{iTomo}(iSlab-1,5) + delta; + slab_list{iTomo}(iSlab,1) = 0; + % we are adding slices from above the specimen in Z so the z shift is positive + slab_list{iTomo}(iSlab-1,6) = (slab_list{iTomo}(iSlab-1,6) + ceil(delta/2)); + elseif (iSlab < n_slabs_to_reconstruct && slab_list{iTomo}(iSlab+1,1)) + delta = slab_list{iTomo}(iSlab,5); + slab_list{iTomo}(iSlab+1,5) = slab_list{iTomo}(iSlab+1,5) + slab_list{iTomo}(iSlab,5); + slab_list{iTomo}(iSlab,1) = 0; + % we are adding slices from below the specimen in Z so the z shift is negative + slab_list{iTomo}(iSlab+1,6) = (slab_list{iTomo}(iSlab+1,6) - ceil(delta/2)); end - - secZ = sectionList{iT}(iSection,2) - sectionList{iT}(iSection,1); - sectionList{iT}(iSection,3) = nZ; - sectionList{iT}(iSection,4) = nZ + secZ; - sectionList{iT}(iSection,5) = secZ +1; - sectionList{iT}(iSection,6) = (secZ+1)./2 + sectionList{iT}(iSection,1); - nZ = nZ + secZ + 1; end - + end - % not a good solution, but not sure just yet why I'm getting some occasionally - % weird results. - if sectionList{iT}(iSection,5) < 3 - sectionList{iT}(iSection,:) = -9999; + % The only time we should have an even Z dimension now is if there is only one slab. if so, pad the top end for reconstruction and trim it off later + for iSlab = 1:n_slabs_to_reconstruct + slab_idx = ((n_slabs_to_reconstruct-1)/-2+(iSlab-1)); + + if (slab_list{iTomo}(iSlab,1) > 0 && mod(slab_list{iTomo}(iSlab,5),2) == 0) + slab_list{iTomo}(iSlab,5) = slab_list{iTomo}(iSlab,5) + 1; + slab_list{iTomo}(iSlab,4) = slab_list{iTomo}(iSlab,4) + 1; end - end % end loop over sections + end + + % % If the slab thickness is even, we need to account for the difference in definition of imod origin, this can happen at the boundaries + % % the origin in imod is -0.5 for even images, but we are applying a shift to the image, so we add +0.5 + % for iSlab = 1:n_slabs_to_reconstruct + % if (slab_list{iTomo}(iSlab,1)) + % if (mod(slab_list{iTomo}(iSlab,5),2) == 0) + % % if (slab_list{iTomo}(iSlab,5) < 0) + % % slab_list{iTomo}(iSlab,6) = slab_list{iTomo}(iSlab,6) - 0.5; + % % else + % slab_list{iTomo}(iSlab,6) = slab_list{iTomo}(iSlab,6) - 0.5; + % % end + % end + % end + % slab_list{iTomo}(iSlab,6) = slab_list{iTomo}(iSlab,6) + 1; + % end + % TroubleShoot tSHT = fopen(sprintf('.tblSht_%s_i%d.txt',tiltName,iTomo),'w'); - fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords(iTomo,:)'); - fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', sectionList{iT}'); + fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords{iTomo}.NX, iCoords{iTomo}.NY, iCoords{iTomo}.NZ, iCoords{iTomo}.dX_specimen_to_tomo, iCoords{iTomo}.dY_specimen_to_tomo, iCoords{iTomo}.dZ_specimen_to_tomo); + fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', slab_list{iTomo}'); fclose(tSHT); end % end loop over tomos - + end @@ -1194,36 +909,35 @@ -function [correctedStack] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,pixelSize,maskedStack,... - maxZ,flgDampenAliasedFrequencies,... - preCombDefocus,samplingRate,... - applyExposureFilter,surfaceFit,... - useSurfaceFit,invertDose, ... - phakePhasePlate, ... - filterProjectionsForTomoCPRBackground,... - flgWhitenPS) -% Correct in strips which is more expensive but (hopefully) more accurate. +function [correctedStack] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... + avgZ,TLT,pixel_size_angstroms,maskedStack,... + specimen_NZ_nm,... + preCombDefocus,samplingRate,... + applyExposureFilter,surfaceFit,... + useSurfaceFit, ... + phakePhasePlate, ... + filterProjectionsForTomoCPRBackground,... + flgWhitenPS, ... + flip_defocus_offset, ... + flip_tilt_offset) +% Correct in strips which is more expensive but (hopefully) more accurate. % For sections with too few subTomos to fit, fall back if isa(surfaceFit,'cell') - surfaceFit = surfaceFit{iSection}; + surfaceFit = surfaceFit{iSection}; if ~isa(surfaceFit,'sfit') - fprintf('Warning, surfaceFit is not an sfit object\n'); + % fprintf('Warning, surfaceFit is not an sfit object\n'); useSurfaceFit = false; end -else +else useSurfaceFit = false; end [d1,d2,nPrjs] = size(maskedStack); - - -PIXEL_SIZE = pixelSize*10^-10; % This is just going to be written out to disk so keep in main memory. correctedStack = zeros(d1,d2,nPrjs,'single'); @@ -1232,60 +946,49 @@ % the center of mass of subtomograms in Z to the focal plane, rather than % the center of mass of the tomograms (specimen) -defocusOffset = (((nSections-1)/-2+(iSection-1))*ctf3dDepth); -fprintf('using offset %3.3e for section %d with COM offset %3.3e\n',defocusOffset,iSection,avgZ); -defocusOffset = (defocusOffset - avgZ)*(1-useSurfaceFit); % The average height of the particles is factored into the surface fit - -% The avg Z seems like it should be added? - -if ( flgDampenAliasedFrequencies ) - % Experiment with dampning higher frequencies where aliasing is going to - % result in nonsens. - flgDampenAlias = 1; - fprintf('\n\nExperimental dampening of aliased CTF terms\n'); +if (useSurfaceFit) + defocusOffset = 0; else - flgDampenAlias = 0; + % FIXME: this should probably be ctf3dDepth/2 + % FIXME: this should be renamed as it is an offset in Z (opposite sign to defocus) + defocusOffset = (((n_slabs_to_reconstruct-1)/-2+(iSection-1))*ctf3dDepth); + fprintf('Not using surface fit, so using offset %3.3e nm for section %d with COM offset %3.3e nm with ctf3dDepth %3.3e\n', defocusOffset*10^9, iSection, avgZ*10^9, ctf3dDepth*10^9); + % Assuming the majority of the fit defocus came from the subtomograms, then the estimated defocus value needs to be moved from + % the origin of the specimen to the origin of the subtomograms. + defocusOffset = (defocusOffset + avgZ); % The average height of the particles is factored into the surface fit +end + +if (flip_defocus_offset) + defocusOffset = -defocusOffset; end -apoSize = 6; -fastFTSize = BH_multi_iterator([d1,d2],'fourier2d'); +fastFTSize = BH_multi_iterator([d1,d2],'fourier2d'); % These should be constant for a given tiltseries Cs = TLT(1,17); WAVELENGTH = TLT(1,18); AMPCONT = TLT(1,19); -%if d2 < padTileSize -% padYdim = padTileSize; -%else -% padYdim = d2; -%end -if ( flgDampenAlias ) - % Calculate a centered grid b/c real space convolution [radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... - 'Cylindrical','GPU', ... - {'none'},1,1,0); -else -[radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... - 'Cylindrical','GPU', ... - {'none'},1,0,0); -end -radialGrid = {radialGrid./PIXEL_SIZE,0,phi}; + 'Cylindrical','GPU', ... + {'none'},1,0,0); +radialGrid = {radialGrid./(pixel_size_angstroms*10^-10),0,phi}; phi = []; -fprintf('%f %f\n',filterProjectionsForTomoCPRBackground,pixelSize); if (filterProjectionsForTomoCPRBackground ~= 0) - bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',pixelSize); + bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',pixel_size_angstroms); + % fprintf('Filtering input projections to %f angstroms with %f pixel size\n',bpFilter,pixel_size_angstroms); else - bpFilter = 1; + bpFilter = 1; end -for iPrj = 1:nPrjs - maxEval = cosd(TLT(iPrj,4)).*(d1/2) + maxZ./2*abs(sind(TLT(iPrj,4))); - oX = ceil((d1+1)./2); - oY = ceil((d2+1)./2); +for iPrj = 1:nPrjs + + maxEval = cosd(TLT(iPrj,4)).*(d1/2) + specimen_NZ_nm./2*abs(sind(TLT(iPrj,4))); + oX = emc_get_origin_index(d1); + oY = emc_get_origin_index(d2); iEvalMask = floor(oX-maxEval):ceil(oX+maxEval); if ( applyExposureFilter ) @@ -1296,46 +999,28 @@ iExposureFilter = iExposureFilter .* bpFilter; - - - - STRIPWIDTH = min(floor((0.5*ctf3dDepth/PIXEL_SIZE)/abs(tand(TLT(iPrj,4)))),512); - STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); - % take at least 1200 Ang & include the taper if equal to STRIPWIDTH - tileSize = floor(max(600./pixelSize, STRIPWIDTH + 28)); - tileSize = tileSize + mod(tileSize,2); - %fprintf('stripwidth tilesize %d %d\n',STRIPWIDTH,tileSize); - incLow = ceil(tileSize./2); - incTop = tileSize - incLow; - border = ceil(incLow+apoSize)+1; - ddF = TLT(iPrj,12); dPhi = TLT(iPrj,13); - D0 = TLT(iPrj,15); - %TLT(iPrj,16); - - + D0 = abs(TLT(iPrj,15)); + padVal = BH_multi_padVal([d1,d2],fastFTSize); trimVal = BH_multi_padVal(fastFTSize,[d1,d2]); - iProjection = BH_padZeros3d(maskedStack(:,:,TLT(iPrj,1)),padVal(1,:),padVal(2,:),'GPU','singleTaper'); iProjectionFT = fftn(iProjection).*iExposureFilter; clear iExposureFilter - correctedPrj = zeros([d1,d2],'single','gpuArray'); + correctedPrj = zeros([d1,d2],'single','gpuArray'); % Gridvectors for the specimen plane [rX,rY,~] = BH_multi_gridCoordinates([d1,d2],'Cartesian','GPU',{'none'},0,1,0); % Assuming the plane fit is from the origin as it is. - - - + if (useSurfaceFit) %rZ = (surfaceFit.p00 + surfaceFit.p10.*(rX+oX)) + surfaceFit.p01.*(rY+oY); try rZ = feval(surfaceFit,rX,rY); catch d1 - d2 + d2 rX rY surfaceFit @@ -1344,142 +1029,117 @@ else rZ = zeros([d1,d2],'single','gpuArray'); end + + if (flip_tilt_offset) + defocus_adj = D0 + (defocusOffset.*cosd(TLT(iPrj,4))); + else + defocus_adj = D0 - (defocusOffset.*cosd(TLT(iPrj,4))); + end + % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) + rA = BH_defineMatrix(TLT(iPrj,4),'TILT','fwdVector') ; - full_defocusOffset = ((defocusOffset.*cosd(TLT(iPrj,4))) + D0); - - rA = BH_defineMatrix([0,TLT(iPrj,4),0],'SPIDER','inv'); % Transform the specimen plane - tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ +oX); - tY = round(rA(2).*rX + rA(5).*rY + rA(8).*rZ +oY); - tZ = PIXEL_SIZE.*(rA(3).*rX + rA(6).*rY + rA(9).*rZ) + full_defocusOffset; - + tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ + oX); + tY = round(rA(2).*rX + rA(5).*rY + rA(8).*rZ + oY); + % undefocus is positive, so we subtract the offset in Z + tZ = defocus_adj - (pixel_size_angstroms*10^-10).*(rA(3).*rX + rA(6).*rY + rA(9).*rZ); + % Some edge pixels can be out of bounds depending on the orientation of - % the plan fit. Setting to zero will will ignore them (assuming defocus - % is always < 0) + % the plan fit. Setting to zero will will ignore them tZ( tX < 1 | tY < 1 | tX > d1 | tY > d2) = 1; - minDefocus = min(tZ(:)); - maxDefocus = max(tZ(tZ<1)); - % Spit out some info -% fprintf('Found a min/max defocus of %3.3e/ %3.3e for tilt %d (%3.3f deg)\n',minDefocus,maxDefocus,iPrj,TLT(iPrj,4)); - + maxDefocus = max(tZ(tZ < 1)); %tZ is in angstrom so always << 1 % To track sampling in case I put in overlap samplingMask = zeros([d1,d2],'single','gpuArray'); - + % TODO: confirm the astigmatism is correct, check - and +/- 90 for iDefocus = minDefocus-ctf3dDepth/1:ctf3dDepth/1:maxDefocus+ctf3dDepth/1 -% fprintf('correcting for iDefocus %3.3e\n',iDefocus); - %search tz take those xy and add to the prj and mask - - defVect = [iDefocus - ddF, iDefocus + ddF, dPhi]; - - if (phakePhasePlate(1) > 0) - if numel(phakePhasePlate) == 2 - modPower = floor(phakePhasePlate(2)); - SNR = rem(phakePhasePlate(2),1); - else - modPower = 1; - SNR = 1; - end - - - [Hqz, ~] = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,1,SNR); - - Hqz = (-1).^modPower.*(phakePhasePlate(1).*Hqz).^1; - - - modHqz = []; + defVect = [iDefocus + ddF, iDefocus - ddF, dPhi]; + + if (phakePhasePlate(1) > 0) + if numel(phakePhasePlate) == 2 + modPower = floor(phakePhasePlate(2)); + SNR = rem(phakePhasePlate(2),1); else - if PIXEL_SIZE < 2.0e-10 - % use double precision - this is not enabled, but needs to be - - % requires changes to radial grid as well. - Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,-1); - else - Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1); - end + modPower = 1; + SNR = 1; end + [Hqz, ~] = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,1,SNR); - - if (flgWhitenPS(3)) - tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz./(abs(Hqz).^2+flgWhitenPS(3)))),trimVal(1,:),trimVal(2,:),'GPU','single'); - else - tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz)),trimVal(1,:),trimVal(2,:),'GPU','single'); - end - - tmpMask = (tZ > iDefocus - ctf3dDepth/2 & tZ <= iDefocus + ctf3dDepth/2); - -% try - - linearIDX = unique(sub2ind([d1,d2],tX(tmpMask),tY(tmpMask))); -% catch -% -% -% ferr=fopen('err.txt','w'); -% fprintf(ferr,'%f %f\n',[tX(tmpMask),tY(tmpMask)]); -% fclose(ferr); -% error('sdf') -% end - - correctedPrj(linearIDX) = correctedPrj(linearIDX) + tmpCorrection(linearIDX); - samplingMask(linearIDX) = samplingMask(linearIDX) + 1; - + Hqz = (-1).^modPower.*(phakePhasePlate(1).*Hqz).^1; + modHqz = []; + else + if (pixel_size_angstroms < 2.0) + % use double precision - this is not enabled, but needs to be - + % requires changes to radial grid as well. + Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,-1); + else + Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1); + end + end + + + if (flgWhitenPS(3)) + tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz./(abs(Hqz).^2+flgWhitenPS(3)))),trimVal(1,:),trimVal(2,:),'GPU','single'); + else + tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz)),trimVal(1,:),trimVal(2,:),'GPU','single'); + end + + % Each loop we increment by ctf3dDepth + tmpMask = (tZ > iDefocus - ctf3dDepth/2 & tZ <= iDefocus + ctf3dDepth/2); + + linearIDX = unique(sub2ind([d1,d2],tX(tmpMask),tY(tmpMask))); + + correctedPrj(linearIDX) = correctedPrj(linearIDX) + tmpCorrection(linearIDX); + samplingMask(linearIDX) = samplingMask(linearIDX) + 1; + end % end loop over defocus values - - samplingMask(samplingMask == 0) = 1; + + if (flgWhitenPS(1)) + correctedStack(:,:,TLT(iPrj,1)) =gather(BH_whitenNoiseSpectrum(correctedPrj./samplingMask,'',pixel_size_angstroms,1)); + else + + correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj./samplingMask); + end - if (flgWhitenPS(1)) - correctedStack(:,:,TLT(iPrj,1)) =gather(BH_whitenNoiseSpectrum(correctedPrj./samplingMask,'',pixelSize,1)); - else - - correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj./samplingMask); - end - clear correctedPrj samplingMask tmpMask tmpCorrection - - clear iProjection iProjectionFT + clear correctedPrj samplingMask tmpMask tmpCorrection + + clear iProjection iProjectionFT end % end loop over projections clear tile Hqz end -function [avgZ, maxZ, tomoNumber,surfaceFit] = calcAvgZ(masterTM,iCoords, ... - tiltName,tomoList,... - nTomos, pixelSize,... - samplingRate,cycleNumber,... - sectionList,calcMaxZ) +function [avgZ, specimen_NZ_nm, surfaceFit] = calcAvgZ(subTomoMeta, ... + iCoords, ... + tiltName, ... + tomoList,... + nTomos, ... + pixel_size_angstroms,... + samplingRate, ... + cycleNumber,... + slab_list, ... + calcMaxZ) % Calculate the maximum extensions in Z and then how many separate sections % need to be corrected. surfaceFit = ''; avgZ = 0; -maxZ = 0; -tomoNumber = zeros(nTomos,1); -for iTomo = 1:nTomos - % The tomograms may not be listed monotonically so explicitly get their - % id number - if isa(masterTM,'struct') - tomoNumber(iTomo) = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - nZdZ = iCoords(tomoNumber(iTomo),[4,6]); - else - tomoNumber(iTomo) = iTomo; - nZdZ = iCoords(iTomo,[4,6]); - end - % half the size in z plus the shift back to the microscope coords. - sZneeded = 2.*ceil(nZdZ(1)/2+abs(nZdZ(2))+1); - if sZneeded > maxZ - maxZ = sZneeded; - end -end -maxZ = maxZ + (samplingRate*2); +[ specimen_NZ_pixels ] = emc_get_max_specimen_NZ( ... + iCoords, ... + tomoList, ... + nTomos, ... + samplingRate); -maxZ = maxZ.*pixelSize./10; -fprintf('combining thickness and shift on tilt %s, found a maxZ %3.3f nm\n',tiltName,maxZ); +specimen_NZ_nm = specimen_NZ_pixels .* pixel_size_angstroms ./ 10; +fprintf('combining the thickness and shift on tilt %s, found a specimen_NZ_nm %3.3f nm\n', tiltName, specimen_NZ_nm); if (calcMaxZ) return; @@ -1487,17 +1147,16 @@ % For now use cycle000, if adding a refinment focused on a specific set of % particles, then consider that later. - try - initGeom = masterTM.(cycleNumber).RawAlign; + initGeom = subTomoMeta.(cycleNumber).RawAlign; fprintf('Loaded the geometry for RawAlign %s\n',cycleNumber); catch - fprintf('Failed to load the geometry for RawAlign %s\nTrying cycle000\n',cycleNumber); + fprintf('Failed to load the geometry for RawAlign %s\nTrying cycle000\n',cycleNumber); try - initGeom = masterTM.cycle000.geometry; + initGeom = subTomoMeta.cycle000.geometry; catch error(['Could not load the initial geometry subTomoMeta.%s.geometry\n or--',... - 'subTomoMeta.cycle000.geometry\n'],cycleNumber); + 'subTomoMeta.cycle000.geometry\n'],cycleNumber); end end @@ -1506,107 +1165,97 @@ % for each tomogram get the size and origin in Z then find mean subTomo % position. -nSections = size(sectionList{1},1); -xFull = cell(nSections,1); -yFull = cell(nSections,1); -zFull = cell(nSections,1); -surfaceFit = cell(nSections,1); +n_slabs_to_reconstruct = size(slab_list{1},1); +xFull = cell(n_slabs_to_reconstruct,1); +yFull = cell(n_slabs_to_reconstruct,1); +zFull = cell(n_slabs_to_reconstruct,1); +surfaceFit = cell(n_slabs_to_reconstruct,1); % Initialize with empty arrays -for iSection = 1:nSections +for iSection = 1:n_slabs_to_reconstruct xFull{iSection} = []; yFull{iSection} = []; zFull{iSection} = []; -end +end -for iT = 1:nTomos - iTomo = tomoNumber(iT); - micDimension = floor(masterTM.tiltGeometry.(tomoList{iT})(1,20:22) ./ samplingRate); +use_subtomo_z_positions = true; - % Already scaled to sampled pixels - tomoOrigin =[ ceil((iCoords(iTomo,1)+1)./2),... - ceil((iCoords(iTomo,3)-iCoords(iTomo,2))./2),... - ceil((iCoords(iTomo,4)+1)/2)]; - micOrigin = [-1*iCoords(iTomo,5), ... - (iCoords(iTomo,2) + tomoOrigin(2)) - ceil((micDimension(2)+1)/2),... - iCoords(iTomo,6)]; - - iTomoName = sprintf('%s_%d',tiltName,iTomo); +for iTomo = 1:nTomos - % shouldn't be any removed particles at this stage but later there would be. - zList = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,13)./samplingRate; + if ~(subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}).is_active) + continue; + end + % X in the Y frame means a vector from the Y lower left to the X origin + % X origin wrt Y origin is a vector from the origin of Y to the X origin + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... + reconGeometry.dY_specimen_to_tomo, ... + reconGeometry.dZ_specimen_to_tomo]; + tomo_origin_in_tomo_frame = emc_get_origin_index([reconGeometry.NX, ... + reconGeometry.NY, ... + reconGeometry.NZ]); + + + % Get the z-coordinates of the origin for all included subtomograms relative to the lower left of the tomogram + % shouldn't be any removed particles at this stage but later there would be. + subtomo_origin_z_in_tomo_frame = initGeom.(tomoList{iTomo})(initGeom.(tomoList{iTomo})(:,26)~=-9999,13); + + % We may get here if we have split a data set into several small classes so skip the centering on average if needed + if isempty(subtomo_origin_z_in_tomo_frame) + use_subtomo_z_positions = false; + fprintf('No subtomograms found for %s', tomoList{iTomo}); + continue; + end + % shift from lower left to centered and include the tomos offset from the - % microscope frame - zList = zList - tomoOrigin(3) + micOrigin(3); - totalZ = totalZ + sum(zList); + subtomo_origin_wrt_specimen_origin = subtomo_origin_z_in_tomo_frame - tomo_origin_in_tomo_frame(3) + tomo_origin_wrt_tilt_origin(3); + subtomo_origin_wrt_specimen_origin = subtomo_origin_wrt_specimen_origin ./ samplingRate; + totalZ = totalZ + sum(subtomo_origin_wrt_specimen_origin); fprintf('%s tomo has %d subTomos with mean Z %3.3f nm\n', ... - iTomoName, length(zList), mean(zList)*pixelSize./10); - nSubTomos = nSubTomos + length(zList); - - for iSection = 1:nSections - + tomoList{iTomo}, length(subtomo_origin_wrt_specimen_origin), ... + mean(subtomo_origin_wrt_specimen_origin) * pixel_size_angstroms ./ 10); - iSecOrigin = sectionList{iT}(iSection,6); - iSecRadius = sectionList{iT}(iSection,5)/2; - inSectionIDX = zList > iSecOrigin - iSecRadius & zList <= iSecOrigin + iSecRadius; + nSubTomos = nSubTomos + length(subtomo_origin_wrt_specimen_origin); + for iSection = 1:n_slabs_to_reconstruct + + iSecOrigin = slab_list{iTomo}(iSection,6); + iSecRadius = slab_list{iTomo}(iSection,5)/2; + inSectionIDX = subtomo_origin_wrt_specimen_origin > iSecOrigin - iSecRadius & subtomo_origin_wrt_specimen_origin <= iSecOrigin + iSecRadius; + + + x = initGeom.(tomoList{iTomo})(initGeom.(tomoList{iTomo})(:,26)~=-9999,11); + x = (x - tomo_origin_in_tomo_frame(1) + tomo_origin_wrt_tilt_origin(1))./samplingRate; + y = initGeom.(tomoList{iTomo})(initGeom.(tomoList{iTomo})(:,26)~=-9999,12); + y = (y - tomo_origin_in_tomo_frame(2) + tomo_origin_wrt_tilt_origin(2))./samplingRate; + - - x = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,11)./samplingRate; - x = x - tomoOrigin(1) + micOrigin(1); - y = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,12)./samplingRate; - y = y - tomoOrigin(2) + micOrigin(2); - - xFull{iSection} = [xFull{iSection} ; x(inSectionIDX)]; yFull{iSection} = [yFull{iSection} ; y(inSectionIDX)]; - zFull{iSection} = [zFull{iSection} ; zList(inSectionIDX)]; - - - - + zFull{iSection} = [zFull{iSection} ; subtomo_origin_wrt_specimen_origin(inSectionIDX)]; + end % loop over sections - clear zList + clear subtomo_origin_wrt_specimen_origin end % loop over tomos -avgZ = totalZ/nSubTomos*pixelSize/10*10^-9; +if (nSubTomos == 0) + avgZ = 0; +else + avgZ = totalZ / nSubTomos*pixel_size_angstroms / 10*10^-9; +end -% sf(x,y) = p00 + p10*x + p01*y; -% surfaceFit = fit([xFull, yFull],zFull,'poly11'); - %surfaceFit = fit([xFull, yFull],zFull,'lowess','Span',0.1); -for iSection = 1:nSections - if length(xFull{iSection}) >= 6 -% try -% try -% exclude = abs(mean(zFull{iSection})-zFull{iSection})>1.5.*std(zFull{iSection}); -% surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'lowess', 'Span', 0.05,'Normalize','on','Exclude',exclude); -% catch -% -% surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'lowess','Robust','on'); -% end - - surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'poly22','Robust','on'); - % end -% -% figure('visible','off'), plot(surfaceFit{iSection},[xFull{iSection},yFull{iSection}],zFull{iSection}); -% saveas(gcf,sprintf('fitThis_%s_%d.pdf',tiltName,iSection)); -% close(gcf); +for iSection = 1:n_slabs_to_reconstruct + + if (use_subtomo_z_positions && length(xFull{iSection}) >= 6) + surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'poly22','Robust','on'); else surfaceFit{iSection} = 0; end end -% save(sprintf('fitThis_%s_%d.mat',tiltName,iSection),'xFull','yFull','zFull','surfaceFit'); - - -fprintf('%s tilt-series has %d subTomos with mean Z %3.3f nm\n', ... - tiltName, nSubTomos,avgZ*10^9); - - - - +fprintf('%s tilt-series has %d subTomos with mean Z %3.3f nm\n', tiltName, nSubTomos,avgZ*10^9); end diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 3a392414..2c74de4f 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -3,7 +3,7 @@ global bh_global_do_2d_fourier_interp; !mkdir -p aliStacks modLocal = false; -if length(varargin) > 3 +if length(varargin) > 3 error('Too many input arguments'); else % PARAMETER_FILE, STACK_BASENAME, gpuIDX @@ -25,13 +25,15 @@ collectionORDER = sprintf('fixedStacks/%s.order',STACK_BASENAME); end -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); gpuIDX = BH_multi_checkGPU(-1); gDev = gpuDevice(gpuIDX); -flgResume = 0; -flgSkip = 0; +% for trouble shooting downstream, default falsw +flgSkip = false; +% For trouble shooting downstream, default true +resample_stack = true; % slightly dampen lower resolution information that may overwhelm the CCC calc, % but don't risk too much noise amplification. 1 = fit just the amplitude (not % PS) 0.5 = take sqrt prior to normalizing. @@ -56,30 +58,25 @@ end end skipFitting = 0; -try - PHASE_PLATE_SHIFT = pBH.('PHASE_PLATE_SHIFT').*pi -catch - PHASE_PLATE_SHIFT = [0,0] -end +% PHASE_PLATE_SHIFT is now handled in BH_parseParameterFile (already multiplied by pi) +PHASE_PLATE_SHIFT = emc.PHASE_PLATE_SHIFT; if sum(PHASE_PLATE_SHIFT) skipFitting = 1; end flgStandardOrdeDoCalc = 1; -try - flgCosineDose = pBH.('oneOverCosineDose'); - startingAngle = pBH.('startingAngle'); - startingDirection = pBH.('startingDirection'); - doseSymmetricIncrement = pBH.('doseSymmetricIncrement'); - doseAtMinTilt = pBH.('doseAtMinTilt'); - - flgOldDose = 0; - tltOrder = calc_dose_scheme(pBH,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT); - - -catch - fprintf('\nFalling back on old dose specification through a *.order file\n\n'); - fprintf('Parameters flgCosineDose=(0/1 bool), \nstartingAngle=, \nstartingDirection=[pos/neg],\ndoseSymmetricIncrement=[0, or # tilts per sweep],\n doseAtMinTilt are needed for the new method.\n'); +% Dose parameters are now handled in BH_parseParameterFile +flgCosineDose = emc.oneOverCosineDose; +startingAngle = emc.startingAngle; +startingDirection = emc.startingDirection; +doseSymmetricIncrement = emc.doseSymmetricIncrement; +doseAtMinTilt = emc.doseAtMinTilt; + +flgOldDose = 0; +tltOrder = calc_dose_scheme(emc,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT); + +% Check if we need to fall back to old dose method +if ~flgCosineDose && startingAngle == 0 && doseAtMinTilt == 0 pause(2); tltOrder = load(collectionORDER); flgOldDose = 1; @@ -90,64 +87,16 @@ end end -% If true, then parameters will be adjusted to make this initial estimate -% faster, since it is less critical to be exact. -try - do_ctf_refine = pBH.('skip_ctf_refine'); -catch - do_ctf_refine = true; -end -PIXEL_SIZE = pBH.('PIXEL_SIZE'); -Cs = pBH.('Cs'); -VOLTAGE = pBH.('VOLTAGE'); -AMPCONT = pBH.('AMPCONT'); -SuperResolution = pBH.('SuperResolution'); - -if (SuperResolution) - if SuperResolution == 1 - % Standard scenario crop to physical nyquist - scalePixelsBy = 2; - elseif SuperResolution > 10^10*PIXEL_SIZE - % Crop to the given pixels size - error('Scaling to arbitrary pixel size is not working\n'); - % Need to factor in the trunctation to integer pixel size. -% scalePixelsBy = SuperResolution/(10^10*PIXEL_SIZE); - else - error('SuperResolution must be 0 (off) 1 (crop to physical Nyquist) or a pixel Size larger than current\n'); - end - PIXEL_SIZE = scalePixelsBy.* PIXEL_SIZE; -else - scalePixelsBy = 1; -end +scalePixelsBy = 1; -% if 10^10*PIXEL_SIZE < 1.2 -% fprintf('PixelSize is less than 1.2 Ang so we have to use the cpu\n'); -% useGPU = 0; -% METHOD = 'cpu'; -% else - useGPU = 1; - METHOD = 'GPU'; -% end - -% Sanity check -if (PIXEL_SIZE > 20e-10 || PIXEL_SIZE < 0) - error('pixel size should be [0,20e-10]'); -elseif (Cs > 10e-3 || Cs < 0) - fprintf('\nWARNING Cs should be[10e-3,0]\n'); -elseif(VOLTAGE > 1000e3 || VOLTAGE < 20e3) - error ('VOLTAGE should be [20e3,1000e3]'); -elseif (AMPCONT < 0.025 || AMPCONT > 0.25) - fprintf('\nWARNING: AMPCONT probably should be [0.025,0.25]\n'); -end - WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; +useGPU = 1; +METHOD = 'GPU'; -if Cs == 0 - Cs = 1e-6; -end - -CUM_e_DOSE = pBH.('CUM_e_DOSE'); +WAVELENGTH = 10^-12*1226.39/sqrt(emc.VOLTAGE + 0.97845*10^-6*emc.VOLTAGE^2) ; + +CUM_e_DOSE = 0; % test astigmatism vals flgAstigmatism = 1; if (flgAstigmatism ~=1 && flgAstigmatism ~= 0) @@ -163,84 +112,51 @@ eraseSigma = 3; -eraseRadius = ceil(1.2.*(pBH.('beadDiameter')./PIXEL_SIZE.*0.5)); +eraseRadius = ceil(1.2.*(emc.('beadDiameter')./emc.pixel_size_si.*0.5)); flgImodErase = 0 - - -% Assuming that the first CTF zero is always less than this value -FIXED_FIRSTZERO = PIXEL_SIZE / (70*10^-10) ; -highCutoff = PIXEL_SIZE/pBH.('defCutOff'); -% I still use the def for underfocus < 0 as this places the origin at the -% focal plan in the microscope rather than on the specimen. Which makes -% more sense to me. -defEST = -1.*pBH.('defEstimate').*10^6 -defWIN = pBH.('defWindow').*10^6 + + +% Assuming that the first CTF zero is always less than this value +FIXED_FIRSTZERO = emc.pixel_size_si / (70*10^-10) ; +highCutoff = emc.pixel_size_si/emc.('defCutOff'); + +defEST = emc.('defEstimate').*10^6 +defWIN = emc.('defWindow').*10^6 tiltRange = [-1]; backGroundBuffer = 0.9985; -try - deltaZTolerance = pBH.('deltaZTolerance'); -catch - if (do_ctf_refine) - deltaZTolerance = 50e-9; - else - deltaZTolerance = 100e-9; - end -end -try - zShift = abs(pBH.('zShift')); -catch - zShift = 150e-9; -end -if abs(zShift) > 100e-7 - error('make sure your zShift values are of reasonable amounts (50-200nm)'); -end -try - maxNumberOfTiles = pBH.('ctfMaxNumberOfTiles'); -catch - if (do_ctf_refine) - maxNumberOfTiles = 4000; - else - maxNumberOfTiles = 10000; - end -end % Starting at +/- 100nm -deltaZTolerance = deltaZTolerance / PIXEL_SIZE; +emc.deltaZTolerance = emc.deltaZTolerance / emc.pixel_size_si; % Use to check for proper gradient. -zShift = zShift / PIXEL_SIZE; +emc.zShift = emc.zShift / emc.pixel_size_si; % Tile size & overlap -try - tileSize = pBH.('ctfTileSize'); -catch - tileSize = floor(680e-10 / PIXEL_SIZE); -end -tileOverlap = 2; -tileSize = tileSize + mod(tileSize,2); -% tileSize = max(tileSize, 384); -fprintf('Using a tile size of %d\n',tileSize); +tileOverlap = emc.('ctf_tile_overlap'); -overlap = floor(tileSize ./ tileOverlap); +% emc.ctf_tile_size = max(emc.ctf_tile_size, 384); +if (emc.ctf_tile_size > 512) + tileOverlap = tileOverlap * 2; +end +fprintf('Using a tile size of %d\n',emc.ctf_tile_size); + +overlap = floor(emc.ctf_tile_size ./ tileOverlap); % Size to padTile to should be even, large, and preferably a power of 2 -try - paddedSize = pBH.('paddedSize'); -catch - paddedSize = 768; -end +% paddedSize is now handled in BH_parseParameterFile +paddedSize = emc.paddedSize; -padVAL = BH_multi_padVal([tileSize,tileSize], [paddedSize,paddedSize]); +padVAL = BH_multi_padVal([emc.ctf_tile_size,emc.ctf_tile_size], [paddedSize,paddedSize]); -if exist(stackNameIN, 'file') +if exist(stackNameIN, 'file') if ( flgOldDose ) TLT = zeros(length(rawTLT),23); @@ -253,7 +169,7 @@ TLT(:,1) = tltOrder(:,1); TLT(:,12) = (tltOrder(:,4)-tltOrder(:,5))./2 .* 10^-10; TLT(:,13) = tltOrder(:,6) .* (pi / 180); - TLT(:,15) = -1.*(tltOrder(:,4)+tltOrder(:,5))./2 .* 10^-10; + TLT(:,15) = 1.*(tltOrder(:,4)+tltOrder(:,5))./2 .* 10^-10; TLT(:,14) = tltOrder(:,3); sorted_dose = sortrows(tltOrder,2); cummul_dose = cumsum(sorted_dose(:,3)); @@ -270,47 +186,44 @@ TLT(:,19) = tltOrder(:,4); end - - + + [pathName,fileName,extension] = fileparts(stackNameIN); if isempty(pathName) pathName = '.'; end else fprintf('ignoring %s, because the file is not found.\n', stackNameIN) - + end - + % Make ctf directory to store diagnostic images -system(sprintf('mkdir -p %s/ctf', pathName)); +system(sprintf('mkdir -p %s/ctf', pathName)); -% if ~(flgResume) +iMrcObj = MRCImage(stackNameIN,0); +% The pixel size should be previously set correctly, but if it is not, then we +% must maintain whatever is there in case beads are to be erased. The model +% used for this process depends on the pixel size in the header when it was +% created in IMod alignment. - iMrcObj = MRCImage(stackNameIN,0); +iHeader = getHeader(iMrcObj); - % The pixel size should be previously set correctly, but if it is not, then we - % must maintain whatever is there in case beads are to be erased. The model - % used for this process depends on the pixel size in the header when it was - % created in IMod alignment. +iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* scalePixelsBy, ... + iHeader.cellDimensionY/iHeader.nY .* scalePixelsBy, ... + iHeader.cellDimensionZ/iHeader.nZ]; +% Reduce the Z dimension after pixel size is calculated +iHeader.nZ = iHeader.nZ - nSkipped; - iHeader = getHeader(iMrcObj); - - iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* scalePixelsBy, ... - iHeader.cellDimensionY/iHeader.nY .* scalePixelsBy, ... - iHeader.cellDimensionZ/iHeader.nZ]; - % Reduce the Z dimension after pixel size is calculated - iHeader.nZ = iHeader.nZ - nSkipped; - - iOriginHeader= [iHeader.xOrigin , ... - iHeader.yOrigin , ... - iHeader.zOrigin ] ./ scalePixelsBy; +iOriginHeader= [iHeader.xOrigin , ... + iHeader.yOrigin , ... + iHeader.zOrigin ] ./ scalePixelsBy; + +d1 = iHeader.nX ; d2 = iHeader.nY ; d3 = iHeader.nZ; - d1 = iHeader.nX ; d2 = iHeader.nY ; d3 = iHeader.nZ; - @@ -323,8 +236,8 @@ TLT(:,2:3) = repmat([0.00,0.00],size(TLT,1),1); TLT(:,5:10) = repmat([0,90.0,1.0,0.0,0.0,1.0],size(TLT,1),1); % Defocus will go at 15 - 12 and 13 currently unused. -TLT(:,16:18) = repmat([PIXEL_SIZE,Cs,WAVELENGTH],size(TLT,1),1); -TLT(:,19) = TLT(:,19) + AMPCONT; +TLT(:,16:18) = repmat([emc.pixel_size_si,emc.Cs,WAVELENGTH],size(TLT,1),1); +TLT(:,19) = TLT(:,19) + emc.AMPCONT; oddSize = [d1,d2,d3] - (1-mod([d1,d2,d3],2)); TLT(:,20:22) = repmat(oddSize,size(TLT,1),1); @@ -332,8 +245,8 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 % Part of the switch to listing dose not bfactor to use the optimal exposure -% filter by Grant/Grigorieff - - nPrjs = size(TLT,1); +% filter by Grant/Grigorieff - +nPrjs = size(TLT,1); if ( flgOldDose && flgStandardOrdeDoCalc ) if CUM_e_DOSE < 0 @@ -345,14 +258,14 @@ flgCosineDose = 0; exposure = CUM_e_DOSE./nPrjs; end - + totalExposure = 0; % If the fit tilt angles have moved alot, you may end up with duplicates, alreadyPicked = zeros(nPrjs,1,'single','gpuArray'); largeVect = alreadyPicked + 1000; for iExposure = 1:nPrjs % find the projection angle most closley matching ( - + [~,iTilt] = min((alreadyPicked.*largeVect)+(abs(TLT(:,4)-tltOrder(iExposure)))); alreadyPicked(iTilt) = 1; if flgCosineDose == 0 @@ -362,664 +275,491 @@ end TLT(iTilt,11) = totalExposure; TLT(iTilt,14) = exposure; - + end - -end +end -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -fprintf('Combining tranformations\n\n'); -% Load in the mapBack alignment -mbEST = load(sprintf('%s.xf',mapBackPrfx)); -mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); +if ( resample_stack) + fprintf('Combining tranformations\n\n'); + % Load in the mapBack alignment + mbEST = load(sprintf('%s.xf',mapBackPrfx)); + mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); -outputStackName = sprintf('aliStacks/%s%s',stackNameOUT,extension) + outputStackName = sprintf('aliStacks/%s%s',stackNameOUT,extension); -try - erase_beads_after_ctf = pBH.('erase_beads_after_ctf'); -catch - erase_beads_after_ctf = false; -end + % erase_beads_after_ctf is now handled in BH_parseParameterFile + erase_beads_after_ctf = emc.erase_beads_after_ctf; -if (erase_beads_after_ctf) - flgEraseBeads = 0; -else - if exist(sprintf('%s.erase',mapBackPrfx),'file') - flgEraseBeads = 1; - else + if (erase_beads_after_ctf) flgEraseBeads = 0; - fprintf('\nDid not find the gold bead file (%s) for erasing, will skip\n\n',sprintf('%s.erase',mapBackPrfx)); + else + if exist(sprintf('%s.erase',mapBackPrfx),'file') + flgEraseBeads = 1; + else + flgEraseBeads = 0; + fprintf('\nDid not find the gold bead file (%s) for erasing, will skip\n\n',sprintf('%s.erase',mapBackPrfx)); + end end -end -if (SuperResolution) - % Forcing output to odd size. - sizeCropped = floor([d1,d2,d3]./2)-(1-mod(floor([d1,d2,d3]./2),2)); -else sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); -end -sizeCropped(3) = d3; - -STACK = zeros(sizeCropped,'single'); - samplingMaskStack = zeros(sizeCropped,'single'); - + sizeCropped(3) = d3; + if (flgReOrderMapBack) + TLT = sortrows(TLT,1); + end -if (flgReOrderMapBack) - TLT = sortrows(TLT,1); -end -% if any([d1,d2] > 4096) -% shiftMETHOD = 'cpu'; -% fprintf('transforming on cpu b/c > 4096\n') -% else shiftMETHOD = 'GPU'; -% end - -% Redefine d3 incase views are ignored -d3 = size(TLT,1); - -osX = 1-mod(d1,2); osY = 1-mod(d2,2); - - + % Redefine d3 incase views are ignored + d3 = size(TLT,1); -for i = 1:d3 -% fprintf('Transforming prj %d in fourier space oversampled by 2x physical Nyquist\n',i); + osX = 1-mod(d1,2); osY = 1-mod(d2,2); - - % Stored in row order as output by imod, st transpose is needed. Inversion - % of the xform is handled in resample2d. - origXF = [1,0;0,1]; + for i = 1:d3 + % fprintf('Transforming prj %d in fourier space oversampled by 2x physical Nyquist\n',i); + origXF = [1,0;0,1]; + + newXF = reshape(mbEST(TLT(i,23),1:4),2,2)'; + + + dXYZ = [(newXF*TLT(i,2:3)')' + mbEST(TLT(i,23),5:6) , 0]; + TLT(i,2:3) = dXYZ(1:2); + dXYZ = dXYZ ./ scalePixelsBy; + + combinedXF = reshape((newXF*origXF)',1,4); + TLT(i,7:10) = combinedXF; - newXF = reshape(mbEST(TLT(i,23),1:4),2,2)'; - - - dXYZ = [(newXF*TLT(i,2:3)')' + mbEST(TLT(i,23),5:6) , 0]; - TLT(i,2:3) = dXYZ(1:2); - dXYZ = dXYZ ./ scalePixelsBy; - - combinedXF = reshape((newXF*origXF)',1,4); - TLT(i,7:10) = combinedXF; - - + sizeODD = [d1,d2]-[osX,osY]; - - - sizeODD = [d1,d2]-[osX,osY]; - % If it is even sized, shift up one pixel so that the origin is in the middle - % of the odd output here we can just read it in this way, unlike super res. - - iProjection = ... - single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],TLT(i,23),'keep')); - - iProjection = real(ifftn(fftn(iProjection).* BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'))); - - largeOutliersMean= mean(iProjection(:)); - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); + % of the odd output here we can just read it in this way, unlike super res. - largeOutliersIDX = []; - - - % Padding to avoid interpolation artifacts. For K3 images this can push - % a 2080 close to or over the limit, so it is been reduced to 1/4 (from - % 1) i.e. the image is paded to 1.25 x unless useFourierInterp is set > - % 1; - sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp*0.25).*max(sizeODD)); -% sizeSQ = floor(([1,1]).*max(sizeODD)); - - padVal = BH_multi_padVal(sizeODD,sizeSQ); - trimVal = BH_multi_padVal(sizeSQ,sizeCropped(1:2)); - - - iProjection = iProjection - mean(iProjection(:)); - - - if ( SuperResolution ) - iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... - padVal(1,:),padVal(2,:),shiftMETHOD,'singleTaper'); - else - iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - shiftMETHOD,'singleTaper'); end - - if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); + base_cmd = sprintf('newstack -mode 12 -meansd 0,1 -xf %s.xf %s %s', mapBackPrfx, stackNameIN, outputStackName); + fprintf('Base command %s\n', base_cmd); + [ newstack_err ] = system(sprintf('%s > /dev/null',base_cmd)); + if (newstack_err) + system(base_cmd); + error('newstack failed'); end - - - % Do the phase shift after rotating - need to invert the scaling since - % we are in reciprocal space - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(combinedXF); - - - - if (bh_global_do_2d_fourier_interp) -% combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - combinedInverted = combinedInverted([1,2,4,5]); - - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); - else - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); - end - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - - iSamplingMask(isnan(iSamplingMask(:))) = 0; - samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); - iSamplingMask = []; - -% % % % % iProjection = real(fftshift(ifftn(ifftshift(iProjection)))); - STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... - trimVal(1,:),trimVal(2,:),... - shiftMETHOD,'single'))); + samplingMaskStack = ones(sizeCropped,'single'); + SAVE_IMG(samplingMaskStack,{sprintf('%s.samplingMask_pre',outputStackName), 'half'}, iPixelHeader,iOriginHeader); + base_cmd = sprintf('newstack -mode 12 -fill 0 -xf %s.xf %s.samplingMask_pre %s.samplingMask',mapBackPrfx,outputStackName,outputStackName); + [ newstack_err ] = system(sprintf('%s > /dev/null',base_cmd)); + if (newstack_err) + system(base_cmd); + error('newstack failed'); + end -end + system(sprintf('rm %s.samplingMask_pre',outputStackName)); +end +STACK = gpuArray(OPEN_IMG('single',outputStackName)); if ( flgEraseBeads ) - STACK = BH_eraseBeads(STACK,eraseRadius, fileName, scalePixelsBy,0,sortrows(TLT,1)); -end - -[ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',100,PIXEL_SIZE*10^10,samplingMaskStack); - + STACK = BH_eraseBeads(STACK,eraseRadius, fileName, scalePixelsBy,0,sortrows(TLT,1)); +end +samplingMaskStack = gpuArray(OPEN_IMG('single',sprintf('%s.samplingMask',outputStackName))); +[ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',100,emc.pixel_size_si*10^10,samplingMaskStack); -SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); -SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName),iPixelHeader,iOriginHeader); +STACK = gather(STACK); +samplingMaskStack = gather(samplingMaskStack); +SAVE_IMG(STACK,{outputStackName, 'half'},iPixelHeader,iOriginHeader); +SAVE_IMG(samplingMaskStack,{sprintf('%s.samplingMask',outputStackName),'half'},iPixelHeader,iOriginHeader); if ~(flgSkip) -gpuDevice(gpuIDX) -[d1,d2,d3] = size(STACK); -if (PIXEL_SIZE*10^10 < 0) + gpuDevice(gpuIDX) + [d1,d2,d3] = size(STACK); + if (emc.pixel_size_si*10^10 < 0) + flgCrop = 1; + [croppedIMG,pixelOUT] = cropIMG(STACK(:,:,1),emc.pixel_size_si*10^10); + [d1C,d2C] = size(croppedIMG); clear croppedIMG + tltForExp = TLT; + tltForExp(:,16) = pixelOUT*10^-10; + pixelOUT + % Redefining things down hear is a stupid thing to do. Fix this if you + % keep the optino for cropping. + FIXED_FIRSTZERO = pixelOUT / 70 ; + highCutoff = (pixelOUT*10^-10)/emc.('defCutOff'); + else + flgCrop = 0; + pixelOUT = emc.pixel_size_si*10^10; + d1C = d1; + d2C = d2; + tltForExp = TLT; + end - flgCrop = 1; - [croppedIMG,pixelOUT] = cropIMG(STACK(:,:,1),PIXEL_SIZE*10^10); - [d1C,d2C] = size(croppedIMG); clear croppedIMG - tltForExp = TLT; - tltForExp(:,16) = pixelOUT*10^-10; - pixelOUT - % Redefining things down hear is a stupid thing to do. Fix this if you - % keep the optino for cropping. - FIXED_FIRSTZERO = pixelOUT / 70 ; - highCutoff = (pixelOUT*10^-10)/pBH.('defCutOff'); + d3 = size(STACK,3); -else + [radialForCTF,phi,~,~,~,~] = ... + BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU', ... + {'none'},1,0,0); - flgCrop = 0; - pixelOUT = PIXEL_SIZE*10^10; - d1C = d1; - d2C = d2; - tltForExp = TLT; + radialForCTF = {radialForCTF./(pixelOUT.*10^-10),0,phi}; clear phi -end - -d3 = size(STACK,3) -% Check for extra large (8k) data which will be too big for the gpu. -% Should set this up to be a hybrid where each slice is on GPU but -% But then pull to the cpu and store there in stack. -if d1C > 4096 || d2C > 4096 || d3 > 40 - prjMaskMethod = 'GPU' - else - prjMaskMethod = 'GPU' -end - -% % % % evalMask = zeros(d1C,d2C,d3,'single'); -% % % % for iPrj = 1:d3 -% % % % tmpTLT = tltForExp(iPrj,:); -% % % % % need to write over the projections position in the stack to not expand beyond 2d -% % % % tmpTLT(1) = 1; -% % % % [ iEvalMask, ~ ] = BH_multi_projectionMask([d1C,d2C,1;d1C,d2C,1], tmpTLT, ... -% % % % 'GPU', [zShift,deltaZTolerance] ); -% % % % -% % % % evalMask(:,:,tltForExp(iPrj,1)) = gather(iEvalMask); -% % % % end -% % % % -% % % % -% % % % %evalMask = gather(evalMask); -% % % % -% % % % -% % % % nTiles = zeros(size(STACK,3),1); -% % % % -% % % % -% % % % for i = 1+tileSize/2:overlap:d1C-tileSize/2 -% % % % for j = 1+tileSize/2:overlap:d2C-tileSize/2 -% % % % for k = 1:size(STACK,3) -% % % % if evalMask(i,j,k) -% % % % nTiles(k) = nTiles(k) + 1; -% % % % end -% % % % end -% % % % end -% % % % end - - - -[radialForCTF,phi,~,~,~,~] = ... - BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU', ... - {'none'},1,0,0); - -radialForCTF = {radialForCTF./(pixelOUT.*10^-10),0,phi}; clear phi - -flgExpFilter = 0; - -inc = (0.5 - FIXED_FIRSTZERO) / (paddedSize/2); -freqVector = [inc+FIXED_FIRSTZERO:inc:0.5 ]; -% % % % clear sumVector radialAvg -% % % % sumVector(length(freqVector)) = gpuArray(double(0)); -% % % % radialAvg(length(freqVector)) = gpuArray(double(0)); - - -tic -nT = 1; -nT2=0; -nT3= 0; - -halfX = floor(paddedSize/2) + 1; -% % % % psTile = zeros([(paddedSize).*[1,1],3],'single','gpuArray'); -psTile = zeros([halfX,paddedSize,3],'single','gpuArray'); - -bhF2 = fourierTransformer(randn(paddedSize,paddedSize,'single','gpuArray')); - -for k = 1:d3 - if (skipFitting) - break - end - - tiltIDX = TLT(k,1); - % Center the pixel coordinates - iEvalMask = BH_multi_gridCoordinates([d1C,1,1],'Cartesian','GPU',{'none'},0,1,0); - - % Convert to the z-height in the projection - iEvalMask = iEvalMask.*(-1.*tand(TLT(tiltIDX,4))); + flgExpFilter = 0; - iEvalPos = iEvalMask; - iEvalNeg = iEvalMask; + inc = (0.5 - FIXED_FIRSTZERO) / (paddedSize/2); + freqVector = [inc+FIXED_FIRSTZERO:inc:0.5 ]; - % Shift by any amount wanted - iEvalPos = iEvalPos - zShift; - iEvalNeg = iEvalNeg + zShift; + tic + nT = 1; + nT2 = 0; + nT3 = 0; - % Select region limited by tolerance - iEvalPos = ( iEvalPos > gpuArray(-deltaZTolerance) & iEvalPos < gpuArray(deltaZTolerance)); - iEvalNeg = ( iEvalNeg > gpuArray(-deltaZTolerance) & iEvalNeg < gpuArray(deltaZTolerance)); - iEvalMask = ( iEvalMask > gpuArray(-deltaZTolerance) & iEvalMask < gpuArray(deltaZTolerance)); - + halfX = emc_get_origin_index(paddedSize); + % % % % psTile = zeros([(paddedSize).*[1,1],3],'single','gpuArray'); + psTile = zeros([halfX,paddedSize,3],'single','gpuArray'); + + bhF2 = fourierTransformer(randn(paddedSize,paddedSize,'single','gpuArray')); - tmpTile = zeros([halfX,paddedSize,3],'single','gpuArray'); - -% % % % tmpTile = zeros([paddedSize.*[1,1],3],'single','gpuArray'); + for k = 1:d3 + if (skipFitting) + break + end + + tiltIDX = TLT(k,1); + % Center the pixel coordinates + iEvalMask = BH_multi_gridCoordinates([d1C,1,1],'Cartesian','GPU',{'none'},0,1,0); + + % Convert to the z-height in the projection + % A positive tilt angle (looking down Y at the origin) is CCW and rotates the positive X axis down in Z farther from focus + % that is the reason for the negative sign + iEvalMask = iEvalMask.*(-1.*tand(TLT(tiltIDX,4))); + + + % zShift is in SI in parameter file, but converted to pixels here. + + % Select region limited by defocus tolerance + iEvalMask = ( iEvalMask > gpuArray(-emc.deltaZTolerance) & iEvalMask < gpuArray(emc.deltaZTolerance)); + + + tmpTile = zeros([halfX,paddedSize,1],'single','gpuArray'); + + % % % % tmpTile = zeros([paddedSize.*[1,1],3],'single','gpuArray'); + + if flgCrop + [iProjection,~] = cropIMG(gpuArray(STACK(:,:,TLT(k,1))),emc.pixel_size_si*10^10); + else + iProjection = (gpuArray(STACK(:,:,TLT(k,1)))); + end + + iProjection = iProjection - ... + BH_movingAverage(iProjection,[emc.ctf_tile_size,emc.ctf_tile_size]); + + iProjection = iProjection ./ ... + BH_movingRMS(iProjection,[emc.ctf_tile_size,emc.ctf_tile_size]); + + reduced_x = floor(emc.ctf_tile_size*cosd(TLT(k,4))); + % ---------------+--------------- + % 000000---------+---------000000 + tile_origin_x = emc_get_origin_index(emc.ctf_tile_size); + reduced_origin_x = emc_get_origin_index(reduced_x); + zeroed_coords = [1:1+(tile_origin_x-reduced_origin_x),(tile_origin_x+reduced_origin_x):emc.ctf_tile_size]; - if flgCrop - [iProjection,~] = cropIMG(gpuArray(STACK(:,:,TLT(k,1))),PIXEL_SIZE*10^10); - else - iProjection = (gpuArray(STACK(:,:,TLT(k,1)))); - end - iProjection = iProjection - ... - BH_movingAverage(iProjection,[tileSize,tileSize]); - - iProjection = iProjection ./ ... - BH_movingRMS(iProjection,[tileSize,tileSize]); - - - - - - for i = 1+tileSize/2:overlap:d1C-tileSize/2 - if min([nT,nT2,nT3])< maxNumberOfTiles && (iEvalMask(i) || iEvalPos(i) || iEvalNeg(i)) - for j = 1+tileSize/2:overlap:d2C-tileSize/2 - - thisTile = abs(bhF2.fwdFFT(BH_padZeros3d(... - (iProjection( ... - i-tileSize/2+1:i+tileSize/2,... - j-tileSize/2+1:j+tileSize/2)),... - padVAL(1,:),padVAL(2,:),... - 'GPU','singleTaper'))); - -% % % % -% % % % thisTile = abs(fftn( ... -% % % % BH_padZeros3d(... -% % % % (iProjection( ... -% % % % i-tileSize/2+1:i+tileSize/2,... -% % % % j-tileSize/2+1:j+tileSize/2)),... -% % % % padVAL(1,:),padVAL(2,:),... -% % % % 'GPU','singleTaper'))); - tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; - - - if (iEvalMask(i)) + for i = 1+emc.ctf_tile_size/2:overlap:d1C-emc.ctf_tile_size/2 + if min([nT,nT2,nT3])< emc.ctfMaxNumberOfTiles && (iEvalMask(i)) + for j = 1+emc.ctf_tile_size/2:overlap:d2C-emc.ctf_tile_size/2 + + thisTile = iProjection( i-emc.ctf_tile_size/2+1:i+emc.ctf_tile_size/2,... + j-emc.ctf_tile_size/2+1:j+emc.ctf_tile_size/2); + + thisTile = thisTile - mean(thisTile(:)); + thisTile = thisTile ./ rms(thisTile(:)); + thisTile(zeroed_coords,:) = 0; + + thisTile = abs(bhF2.fwdFFT(BH_padZeros3d(... + thisTile,... + padVAL(1,:),padVAL(2,:),... + 'GPU','singleTaper'))); + + tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; + + if (iEvalMask(i)) nT = nT+1; tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; - end - if ( iEvalPos(i) ) - nT2 = nT2+1; - tmpTile(:,:,2) = tmpTile(:,:,2) + thisTile; - end - if (iEvalNeg(i) ) - nT3 = nT3+1; - tmpTile(:,:,3) = tmpTile(:,:,3) + thisTile; - end - + end + + end % end of j + end % end of if iEvalMask + end % end of i + + % Apply the dose filter to the sum of each projection to save a bunch of + % multiplicaiton + psTile = psTile + tmpTile; + + end % end of k + clear tmpTile + toc + + rotAvgPowerSpec = zeros([paddedSize,paddedSize,1],'single','gpuArray'); + % tmp = bhF2.swapIndexFWD(psTile(:,:,iTile)); + iTile = 1; + psTile(:,:,iTile) = bhF2.swapIndexFWD(psTile(:,:,iTile)); + rotAvgPowerSpec(:,:,iTile) = BH_multi_makeHermitian(psTile(:,:,iTile),[paddedSize,paddedSize],1); + + clear psTile + + if ~(skipFitting) + AvgPowerSpec = rotAvgPowerSpec; + % TODO make a better rotational averaging funciton + [rot1, rot2, ~, r1,r2, ~] = BH_multi_gridCoordinates(paddedSize.*[1,1], ... + 'Cartesian','GPU', ... + {'none'},0,1,0); + + for i = 0.5:0.5:360 + R = BH_defineMatrix([i,0,0],'Bah','forward'); + ROT1 = R(1).*rot1 + R(4).*rot2; + ROT2 = R(2).*rot1 + R(5).*rot2; + + for iTile = 1 + rotAvgPowerSpec(:,:,iTile) = rotAvgPowerSpec(:,:,iTile) + ... + interpn(r1,r2,AvgPowerSpec(:,:,iTile),... + ROT1,ROT2,'linear',0); end end - end - fprintf('%d tiles at dZ= 0\t%d tiles at dZ > 0\t%d tiles at dZ < 0, after tilt %d\n',nT,nT2,nT3,k); - - % Apply the dose filter to the sum of each projection to save a bunch of - % multiplicaiton - psTile = psTile + tmpTile; - -end -clear tmpTile -toc - -rotAvgPowerSpec = zeros([paddedSize,paddedSize,3],'single','gpuArray'); -for iTile = 1:3 - tmp = bhF2.swapIndexFWD(psTile(:,:,iTile)); - psTile(:,:,iTile) = bhF2.swapIndexFWD(psTile(:,:,iTile)); - rotAvgPowerSpec(:,:,iTile) = BH_multi_makeHermitian(psTile(:,:,iTile),[paddedSize,paddedSize],1); -end - -clear psTile - -if ~(skipFitting) -% % % % for iTile = 1:3 -% % % % rotAvgPowerSpec(:,:,iTile) = (fftshift(rotAvgPowerSpec(:,:,iTile))); -% % % % end - AvgPowerSpec = rotAvgPowerSpec; - - % TODO make a better rotational averaging funciton - [rot1, rot2, ~, r1,r2, ~] = BH_multi_gridCoordinates(paddedSize.*[1,1], ... - 'Cartesian','GPU', ... - {'none'},0,1,0); - - for i = 0.5:0.5:360 - R = BH_defineMatrix([i,0,0],'Bah','forward'); - ROT1 = R(1).*rot1 + R(4).*rot2; - ROT2 = R(2).*rot1 + R(5).*rot2; + clear ROT1 ROT2 + % Normalize on avgerage #, doseWeighting, and a radial filter to account + % for rotational averaging. + rotAvgPowerSpec = rotAvgPowerSpec ./ (720); clear a + % rotAvgPowerSpec = rotAvgPowerSpec ./ (720.*(sqrt(fftshift(radialForCTF{1}.*(pixelOUT.*10^-10))))); clear a - for iTile = 1:3 - rotAvgPowerSpec(:,:,iTile) = rotAvgPowerSpec(:,:,iTile) + ... - interpn(r1,r2,AvgPowerSpec(:,:,iTile),... - ROT1,ROT2,'linear',0); + is_a_bummer = ~isfinite(rotAvgPowerSpec); + if sum(is_a_bummer,'all') > 0.5*numel(rotAvgPowerSpec) + error('the rotated Avg power spectrum is more than half nan or inf'); + else + rotAvgPowerSpec(is_a_bummer) = 0; end - end - clear ROT1 ROT2 - % Normalize on avgerage #, doseWeighting, and a radial filter to account - % for rotational averaging. - rotAvgPowerSpec = rotAvgPowerSpec ./ (720); clear a - % rotAvgPowerSpec = rotAvgPowerSpec ./ (720.*(sqrt(fftshift(radialForCTF{1}.*(pixelOUT.*10^-10))))); clear a - - - is_a_bummer = ~isfinite(rotAvgPowerSpec); - if sum(is_a_bummer,'all') > 0.5*numel(rotAvgPowerSpec) - error('the rotated Avg power spectrum is more than half nan or inf'); - else - rotAvgPowerSpec(is_a_bummer) = 0; - end + + + currentDefocusEst = defEST; + currentDefocusWin = defWIN; + end % end of not skip fitting - - currentDefocusEst = defEST; - currentDefocusWin = defWIN; - measuredVsExpected = zeros(2,3); -end - -for iTilt = 1:3 - - if (skipFitting) - currentDefocusEst = defEST; - - - % Add the determined defocus, and write out with mic paramters as well. - TLT(:,15) = repmat(-1.*defEST,size(TLT,1),1); -% if (flgAstigmatism) && (refineCCC(c,3)~=-9999) -% TLT(:,12) = repmat(gather(refineCCC(c,2)),size(TLT,1),1); -% TLT(:,13) = repmat(gather(refineCCC(c,1)),size(TLT,1),1); -% end + for iTilt = 1 - - [~, idx] = sortrows(abs(TLT(:,4)), -1); - TLT = TLT(idx,:); - % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt - % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast - fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); - fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%8.2f\n'], TLT'); - fclose(fileID); - fprintf('\n\nUsing the estimated value for defocus and phase shift provided\n\n'); - return; - end - - radialAvg = [rotAvgPowerSpec((paddedSize/2)+1:end,(paddedSize/2)+1,iTilt)]'; -% radialPS = [AvgPowerSpec((paddedSize/2)+1:end,(paddedSize/2)+1,iTilt)]'; - - - defRange = [currentDefocusEst-currentDefocusWin,currentDefocusEst+currentDefocusWin]; - -% % % if defRange(2) > -0.05 -% % % fprintf('\n\nCapping defocus to 50nm from wanted %f. Do you mean to search so close to focus??\n\n',abs(defRange(2))); -% % % defRange(2) = -0.05; -% % % end - defInc = [0.01]; - - defVal = (defRange(1):defInc:defRange(2))'; - - cccStorage = zeros(length(defVal),2); - cccStorage(:,1) = defVal; - nDF = 1; - for iDF = defVal' - DF = iDF*10^-6; - - % TODO add a global switch for the damping -% [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); - - if (PIXEL_SIZE < 1*10^-10) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0,-1); - else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); + if (skipFitting && resample_stack) + currentDefocusEst = defEST; + + % Add the determined defocus, and write out with mic paramters as well. + TLT(:,15) = repmat(-1.*defEST,size(TLT,1),1); + + [~, idx] = sortrows(abs(TLT(:,4)), -1); + TLT = TLT(idx,:); + % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt + % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY + % scaleFactor, defocus, emc.pixel_size_si, CS, Wavelength, Amplitude contrast + fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); + fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%8.2f\n'], TLT'); + fclose(fileID); + fprintf('\n\nUsing the estimated value for defocus and phase shift provided\n\n'); + return; end - - try - [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, 0); - catch -% figure, imshow3D(gather(Hqz)); -% figure, imshow3D(gather(rotAvgPowerSpec)); -% highCutoff -% figure, plot(freqVector); -% figure, plot(radialAvg); + + radialAvg = [rotAvgPowerSpec((paddedSize/2)+1:end,(paddedSize/2)+1,iTilt)]'; + + + defRange = [currentDefocusEst-currentDefocusWin,currentDefocusEst+currentDefocusWin]; + + defInc = 0.01; + + defVal = (defRange(1):defInc:defRange(2))'; + + cccStorage = zeros(length(defVal),2); + cccStorage(:,1) = defVal; + nDF = 1; + for iDF = defVal' + DF = iDF*10^-6; + + % TODO add a global switch for the damping + + if (emc.pixel_size_si < 1*10^-10) + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH,DF,paddedSize,-emc.AMPCONT,-1.0,-1); + else + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH,DF,paddedSize,-emc.AMPCONT,-1.0); + end + + try + [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, 0); + catch + % figure, imshow3D(gather(Hqz)); + % figure, imshow3D(gather(rotAvgPowerSpec)); + % highCutoff + % figure, plot(freqVector); + % figure, plot(radialAvg); + end + + [ iCCC ] = calc_CCC( freqVector, bg, bandpass, radialAvg, rV, cccScale); + + cccStorage(nDF, 2) = gather(iCCC); + nDF = nDF +1; end - - [ iCCC ] = calc_CCC( freqVector, bg, bandpass, radialAvg, rV, cccScale); - - cccStorage(nDF, 2) = gather(iCCC); - nDF = nDF +1; - end - - [~,maxVal] = max(cccStorage(:,2)); - maxDef = cccStorage(maxVal,1) - - if (iTilt == 1) + + [~,maxVal] = max(cccStorage(:,2)); + maxDef = cccStorage(maxVal,1); + + if (iTilt == 1) % Only save for the "true" defocus at the tilt-axes - figure('Visible','off'), scatter(cccStorage(:,1), cccStorage(:,2)); - title(sprintf('CCC\nmax = %03.3f micron', maxDef)); - xlabel('defocus (micron)'); ylabel('CCC'); - + figure('Visible','off'), scatter(cccStorage(:,1), cccStorage(:,2)); + title(sprintf('CCC\nmax = %03.3f micron', maxDef)); + xlabel('defocus (micron)'); ylabel('CCC'); + saveas(gcf,sprintf('%s/ctf/%s_ccFIT.pdf',pathName,stackNameOUT), 'pdf') - end + end DF = maxDef*10^-6; - - if (PIXEL_SIZE < 1*10^-10) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0,-1); + + if (emc.pixel_size_si < 1*10^-10) + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH,DF,paddedSize,-emc.AMPCONT,-1.0,-1); else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH,DF,paddedSize,-emc.AMPCONT,-1.0); end - + [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, 0); - - - if (iTilt == 1) + + + if (iTilt == 1) % Only save for the "true" defocus at the tilt-axes - figure('Visible','off'), plot(freqVector(bandpass),backGroundBuffer.*bg(freqVector(bandpass)),freqVector(bandpass),abs(radialAvg(bandpass))); - title('Background fitting') - - saveas(gcf,sprintf('%s/ctf/%s_bgFit.pdf',pathName,stackNameOUT), 'pdf') - end - - - % [ diagnosticIMG ] = make_diagnosticIMG( Hqz, pixelOUT, bandpass, bg, {rotAvgPowerSpec}); - % - % SAVE_IMG(MRCImage(diagnosticIMG), ... - % sprintf('%s/ctf/%s_diag%s',pathName,fileName,extension)); - - clear STACK exposureFilter - - pdfOUT = sprintf('%s/ctf/%s_psRadial_%d.pdf',pathName,stackNameOUT,iTilt) - - + figure('Visible','off'), plot(freqVector(bandpass),backGroundBuffer.*bg(freqVector(bandpass)),freqVector(bandpass),abs(radialAvg(bandpass))); + title('Background fitting') + + saveas(gcf,sprintf('%s/ctf/%s_bgFit.pdf',pathName,stackNameOUT), 'pdf') + end + + + clear STACK exposureFilter + + pdfOUT = sprintf('%s/ctf/%s_psRadial_%d.pdf',pathName,stackNameOUT,iTilt); + + bgSubPS = (abs(radialAvg) - bg(freqVector)').*bandpass; bgSubPS = bgSubPS ./ max(bgSubPS(:)); figure('Visible','off'), plot(freqVector(bandpass)./(pixelOUT),bgSubPS(bandpass), freqVector(bandpass)./(pixelOUT),abs(rV(bandpass)).^2./max(abs(rV(bandpass)).^2),'-g'); title(sprintf('CTF fit\n%03.3f Îŧm ', maxDef)); xlabel('Spatial Frequency (1/Å)'); ylabel('Relative Power'); - + saveas(gcf,pdfOUT, 'pdf') - - - + + + if (flgAstigmatism) - - radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; + + radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; [radialAstig,~,~,~,~,~] = ... - BH_multi_gridCoordinates(size(Hqz),'Cartesian',... - 'GPU',{'none'},1,1,1); - + BH_multi_gridCoordinates(size(Hqz),'Cartesian',... + 'GPU',{'none'},1,1,1); + % Hqz from max defocus still exisists - + [ bg, bandpass, ~ ] = prepare_spectrum( Hqz, highCutoff ,... - freqVector, AvgPowerSpec(:,:,iTilt), radialAstig); - - [ bgSubPS ] = calc_CCC( radialAstig, bg, bandpass, AvgPowerSpec(:,:,iTilt),-9999, cccScale) ; - + freqVector, AvgPowerSpec(:,:,iTilt), radialAstig); + + [ bgSubPS ] = calc_CCC( radialAstig, bg, bandpass, AvgPowerSpec(:,:,iTilt),-9999, cccScale) ; + SAVE_IMG(MRCImage(gather(bgSubPS)), ... - sprintf('%s/ctf/%s_avgPS-bgSub%s',pathName,fileName,extension)); - - - - + sprintf('%s/ctf/%s_avgPS-bgSub%s',pathName,fileName,extension)); + + + + SAVE_IMG(MRCImage(gather(AvgPowerSpec)), ... - sprintf('%s/ctf/%s_avgPS%s',pathName,fileName,extension)); - - + sprintf('%s/ctf/%s_avgPS%s',pathName,fileName,extension)); + + coarseDefSearch = 0:floor(maxAstig/astigStep); coarseAngSearch = -pi/2:coarseAngStep:pi/2; - + fineAngSearch = -1*coarseAngStep/2:fineAngStep:coarseAngStep/2; fineDefSearch = -astigStep/2:astigStep/4:astigStep/2; - - - + + + initAstigCCC = zeros(length(coarseDefSearch)*length(coarseAngSearch),3, 'gpuArray'); - - + + n=1; - + for iAng = coarseAngSearch for iDelDF = coarseDefSearch - df1 = maxDef*10^-6 - iDelDF*astigStep; - df2 = maxDef*10^-6 + iDelDF*astigStep; - - if (PIXEL_SIZE < 1*10^-10) - - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0,-1); - else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0); - end - - % [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff ,... - % freqVector, radialPS, radialAstig); - - - [ iCCC ] = calc_CCC( radialAstig, bgSubPS, bandpass, AvgPowerSpec, Hqz, cccScale) - - - - initAstigCCC(n,:) = [iAng,iDelDF*astigStep,iCCC]; - n = n + 1; - fprintf('%d / %d coarse astigmatism search\n',n,size(initAstigCCC,1)); + df1 = maxDef*10^-6 + iDelDF*astigStep; + df2 = maxDef*10^-6 - iDelDF*astigStep; + + if (emc.pixel_size_si < 1*10^-10) + + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH, ... + [df1,df2,iAng],size(radialForCTF{1}),-emc.AMPCONT,-1.0,-1); + else + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH, ... + [df1,df2,iAng],size(radialForCTF{1}),-emc.AMPCONT,-1.0); + end + + + [ iCCC ] = calc_CCC( radialAstig, bgSubPS, bandpass, AvgPowerSpec, Hqz, cccScale); + + initAstigCCC(n,:) = [iAng,iDelDF*astigStep,iCCC]; + n = n + 1; end end - - + nPeaks = 1; top3 = zeros(nPeaks,3,'gpuArray'); - + for iCCC = 1:nPeaks [~,c] = max(initAstigCCC(:,3)); top3(iCCC,:) = initAstigCCC(c,:); initAstigCCC = initAstigCCC(initAstigCCC(:,1)~=initAstigCCC(c,1),:); end - - top3 - + refineCCC = zeros(length(fineDefSearch)*length(fineAngSearch)*nPeaks,3,'gpuArray'); - + n=1; for iPeak = 1:nPeaks mAng = top3(iPeak,1); mDef = top3(iPeak,2); - - for iAng = fineAngSearch + + for iAng = fineAngSearch for iDelDF = fineDefSearch - - df1 = maxDef*10^-6 - (mDef + iDelDF); - df2 = maxDef*10^-6 + (mDef + iDelDF); - + + df1 = maxDef*10^-6 + (mDef + iDelDF); + df2 = maxDef*10^-6 - (mDef + iDelDF); + % For values very close to zero, the search range may include % values |df1| < |df2| which is against convention. if abs(df1) >= abs(df2) - - if (PIXEL_SIZE < 1*10^-10) - - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng+mAng], ... - size(radialForCTF{1}), -AMPCONT,-1.0,-1); - else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng+mAng], ... - size(radialForCTF{1}), -AMPCONT,-1.0); - end - - + + if (emc.pixel_size_si < 1*10^-10) + + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH, ... + [df1,df2,iAng+mAng], ... + size(radialForCTF{1}), -emc.AMPCONT,-1.0,-1); + else + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH, ... + [df1,df2,iAng+mAng], ... + size(radialForCTF{1}), -emc.AMPCONT,-1.0); + end + + [ iCCC ] = calc_CCC( radialAstig,bgSubPS, bandpass, ... - AvgPowerSpec, Hqz,cccScale ) - + AvgPowerSpec, Hqz,cccScale ); + else - iCCC = -9999 + iCCC = -9999; end - - refineCCC(n,:) = [iAng+mAng,mDef + iDelDF,iCCC]; - n = n + 1; - fprintf('%d / %d fine astigmatism search\n',n,size(refineCCC,1)); + + refineCCC(n,:) = [iAng+mAng,mDef + iDelDF,iCCC]; + n = n + 1; end end end @@ -1029,306 +769,284 @@ topScore = fopen(sprintf('%s/ctf/%s_astig.txt',pathName,fileName),'w'); fprintf(topScore,'%7.7e %7.7e %2.7f\n',refineCCC(c,:)); fclose(topScore); - - - - - end - - if ( iTilt == 1) - - radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; - currentDefocusEst = maxDef; - currentDefocusWin = (defWIN*.25); - measuredVsExpected(1,:) = [maxDef + zShift*PIXEL_SIZE*10^6, maxDef, maxDef - zShift*PIXEL_SIZE*10^6]; - measuredVsExpected(2,2) = maxDef; - % Add the determined defocus, and write out with mic paramters as well. - TLT(:,15) = repmat(maxDef*10^-6,size(TLT,1),1); - if (flgAstigmatism) && (refineCCC(c,3)~=-9999) - TLT(:,12) = repmat(gather(refineCCC(c,2)),size(TLT,1),1); - TLT(:,13) = repmat(gather(refineCCC(c,1)),size(TLT,1),1); + + + + end - % Turn off astigmatism and restrict search range for handedness check. - flgAstigmatism = 0; - - % Sort descending along the magnitude of the tilt angles because higher tilts take - % longer on CTF correction. If more processor available than projections, - % this doesn't affect anything. + if ( iTilt == 1) + + radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; + currentDefocusEst = maxDef; + currentDefocusWin = (defWIN*.25); + measuredVsExpected(1,:) = [maxDef + emc.zShift*emc.pixel_size_si*10^6, maxDef, maxDef - emc.zShift*emc.pixel_size_si*10^6]; + measuredVsExpected(2,2) = maxDef; + % Add the determined defocus, and write out with mic paramters as well. + TLT(:,15) = repmat(maxDef*10^-6,size(TLT,1),1); + if (flgAstigmatism) && (refineCCC(c,3)~=-9999) + TLT(:,12) = repmat(gather(refineCCC(c,2)),size(TLT,1),1); + TLT(:,13) = repmat(gather(refineCCC(c,1)),size(TLT,1),1); + end + + % Turn off astigmatism and restrict search range for handedness check. + flgAstigmatism = 0; + + % Sort descending along the magnitude of the tilt angles because higher tilts take + % longer on CTF correction. If more processor available than projections, + % this doesn't affect anything. + [~, idx] = sortrows(abs(TLT(:,4)), -1); + TLT = TLT(idx,:); + % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt + % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY + % scaleFactor, defocus, emc.pixel_size_si, CS, Wavelength, Amplitude contrast + fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); + fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%8.2f\n'], TLT'); + fclose(fileID); + + end % Stuff we only do on the full determin (tilt1) + + end % Loop on handedness check + +else + % Found an average score: 0.065745 and an average inverted hand score: 0.214261 for tilt tilt60_ali1_ctf + if (resample_stack) [~, idx] = sortrows(abs(TLT(:,4)), -1); TLT = TLT(idx,:); % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast + % scaleFactor, defocus, emc.pixel_size_si, CS, Wavelength, Amplitude contrast fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%8.2f\n'], TLT'); + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%8.2f\n'], TLT'); fclose(fileID); + end +end % end flgSkip - elseif iTilt == 2 - measuredVsExpected(2,1) = maxDef; - elseif iTilt == 3 - measuredVsExpected(2,3) = maxDef; - end % Stuff we only do on the full determin (tilt1) +% TODO should I restart the parallel pool +BH_ctf_Refine2(varargin{1},varargin{2}); -end % Loop on handedness check -if sum(abs(diff(measuredVsExpected,1))) > sum(abs(measuredVsExpected(1,:) - flip(measuredVsExpected(2,:)))) - warnInvertedHand = 1; -else - warnInvertedHand = 0; -end -fprintf('\n******************************************************\n\n'); -fprintf('\nCloser to focus |\tAt focus |\tFarther from focus\n\n'); -fprintf('Expected defocus %3.2f %3.2f %3.2f\n\n', abs(measuredVsExpected(1,:))); -fprintf('Measured defocus %3.2f %3.2f %3.2f\n\n' ,abs(measuredVsExpected(2,:))); -if ( warnInvertedHand ) - fprintf('\nIt looks like your handedness may be inverted!!\n'); -else - fprintf('\nIt looks like your handedness is probably correct.\n'); -end -fprintf('\n******************************************************\n\n\n'); +end % end of ctf estimate function -else - - [~, idx] = sortrows(abs(TLT(:,4)), -1); - TLT = TLT(idx,:); - % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt - % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast - fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); - fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%8.2f\n'], TLT'); - fclose(fileID); +function [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, dualAxis) - -end % end flgSkip -if (do_ctf_refine) - % TODO should I restart the parallel pool - BH_ctf_Refine2(varargin{1},varargin{2}); - +if numel(dualAxis) ~= max(size(dualAxis)) + bandpass2d = dualAxis; + dualAxis = 1; end -end % end of ctf estimate function -function [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, dualAxis) - - if numel(dualAxis) ~= max(size(dualAxis)) - bandpass2d = dualAxis; - dualAxis = 1; - end - - - - - paddedSize = size(Hqz, 1); +paddedSize = size(Hqz, 1); % if (dualAxis) % rV = Hqz(1+paddedSize/2,1+paddedSize/2:end); % else - rV = Hqz(1,1:paddedSize/2); +rV = Hqz(1,1:paddedSize/2); % end - - % smooth the spectrum a little to fit the zeros, and more to fit the max - % values - rVmin = convn(rV,[0.0180,0.0891,0.2327,0.3204,0.2327,0.0891,0.0180],'same'); - [~,firstAbsMax] = max(abs(rV)); - knots = []; - n = 1; +% smooth the spectrum a little to fit the zeros, and more to fit the max +% values +rVmin = convn(rV,[0.0180,0.0891,0.2327,0.3204,0.2327,0.0891,0.0180],'same'); + +[~,firstAbsMax] = max(abs(rV)); +knots = []; +n = 1; +phs = -1; +while n < length(rV) + + % should start negative + + + if phs < 0 + kn = find(rVmin(n:end) >= 0, 1, 'first'); + if isempty(kn) + break + end + phs = 1; + n = n + kn + 2 ; + knots = [knots; n-3]; + else + kn = find(rVmin(n:end) <= 0, 1, 'first'); + if isempty(kn) + break + end phs = -1; - while n < length(rV) + n = n + kn + 2; + knots = [knots; n-3]; + end + +end - % should start negative +% Take 15% of the first peak +halfPastFirstMax = knots(1) - floor(0.15*(knots(1) - firstAbsMax)); +% halfPastFirstMax = knots(2); - if phs < 0 - kn = find(rVmin(n:end) >= 0, 1, 'first'); - if isempty(kn) - break - end - phs = 1; - n = n + kn + 2 ; - knots = [knots; n-3]; - else - kn = find(rVmin(n:end) <= 0, 1, 'first'); - if isempty(kn) - break - end - phs = -1; - n = n + kn + 2; - knots = [knots; n-3]; - end - end +if (dualAxis) + + + nCone = 1; + + for iCone = 2.5:5:90-2.5 + rImg = BH_resample2d(radialAvg,[0,0,iCone],[0,0,0],'Bah','GPU','inv',1,[paddedSize,paddedSize]); + rAvg = abs(gather(rImg(1+paddedSize/2,1+paddedSize/2:end))); - % Take 15% of the first peak - halfPastFirstMax = knots(1) - floor(0.15*(knots(1) - firstAbsMax)); -% halfPastFirstMax = knots(2); - + try + bg{nCone} = fit(gather(freqVector(knots))', ... + double(gather(rAvg(knots))'), ... + 'smoothingSpline'); + catch + bg{nCone} = fit(gather(freqVector(knots(1:end-1))'), ... + double(gather(rAvg(knots(1:end-1)))'), ... + 'smoothingSpline'); + end + nCone = nCone +1; - if (dualAxis) - - - nCone = 1; + end + +else + + if length(knots) >= 2 + try + bg = fit(gather(freqVector(knots))', ... + double(gather(abs(radialAvg(knots)))'), ... + 'smoothingSpline'); - for iCone = 2.5:5:90-2.5 - rImg = BH_resample2d(radialAvg,[0,0,iCone],[0,0,0],'Bah','GPU','inv',1,[paddedSize,paddedSize]); - rAvg = abs(gather(rImg(1+paddedSize/2,1+paddedSize/2:end))); - - - try - bg{nCone} = fit(gather(freqVector(knots))', ... - double(gather(rAvg(knots))'), ... - 'smoothingSpline'); - catch - bg{nCone} = fit(gather(freqVector(knots(1:end-1))'), ... - double(gather(rAvg(knots(1:end-1)))'), ... - 'smoothingSpline'); - end - nCone = nCone +1; - - end + catch + bg = fit(gather(freqVector(knots(1:end-1)))', ... + double(gather(abs(radialAvg(knots(1:end-1))))'), ... + 'smoothingSpline'); - else - - if length(knots) >= 2 - try - bg = fit(gather(freqVector(knots))', ... - double(gather(abs(radialAvg(knots)))'), ... - 'smoothingSpline'); - - catch - bg = fit(gather(freqVector(knots(1:end-1)))', ... - double(gather(abs(radialAvg(knots(1:end-1))))'), ... - 'smoothingSpline'); - - end - - - else - fprintf('fewer than 2 minima detected\n') - end end - bandpass = false(1,length(freqVector)); - bandpass(halfPastFirstMax:find(freqVector > highCutoff, 1, 'first')) = true; - - if (dualAxis) - avg = zeros(size(freqVector))'; - for iFit = 1:nCone-1 - avg = avg + bg{iFit}(freqVector); - end + + else + fprintf('fewer than 2 minima detected\n') + end +end - avg = avg ./ (nCone-1); +bandpass = false(1,length(freqVector)); +bandpass(halfPastFirstMax:find(freqVector > highCutoff, 1, 'first')) = true; - avgFit = fit(gather(freqVector)',avg,'cubicSpline'); - bg = avgFit; - end - +if (dualAxis) + avg = zeros(size(freqVector))'; + for iFit = 1:nCone-1 + avg = avg + bg{iFit}(freqVector); + end + + avg = avg ./ (nCone-1); - if (dualAxis) + avgFit = fit(gather(freqVector)',avg,'cubicSpline'); + bg = avgFit; +end - freqLow = freqVector(find(bandpass,1,'first')); - freqTop = freqVector(find(bandpass,1,'last')); - bandpass = (bandpass2d > freqLow & bandpass2d < freqTop); - end +if (dualAxis) + + freqLow = freqVector(find(bandpass,1,'first')); + freqTop = freqVector(find(bandpass,1,'last')); + bandpass = (bandpass2d > freqLow & bandpass2d < freqTop); + +end end function [ iCCC ] = calc_CCC( freqVector, bg, bandpass, radialAvg, rV, cccScale) + +if (numel(radialAvg) == max(size(radialAvg))) + % One dimensional case + bgSubPS = (abs(radialAvg) - bg(freqVector)').*bandpass; + %bgSubPS = (bgSubPS - mean(bgSubPS(bandpass))) .* bandpass; + if cccScale == 0.5 + % amplify the contribution of higher frequency information that may + % otherwise be overwhelmed - particularly in CCD images. Only option is to + % take the sqrt prior to normalizing. Provides some balance without really + % risking amplifying too much noise. + bgSubPS = (bgSubPS - min(bgSubPS(:)) + 1).^0.5; + elseif cccScale ~= 1 + error('cccScale must be 1 or 0.5') + end - if (numel(radialAvg) == max(size(radialAvg))) - % One dimensional case - bgSubPS = (abs(radialAvg) - bg(freqVector)').*bandpass; - %bgSubPS = (bgSubPS - mean(bgSubPS(bandpass))) .* bandpass; - if cccScale == 0.5 - % amplify the contribution of higher frequency information that may - % otherwise be overwhelmed - particularly in CCD images. Only option is to - % take the sqrt prior to normalizing. Provides some balance without really - % risking amplifying too much noise. - bgSubPS = (bgSubPS - min(bgSubPS(:)) + 1).^0.5; - elseif cccScale ~= 1 - error('cccScale must be 1 or 0.5') - end - - bgSubPS = bgSubPS ./ max(bgSubPS(:)); -% % % % % bgSubPS = bgSubPS - mean(bgSubPS(:)); + bgSubPS = bgSubPS ./ max(bgSubPS(:)); + % % % % % bgSubPS = bgSubPS - mean(bgSubPS(:)); +else + if isnumeric(bg) + % Use existing (otherwise bg is a cfit object) + bgSubPS = bg; else - if isnumeric(bg) - % Use existing (otherwise bg is a cfit object) - bgSubPS = bg; - else bgSubPS = abs(radialAvg) - reshape(bg(freqVector),size(radialAvg)); - + bgSubPS = bgSubPS .* bandpass; - - S = std2(bgSubPS(bandpass)); + + S = std2(bgSubPS(bandpass)); bgSubPS(bgSubPS > S*2.5) = bgSubPS(bgSubPS>S*2.5).* ... - (rand(gather(sum(bgSubPS(:)>S*2.5)),1)+0.5)./2 ; - -% bgSubPS = bgSubPS - BH_movingAverage(bgSubPS,[8,8]); -% bgSubPS = bgSubPS ./ BH_movingRMS(bgSubPS,[8,8]); - -% % % % % bgSubPS = bgSubPS ./ max(abs(bgSubPS(bandpass))); -% % % % % bgSubPS = (bgSubPS - mean(bgSubPS(bandpass))).*bandpass; + (rand(gather(sum(bgSubPS(:)>S*2.5)),1)+0.5)./2 ; + + % bgSubPS = bgSubPS - BH_movingAverage(bgSubPS,[8,8]); + % bgSubPS = bgSubPS ./ BH_movingRMS(bgSubPS,[8,8]); + + % % % % % bgSubPS = bgSubPS ./ max(abs(bgSubPS(bandpass))); + % % % % % bgSubPS = (bgSubPS - mean(bgSubPS(bandpass))).*bandpass; bgSubPS = bgSubPS ./ max(abs(bgSubPS(bandpass))).*bandpass; - end end +end + +if gather(rV(1)) == -9999 + % return the bgSubPS to save + iCCC = bgSubPS; +else + ctfSQ = abs(rV).*bandpass; + % % % % % ctfSQ = (ctfSQ- mean(ctfSQ(bandpass)).*bandpass); + ctfSQ = ctfSQ ./ max(abs(ctfSQ(:))); + iCCC = sum(sum((bgSubPS .* ctfSQ))) ./ ... + ( numel(ctfSQ(bandpass)).*... + std2(ctfSQ(bandpass)).*... + std2(bgSubPS(bandpass)) ); - if gather(rV(1)) == -9999 - % return the bgSubPS to save - iCCC = bgSubPS; - else - ctfSQ = abs(rV).*bandpass; -% % % % % ctfSQ = (ctfSQ- mean(ctfSQ(bandpass)).*bandpass); - ctfSQ = ctfSQ ./ max(abs(ctfSQ(:))); - iCCC = sum(sum((bgSubPS .* ctfSQ))) ./ ... - ( numel(ctfSQ(bandpass)).*... - std2(ctfSQ(bandpass)).*... - std2(bgSubPS(bandpass)) ); +end - end end -function [ diagnosticIMG ] = make_diagnosticIMG( Hqz, pixelSize, bandpass, bg, IMG) +function [ diagnosticIMG ] = make_diagnosticIMG( Hqz, pixel_size_si, bandpass, bg, IMG) - - iImg = 1; - Hqz = fftshift(Hqz); - paddedSize = size(Hqz,1); - [radialGrid,~,~,~,~,~] = ... - BH_multi_gridCoordinates(size(Hqz),'Cartesian',... - 'GPU',{'none'},1,0,1); - - radialGrid = radialGrid ./ pixelSize; - lowCut = radialGrid(1, find(bandpass , 1,'first')); - highCut= radialGrid(1, find(bandpass , 1,'last')); - - radialGrid = fftshift(radialGrid); - bandpass2d = (radialGrid < highCut & radialGrid > lowCut); - bgSubPS2d = (abs(IMG{iImg}) - reshape(bg(radialGrid),size(Hqz))).*bandpass2d; - diagnosticIMG = zeros(size(Hqz)); - - Hqz = abs(Hqz).* bandpass2d; - Hqz = 1.0.*Hqz ./ max(Hqz(bandpass2d)).*bandpass2d; - diagnosticIMG(1:(paddedSize/2),:,1) = gather(Hqz(1:(paddedSize/2),:)); +iImg = 1; +Hqz = fftshift(Hqz); +paddedSize = size(Hqz,1); +[radialGrid,~,~,~,~,~] = ... + BH_multi_gridCoordinates(size(Hqz),'Cartesian',... + 'GPU',{'none'},1,0,1); + +radialGrid = radialGrid ./ pixel_size_si; +lowCut = radialGrid(1, find(bandpass , 1,'first')); +highCut= radialGrid(1, find(bandpass , 1,'last')); + +radialGrid = fftshift(radialGrid); +bandpass2d = (radialGrid < highCut & radialGrid > lowCut); +bgSubPS2d = (abs(IMG{iImg}) - reshape(bg(radialGrid),size(Hqz))).*bandpass2d; + +diagnosticIMG = zeros(size(Hqz)); +Hqz = abs(Hqz).* bandpass2d; +Hqz = 1.0.*Hqz ./ max(Hqz(bandpass2d)).*bandpass2d; +diagnosticIMG(1:(paddedSize/2),:,1) = gather(Hqz(1:(paddedSize/2),:)); - bgSubPS2d = bgSubPS2d ./ max(bgSubPS2d(:)); - diagnosticIMG((paddedSize/2)+1:(paddedSize/2)*2,:,1) = gather(bgSubPS2d((paddedSize/2)+1:(paddedSize/2)*2,:)); + +bgSubPS2d = bgSubPS2d ./ max(bgSubPS2d(:)); +diagnosticIMG((paddedSize/2)+1:(paddedSize/2)*2,:,1) = gather(bgSubPS2d((paddedSize/2)+1:(paddedSize/2)*2,:)); end @@ -1336,215 +1054,215 @@ function [ croppedIMG,pixelOUT ] = cropIMG(IMG,pixelIN) - maxRes = 3.3; - targetNyquist = (0.45*maxRes)/0.5; - [d1,d2] = size(IMG); - - [radialGrid] = BH_multi_gridCoordinates( [d1,d2,1],'Cartesian','GPU', ... - {'none'},1,0,1); - - % For now, pixelIN only applies to pre-fixedpattern noise removal, while - % pixelOUT is the desired final cropping. - radialGrid = radialGrid./pixelIN; - rVX = radialGrid(1:ceil((d1+1)/2)); - rVY = radialGrid(1:ceil((d2+1)/2)); - clear radialGrid - - cutX = find(rVX >= 1/targetNyquist, 1,'first'); - cutY = find(rVY >= 1/targetNyquist, 1,'first'); - - % The actual nyquist will deviate from the target since it is trunctated - % to some pixel value - newNyquist = 1/rVX(cutX); - pixelOUT = gather(0.5*newNyquist); - +maxRes = 3.3; +targetNyquist = (0.45*maxRes)/0.5; +[d1,d2] = size(IMG); +[radialGrid] = BH_multi_gridCoordinates( [d1,d2,1],'Cartesian','GPU', ... + {'none'},1,0,1); - - % Prior approach: fftshift, trim (with taper and bandpass), ifftshift - % Trial approach: shift and trim with logical fftMask, ifftshift - fftMask = BH_fftShift([cutX,cutY],[d1,d2],1); - ifftMask = BH_fftShift(0,-2.*[cutX,cutY],1); +% For now, pixelIN only applies to pre-fixedpattern noise removal, while +% pixelOUT is the desired final cropping. +radialGrid = radialGrid./pixelIN; +rVX = radialGrid(1:ceil((d1+1)/2)); +rVY = radialGrid(1:ceil((d2+1)/2)); +clear radialGrid - croppedIMG = fftn(gpuArray(IMG)); - croppedIMG = croppedIMG(fftMask); +cutX = find(rVX >= 1/targetNyquist, 1,'first'); +cutY = find(rVY >= 1/targetNyquist, 1,'first'); - croppedIMG = real(ifftn(croppedIMG(ifftMask))); - - clear fftMask ifftMask - - - +% The actual nyquist will deviate from the target since it is trunctated +% to some pixel value +newNyquist = 1/rVX(cutX); +pixelOUT = gather(0.5*newNyquist); + + + + +% Prior approach: fftshift, trim (with taper and bandpass), ifftshift +% Trial approach: shift and trim with logical fftMask, ifftshift +fftMask = BH_fftShift([cutX,cutY],[d1,d2],1); +ifftMask = BH_fftShift(0,-2.*[cutX,cutY],1); + +croppedIMG = fftn(gpuArray(IMG)); +croppedIMG = croppedIMG(fftMask); + +croppedIMG = real(ifftn(croppedIMG(ifftMask))); + +clear fftMask ifftMask + + + +end + +function [ tltOrder ] = calc_dose_scheme(emc,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT) + +flgCosineDose = emc.('oneOverCosineDose'); +startingAngle = emc.('startingAngle'); +startingDirection = emc.('startingDirection'); +doseSymmetricIncrement = emc.('doseSymmetricIncrement'); +doseAtMinTilt = emc.('doseAtMinTilt'); +nPrjs = length(rawTLT); +tltOrder = zeros(nPrjs,5); + +nAngle = 2; + +if (doseSymmetricIncrement < 0) + % For doseSymmetricIncrement = 2, 3 deg + % 0, 3, -3, -6, 6, 9 ... + doseSymmetricIncrement = abs(doseSymmetricIncrement); + flgFirstTilt = 0; +else + % For doseSymmetricIncrement = 2, 3 deg + % 0, 3, 6, -3, -6, 9 ... + flgFirstTilt=1; +end + +if any(PHASE_PLATE_SHIFT) + if diff(PHASE_PLATE_SHIFT) < 1e-3 + PHASE_PLATE_SHIFT(2) = PHASE_PLATE_SHIFT(1) + 1e-3; + end + extraPhaseShift = [PHASE_PLATE_SHIFT(1):(PHASE_PLATE_SHIFT(2) - PHASE_PLATE_SHIFT(1))./nPrjs:PHASE_PLATE_SHIFT(2)]; +else + extraPhaseShift = zeros(nPrjs,1); +end + +totalDose = doseAtMinTilt; + +if (anglesSkipped) + % Get the actual angles from the index + anglesToSkip = rawTLT(anglesSkipped); + anglesToKeep = ~ismember(1:nPrjs,anglesSkipped); +else + anglesToSkip = []; + anglesToKeep = true(nPrjs,1); end -function [ tltOrder ] = calc_dose_scheme(pBH,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT) - flgCosineDose = pBH.('oneOverCosineDose'); - startingAngle = pBH.('startingAngle'); - startingDirection = pBH.('startingDirection'); - doseSymmetricIncrement = pBH.('doseSymmetricIncrement'); - doseAtMinTilt = pBH.('doseAtMinTilt'); - nPrjs = length(rawTLT); - tltOrder = zeros(nPrjs,5); +% We always start from the first tilt. +[~,firstTilt] = min(abs(rawTLT-startingAngle)); +tltOrder(firstTilt,:) = [firstTilt,rawTLT(firstTilt),totalDose,extraPhaseShift(1),0]; +% Remove this angle to get those remaining +tmpTLT = rawTLT([1:firstTilt-1,firstTilt+1:end]); +largerAngles = tmpTLT(tmpTLT-startingAngle > 0); +smallerAngles= tmpTLT(tmpTLT-startingAngle < 0); + +% Now split into thos that are larger or smaller than the min tilt +if ( startingAngle >= 0 ) + largerAngles = sort(largerAngles,'ascend'); + smallerAngles = sort(smallerAngles,'descend'); +else + largerAngles = sort(largerAngles,'ascend'); + smallerAngles = sort(smallerAngles,'descend'); +end + +clear tmpTLT + +if ( doseSymmetricIncrement ) + % It is assumed that blocks of this many tilts are collected NOT + % including the first tilt. If the original dose symmetric scheme is + % requested (negative Increment) then the first tilt IS included, and + % so we need to subtract one from the counter. + switchAfterNTilts = doseSymmetricIncrement - (1-flgFirstTilt) + flgFirstTilt=0; +else + if strcmpi(startingDirection,'pos') + switchAfterNTilts = length(largerAngles); + elseif strcmpi(startingDirection,'neg') + switchAfterNTilts = length(smallerAngles); + else + error('flgDose symmetric is 0 and starting direction must be pos or neg'); + end +end + + +% while (~isempty(largerAngles) || ~isempty(smallerAngles)) && nAngle <= nPrjs +for iPrj = 1:nPrjs + if iPrj == firstTilt + continue; + end - nAngle = 2; - - if (doseSymmetricIncrement < 0) - % For doseSymmetricIncrement = 2, 3 deg - % 0, 3, -3, -6, 6, 9 ... - doseSymmetricIncrement = abs(doseSymmetricIncrement); - flgFirstTilt = 0; + if strcmpi(startingDirection,'pos') + try + nextTilt = largerAngles(1); + if length(largerAngles) > 1 + largerAngles = largerAngles(2:end); + else + largerAngles = []; + end + catch + nextTilt = smallerAngles(1); + if length(smallerAngles) > 1 + smallerAngles = smallerAngles(2:end); + else + smallerAngles = []; + end + end + else - % For doseSymmetricIncrement = 2, 3 deg - % 0, 3, 6, -3, -6, 9 ... - flgFirstTilt=1; + try + nextTilt = smallerAngles(1); + if length(smallerAngles) > 1 + smallerAngles = smallerAngles(2:end); + else + smallerAngles = []; + end + catch + nextTilt = largerAngles(1); + if length(largerAngles) > 1 + largerAngles = largerAngles(2:end); + else + largerAngles = []; + end + end end - if any(PHASE_PLATE_SHIFT) - if diff(PHASE_PLATE_SHIFT) < 1e-3 - PHASE_PLATE_SHIFT(2) = PHASE_PLATE_SHIFT(1) + 1e-3; + [~,iTilt] = min(abs(rawTLT-nextTilt)); + + switchAfterNTilts = switchAfterNTilts -1; + + if (switchAfterNTilts == 0) + if strcmpi(startingDirection,'pos') + startingDirection = 'neg'; + elseif strcmpi(startingDirection,'neg') + startingDirection = 'pos'; end - extraPhaseShift = [PHASE_PLATE_SHIFT(1):(PHASE_PLATE_SHIFT(2) - PHASE_PLATE_SHIFT(1))./nPrjs:PHASE_PLATE_SHIFT(2)]; + if ( doseSymmetricIncrement ) + switchAfterNTilts = doseSymmetricIncrement; + end + end + + if (flgCosineDose) + totalDose = totalDose + (1/cosd(rawTLT(iTilt)))*doseAtMinTilt; else - extraPhaseShift = zeros(nPrjs,1); + totalDose = totalDose + doseAtMinTilt; end - totalDose = doseAtMinTilt; - if (anglesSkipped) - % Get the actual angles from the index - anglesToSkip = rawTLT(anglesSkipped); - anglesToKeep = ~ismember(1:nPrjs,anglesSkipped); + % The dose is incremented but don't add to the list. + + if ~ismember(rawTLT(iTilt),anglesToSkip) + tltOrder(iTilt,:) = [iTilt,rawTLT(iTilt),totalDose,extraPhaseShift(nAngle),-1]; else - anglesToSkip = []; - anglesToKeep = true(nPrjs,1); + tltOrder(iTilt,:) = [iTilt,rawTLT(iTilt),-1,-1,-1]; end + nAngle = nAngle + 1; + +end - % We always start from the first tilt. - [~,firstTilt] = min(abs(rawTLT-startingAngle)); - tltOrder(firstTilt,:) = [firstTilt,rawTLT(firstTilt),totalDose,extraPhaseShift(1),0]; - % Remove this angle to get those remaining - tmpTLT = rawTLT([1:firstTilt-1,firstTilt+1:end]); - largerAngles = tmpTLT(tmpTLT-startingAngle > 0); - smallerAngles= tmpTLT(tmpTLT-startingAngle < 0); - - % Now split into thos that are larger or smaller than the min tilt - if ( startingAngle >= 0 ) - largerAngles = sort(largerAngles,'ascend'); - smallerAngles = sort(smallerAngles,'descend'); - else - largerAngles = sort(largerAngles,'ascend'); - smallerAngles = sort(smallerAngles,'descend'); - end - - clear tmpTLT - - if ( doseSymmetricIncrement ) - % It is assumed that blocks of this many tilts are collected NOT - % including the first tilt. If the original dose symmetric scheme is - % requested (negative Increment) then the first tilt IS included, and - % so we need to subtract one from the counter. - switchAfterNTilts = doseSymmetricIncrement - (1-flgFirstTilt) - flgFirstTilt=0; - else - if strcmpi(startingDirection,'pos') - switchAfterNTilts = length(largerAngles); - elseif strcmpi(startingDirection,'neg') - switchAfterNTilts = length(smallerAngles); - else - error('flgDose symmetric is 0 and starting direction must be pos or neg'); - end - end +tltOrder = tltOrder(anglesToKeep,:); +tltOrder(:,5) = tltOrder(:,1); +tltOrder(:,1) = 1:size(tltOrder,1); +tltOrder -% while (~isempty(largerAngles) || ~isempty(smallerAngles)) && nAngle <= nPrjs - for iPrj = 1:nPrjs - if iPrj == firstTilt - continue; - end - - if strcmpi(startingDirection,'pos') - try - nextTilt = largerAngles(1); - if length(largerAngles) > 1 - largerAngles = largerAngles(2:end); - else - largerAngles = []; - end - catch - nextTilt = smallerAngles(1); - if length(smallerAngles) > 1 - smallerAngles = smallerAngles(2:end); - else - smallerAngles = []; - end - end - - else - try - nextTilt = smallerAngles(1); - if length(smallerAngles) > 1 - smallerAngles = smallerAngles(2:end); - else - smallerAngles = []; - end - catch - nextTilt = largerAngles(1); - if length(largerAngles) > 1 - largerAngles = largerAngles(2:end); - else - largerAngles = []; - end - end - end - - [~,iTilt] = min(abs(rawTLT-nextTilt)); - - switchAfterNTilts = switchAfterNTilts -1; - - if (switchAfterNTilts == 0) - if strcmpi(startingDirection,'pos') - startingDirection = 'neg'; - elseif strcmpi(startingDirection,'neg') - startingDirection = 'pos'; - end - if ( doseSymmetricIncrement ) - switchAfterNTilts = doseSymmetricIncrement; - end - end - - if (flgCosineDose) - totalDose = totalDose + (1/cosd(rawTLT(iTilt)))*doseAtMinTilt; - else - totalDose = totalDose + doseAtMinTilt; - end - - - % The dose is incremented but don't add to the list. +size(tltOrder) - if ~ismember(rawTLT(iTilt),anglesToSkip) - tltOrder(iTilt,:) = [iTilt,rawTLT(iTilt),totalDose,extraPhaseShift(nAngle),-1]; - else - tltOrder(iTilt,:) = [iTilt,rawTLT(iTilt),-1,-1,-1]; - end - nAngle = nAngle + 1; - end - - - tltOrder = tltOrder(anglesToKeep,:); - tltOrder(:,5) = tltOrder(:,1); - tltOrder(:,1) = 1:size(tltOrder,1); - tltOrder +% This will be a naive run through that works only if the angles are in +% order. - size(tltOrder) - - - - % This will be a naive run through that works only if the angles are in - % order. - end - + diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index abb5c151..91fa62d0 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -1,22 +1,20 @@ function [ cccStorage, maxAst, maxAng, astigAngSearch] = BH_ctf_Refine2(PARAMETER_FILE, STACK_PRFX) % Script to test refinement of ctf estimate by scaling tiles from tilted images % to change their nominal magnification so that the defocus matches that of the -% mean +% mean % Load in the tomo and tilt info -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + % Load using wrapper + subTomoMeta = BH_loadSubTomoMeta(emc.('subTomoMeta'), emc.('metadata_format')); mapBackIter = subTomoMeta.currentTomoCPR; clear subTomoMeta catch mapBackIter = 0; end -try - testNoRefine = pBH.('force_no_defocus_stretch'); -catch - testNoRefine = false; -end +% force_no_defocus_stretch is now handled in BH_parseParameterFile +testNoRefine = emc.force_no_defocus_stretch; if (testNoRefine) fprintf('\nWarning force_no_defocus_stretch is only for testing!\n\n'); end @@ -24,93 +22,59 @@ % As long as the material coming into view at high tilt is at the same % plane and does not have wildly different image stats, using it could % improve the thon rings on tilted data. Zero will produce the "normal" -% process, 1 will use the same area as the min tilt, -try - fraction_of_extra_tilt_data = pBH.('fraction_of_extra_tilt_data') -catch - fraction_of_extra_tilt_data = 0.25 -end +% process, 1 will use the same area as the min tilt, +% fraction_of_extra_tilt_data is now handled in BH_parseParameterFile +fraction_of_extra_tilt_data = emc.fraction_of_extra_tilt_data; % set the search ranges - should change ctf_est to save the parameters used so % this can be loaded automatically. -nWorkers = min(pBH.('nCpuCores'),7*pBH.('nGPUs')); +nWorkers = min(emc.('nCpuCores'),7*emc.('nGPUs')); nWorkers = BH_multi_parallelWorkers(nWorkers) gpuIDX = BH_multi_checkGPU(-1); gDev = gpuDevice(gpuIDX); -flgAstigmatism= 1; -flgGroupProjections =1;% pBH.('flgGroupProjections'); -calcAvg = 1;%pBH.('flgCalcAvg'); -outputForCTFFIND = 1; + reScaleRealSpace = 0; -normFactor=0;%pBH.('normalizationFactor'); PRJ_STACK = {sprintf('aliStacks/%s_ali%d.fixed',STACK_PRFX,mapBackIter+1)}; -[pathName,fileName,extension] = fileparts(PRJ_STACK{1}) +[pathName,fileName,extension] = fileparts(PRJ_STACK{1}); if isempty(pathName) - pathName = '.' + pathName = '.'; end -PIXEL_SIZE = pBH.('PIXEL_SIZE'); +Cs = emc.('Cs'); +VOLTAGE = emc.('VOLTAGE'); +AMPCONT = emc.('AMPCONT'); -Cs = pBH.('Cs'); -VOLTAGE = pBH.('VOLTAGE'); -AMPCONT = pBH.('AMPCONT') +ctfParams = [emc.pixel_size_si*10^10,VOLTAGE./1000,Cs.*1000,AMPCONT]; -ctfParams = [PIXEL_SIZE*10^10,VOLTAGE./1000,Cs.*1000,AMPCONT] +WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; -% Sanity check -if (PIXEL_SIZE > 20e-10 || PIXEL_SIZE < 0) - error('pixel size should be [0,20e-10]'); -elseif (Cs > 5*10^-3 || Cs < 0) - error('Cs should be[1e-3,10e-3]'); -elseif(VOLTAGE > 1000e3 || VOLTAGE < 20e3) - error ('VOLTAGE should be [20e3,1000e3]'); -elseif (AMPCONT < 0.025 || AMPCONT > 0.25) - error('AMPCONT should be [0.025,0.25]'); -else - WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; -end - -if Cs == 0 - fprintf('You set Cs to zero, over-riding to 5 micron\n'); - Cs = 5e-6; -end - -% Assuming that the first CTF zero is always less than this value -FIXED_FIRSTZERO = PIXEL_SIZE / 40*10^-10 ; - -% highCutoff = PIXEL_SIZE/pBH.('defCutOff'); -highCutoff = 1/pBH.('defCutOff'); - +% Assuming that the first CTF zero is always less than this value +FIXED_FIRSTZERO = emc.pixel_size_si / 40*10^-10 ; % Size to padTile to should be even, large, and preferably a power of 2 -try - paddedSize = pBH.('paddedSize'); -catch - paddedSize = 768; -end +% paddedSize is now handled in BH_parseParameterFile +paddedSize = emc.paddedSize; % Tile size & overlap -tileOverlap = 4; -tileSize = floor(680e-10 / PIXEL_SIZE); -tileSize = tileSize + mod(tileSize,2); -fprintf('Using a tile size of %d',tileSize); -overlap = floor(tileSize ./ tileOverlap) +tileOverlap = emc.('ctf_tile_overlap'); +emc.ctf_tile_size = BH_multi_iterator(emc.ctf_tile_size.*[1,1],'fourier2d'); +emc.ctf_tile_size = emc.ctf_tile_size(1); +% if (tileSize > 512) +% tileOverlap = tileOverlap * 2; +% end +fprintf('Using a tile size of %d\n',emc.ctf_tile_size); +overlap = floor(emc.ctf_tile_size ./ tileOverlap); -% Starting at +/- 750nm -deltaZTolerance = 750e-9 / PIXEL_SIZE; -% Use to check for proper gradient. -zShift = 0; - inc = (0.5 - FIXED_FIRSTZERO) / (paddedSize/2); freqVector = [inc+FIXED_FIRSTZERO:inc:0.5 ]; @@ -119,106 +83,97 @@ %PRJ_OUT = {fileName}; nStacks = length(tlt); - stacksFound = [] +stacksFound = []; INPUT_CELL = cell(nStacks,6); -for i = 1:nStacks - if exist(tlt{i}, 'file') && exist(PRJ_STACK{i}, 'file') - INPUT_CELL{i,1} = load(tlt{i}); - INPUT_CELL{i,2} = PRJ_STACK{i}; - [pathName,fileName,extension] = fileparts(PRJ_STACK{i}); +for iStack = 1:nStacks + if exist(tlt{iStack}, 'file') && exist(PRJ_STACK{iStack}, 'file') + INPUT_CELL{iStack,1} = load(tlt{iStack}); + INPUT_CELL{iStack,2} = PRJ_STACK{iStack}; + [pathName,fileName,extension] = fileparts(PRJ_STACK{iStack}); if isempty(pathName) pathName = '.'; end - INPUT_CELL{i,3} = pathName; - INPUT_CELL{i,4} = fileName; - INPUT_CELL{i,5} = extension; - %INPUT_CELL{i,6} = PRJ_OUT; - + INPUT_CELL{iStack,3} = pathName; + INPUT_CELL{iStack,4} = fileName; + INPUT_CELL{iStack,5} = extension; + %INPUT_CELL{iStack,6} = PRJ_OUT; + else - fprintf('ignoring %s, because the file is not found.\n', tlt{i}) + fprintf('ignoring %s, because the file is not found.\n', tlt{iStack}) end -end - -for iStack = 1%stacksFound - iStack - - STACK = single(getVolume(MRCImage(INPUT_CELL{iStack,2}))); - % The pixel size should be previously set correctly, but if it is not, then we - % must maintain whatever is there in case beads are to be erased. The model - % used for this process depends on the pixel size in the header when it was - % created in IMod alignment. - [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); - iPixelHeader = EMC_str2double(iPixelHeader); - [d1,d2,d3] = size(STACK) - - - - TLT = INPUT_CELL{iStack,1}; - pathName = INPUT_CELL{iStack,3} - fileName = INPUT_CELL{iStack,4} - extension = INPUT_CELL{iStack,5} - - - - SIZEOUT = [d1,d2]; - - - - - - [radialForCTF,phi,~,~,~,~] = ... - BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU',{'none'},1,1,0); +end - radialForCTF = {radialForCTF./PIXEL_SIZE,1,phi} ; - - clear phi - -if (calcAvg) -% [exposureFilter] = BH_exposureFilter(paddedSize.*[1,1], TLT,'cpu',1, 1); +for iStack = 1%stacksFound + STACK = OPEN_IMG('single',INPUT_CELL{iStack,2}); + % The pixel size should be previously set correctly, but if it is not, then we + % must maintain whatever is there in case beads are to be erased. The model + % used for this process depends on the pixel size in the header when it was + % created in IMod alignment. + [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); + iPixelHeader = EMC_str2double(iPixelHeader); + [d1,d2,d3] = size(STACK); + + + + TLT = INPUT_CELL{iStack,1}; + pathName = INPUT_CELL{iStack,3}; + fileName = INPUT_CELL{iStack,4}; + extension = INPUT_CELL{iStack,5}; + + + SIZEOUT = [d1,d2]; + + [radialForCTF,phi,~,~,~,~] = ... + BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU',{'none'},1,1,0); + + + radialForCTF = {radialForCTF./emc.pixel_size_si,1,phi} ; + + clear phi + clear sumVector radialAvg sumVector(length(freqVector)) = gpuArray(double(0)); radialAvg(length(freqVector)) = gpuArray(double(0)); - + tic - - + psTile = zeros([paddedSize,paddedSize,d3],'single'); - - + psTile_inv = zeros([paddedSize,paddedSize,d3],'single'); + flgReplaceStack = 0; for iPrj = 1:d3 iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); iProjection = iProjection - ... - BH_movingAverage(iProjection,[tileSize,tileSize]); + BH_movingAverage(iProjection,[emc.ctf_tile_size,emc.ctf_tile_size]); iProjection = iProjection ./ ... - BH_movingRMS(iProjection,[tileSize,tileSize]); + BH_movingRMS(iProjection,[emc.ctf_tile_size,emc.ctf_tile_size]); % Taking a cue from Alexis maxPixelSizeWanted = 2.0e-10; - if TLT(iPrj,16) < maxPixelSizeWanted + if TLT(iPrj,16) < maxPixelSizeWanted %fprintf(ftmp,'Resampling pixel size\n'); % Resample to 2Ang/pix padSq = BH_multi_padVal(size(iProjection),max(size(iProjection)).*[1,1]); - + iProjection = BH_padZeros3d(iProjection,padSq(1,:),padSq(2,:),'GPU','singleTaper'); sizeIN = size(iProjection,1); % Replace with BH_fftShift if this works iProjection = fftshift(fftn(iProjection)); trimVal = BH_multi_padVal(size(iProjection), floor(size(iProjection).*(TLT(iPrj,16)./maxPixelSizeWanted))); iProjection = real(ifftn(ifftshift(BH_padZeros3d(iProjection,trimVal(1,:),trimVal(2,:),'GPU','single')))); - sizeOUT = size(iProjection); + sizeOUT = size(iProjection); if iPrj == 1 flgReplaceStack = 1; newSTACK = zeros([sizeOUT,d3],'single'); end newSTACK(:,:,TLT(iPrj,1)) = gather(iProjection); - clear iProjection + iProjection = []; % Actual new pixel size pixelSize = sizeIN./sizeOUT(1).*TLT(iPrj,16); @@ -227,69 +182,68 @@ %fprintf(ftmp,'%d %d %d %d %d %d\n',trimVal); %fprintf(ftmp,'pixelOld %3.3e, pixelNew %3.3e\n',TLT(iPrj,16),pixelSize); - + else pixelSize = TLT(iPrj,16); end - end - + end % iPrj 1:d3 + if ( flgReplaceStack ) STACK = newSTACK ; clear newSTACK; end - [d1,d2,d3] = size(STACK); - [Xnew, Ynew, ~, x1,y1, ~] = BH_multi_gridCoordinates([tileSize,d2], ... - 'Cartesian','GPU', ... - {'none'},0,1,0); - - [X, Y, ~,~,~, ~] = BH_multi_gridCoordinates([tileSize,tileSize], ... - 'Cartesian','GPU', ... - {'none'},0,1,0); - - coordShift = (-1).^(X+Y); - clear X Y - % with > 250,000 tiles, it doesn't make sense to call resample2d for just a - % simple rescaling. + [d1,d2,d3] = size(STACK) + + + debug_without_parallel = false; + if (debug_without_parallel) + for iPrj = 1:d3 + fprintf('Calculating stretched tiles on prj %d/ %d in serial debug mode\n',iPrj,d3); + [psTile(:,:,TLT(iPrj,1)),psTile_inv(:,:,TLT(iPrj,1)),pixelSize] = runAvgTiles(TLT, paddedSize, emc.ctf_tile_size, ... + d1,d2, iPrj, overlap, ... + STACK(:,:,TLT(iPrj,1)), ... + 1, ... + 1, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); - try - ppool = EMC_parpool(nWorkers); - catch + end + else + try + ppool = EMC_parpool(nWorkers); + catch delete(gcp('nocreate')); ppool = EMC_parpool(nWorkers); - end - - for iPrj = 1:d3 - - - - pFuture(iPrj) = parfeval(ppool,@runAvgTiles,2, TLT, paddedSize, tileSize, ... - d1,d2, iPrj, overlap, ... - STACK(:,:,TLT(iPrj,1)), ... - 1, ... - 1, ... - x1, y1, Xnew, Ynew,coordShift, ... - reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); - - - end + end + + for iPrj = 1:d3 + + pFuture(iPrj) = parfeval(ppool,@runAvgTiles,3, TLT, paddedSize, emc.ctf_tile_size, ... + d1,d2, iPrj, overlap, ... + STACK(:,:,TLT(iPrj,1)), ... + 1, ... + 1, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); + + + end + + for iWorker = 1:d3 + fprintf('Calculating stretched tiles on prj %d/ %d\n',iWorker,d3); + [iPrj, ctfCorr,ctfCorr_inv, pixelSize] = fetchNext(pFuture); + + psTile(:,:,TLT(iPrj,1)) = ctfCorr; + psTile_inv(:,:,TLT(iPrj,1)) = ctfCorr_inv; + end + end % debug without parallel - for i = 1:d3 - fprintf('Refining defocus on prj %d/ %d\n',i,d3); - [iPrj, ctfCorr,pixelSize] = fetchNext(pFuture); - - psTile(:,:,TLT(iPrj,1)) = ctfCorr; - - - end - %%pixelSize = PIXEL_SIZE*10^10; pixelSize = pixelSize*10^10; SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS.mrc',fileName),pixelSize); bpLog = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); bpLog = bpLog > 0.99; - bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); - bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); + bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.0314,max(8,2.*pixelSize),2.*pixelSize,'GPU',pixelSize)); + bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,40,2.*pixelSize,'GPU',pixelSize)); for iPrj = 1:d3 iTile = gpuArray(psTile(:,:,iPrj)); @@ -298,590 +252,249 @@ psTile(:,:,iPrj) = gather(iTile); end - SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName),pixelSize); - delete(ppool); -else - psTile = gpuArray(getVolume(MRCImage(sprintf('fixedStacks/ctf/%s-PS.mrc',fileName)))); - end - - if (calcAvg) - delete(gcp('nocreate')) - end - - %%%%%%%%%%%%%%%%%%%%%%%%% - if ( outputForCTFFIND ) - % exit an fit the PS using CTFFIND4 - BH_runCtfFind(sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName), ... - sprintf('%s_ctf.tlt',fileName), ctfParams,TLT) - return - end - %%%%%%%%%%%%%%%%%%%%%%%%% - defInc = cell(3,1); astInc = cell(3,1); - defRange = cell(3,1); astRange = cell(3,1); - defSearch = cell(3,d3); astSearch = cell(3,1); - - % Find a close value for symmetric defocus. Do this on a rotationally averaged - % image so that the value is centered between the astigmatic extremes rather - % than potentially sitting at one end or the other. - % Search around this value to get a ballpark on astigmatism. - % Use this astigmatic value to refine symmetric results, and this to refine - % astigmatic results. - % Lather rinse and repeat 1x. - - - defInc{1} = 25*10^-9; - defRange{1} = 1000 *10^-9; - - defInc{2} = 10*10^-9; - defRange{2} = 100*10^-9; - - defInc{3} = 5 *10^-9; - defRange{3} = 50*10^-9; - - maxAstig = 200*10^-9; - astigStep =10*10^-9; - coarseAngStep = (pi/180)*10; - - astigDefSearch{1} = 0:astigStep:maxAstig; -% astigAngSearch{1} = -pi/2:coarseAngStep:pi/2; - astigAngSearch{1} = 0:coarseAngStep:pi; - - astigDefSearch{2} = -5*astigStep:astigStep/2:astigStep*5; - astigAngSearch{2} = -2*coarseAngStep:coarseAngStep/5:coarseAngStep*2; - - astigDefSearch{3} = -3*astigStep:astigStep/4:astigStep*3; - astigAngSearch{3} = -coarseAngStep/2:coarseAngStep/20:coarseAngStep/2; - - astigCCC = cell(3,d3); - for iSearch = 1:3 - for iPrj = 1:d3 - astigCCC{iSearch}{iPrj} = zeros(length(astigDefSearch{iSearch})* ... - length(astigAngSearch{iSearch}),3, 'gpuArray'); - end - end - - rotationalAvg = 1; - if (rotationalAvg) - rotBgSubPS = zeros(size(psTile),'single','gpuArray'); - end - - bgSubPS = zeros(size(psTile),'single','gpuArray'); - minRes = calcMinResolution(TLT, radialForCTF, Cs,WAVELENGTH,AMPCONT); - fprintf('\nMin Resolution fit is %3.3f Angstrom.\n',minRes); - nBgPix = floor(paddedSize.*PIXEL_SIZE.*10^10*sqrt(2)/minRes); - nBgPix = nBgPix + mod(nBgPix,2) - if (normFactor) - nRMSpix = floor(nBgPix/normFactor) + mod(nBgPix/normFactor,2); - end - cccResults = zeros(d3,1); -% + SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName),pixelSize); - [rot1, rot2, ~, r1,r2, ~] = BH_multi_gridCoordinates(paddedSize.*[1,1], ... - 'Cartesian','GPU', ... - {'none'},0,1,0); - if ( flgGroupProjections ) - % Create a temporary copy of the tiles scaled by the sin of the tilt angle - % which results in stronger averaging at high tilts - scaledTile = zeros(size(psTile),'single'); - - % Since they will be averaged, first center and scale the total per - % prj intensities - for iPrj = 1:d3 - psTile(:,:,iPrj) = psTile(:,:,iPrj) - mean(mean(psTile(:,:,iPrj))); - psTile(:,:,iPrj) = psTile(:,:,iPrj) ./ rms(rms( psTile(:,:,iPrj))); - end + SAVE_IMG(MRCImage(gather(psTile_inv)),sprintf('fixedStacks/ctf/%s-PS_inv.mrc',fileName),pixelSize); + for iPrj = 1:d3 - scaledTile(:,:,TLT(iPrj,1)) = scaledTile(:,:,TLT(iPrj,1)) .* ... - abs(sind(TLT(iPrj,4)))+0.05; - end - - end - - for iPrj = 1:d3 - - if (flgGroupProjections) - - % Take the full value at the projection of interest, sin(tiltangle) for the - if iPrj == 1 - gTMP = psTile(:,:,1) + scaledTile(:,:,2) + scaledTile(:,:,3); - elseif iPrj == d3 - gTMP = psTile(:,:,d3) + scaledTile(:,:,d3-1) + scaledTile(:,:,d3-2) - else - gTMP = psTile(:,:,iPrj)+ scaledTile(:,:,iPrj-1) + scaledTile(:,:,iPrj+1); - end - else + iTile = gpuArray(psTile_inv(:,:,iPrj)); + iTile = iTile.*bp.*bp2; + iTile(~bpLog) = mean(iTile(bpLog)); + psTile_inv(:,:,iPrj) = gather(iTile); - gTMP = psTile(:,:,iPrj); end + SAVE_IMG(MRCImage(gather(psTile_inv)),sprintf('fixedStacks/ctf/%s-PS2_inv.mrc',fileName),pixelSize); -% % Take the full value at the projection of interest, sin(tiltangle) for the -% if iPrj == 1 -% gTMP=(psTile(:,:,1) + ... -% psTile(:,:,2).*0.55 +... -% psTile(:,:,3).*0.25)./1.9; -% elseif (iPrj > 1 && iPrj < 7) || (iPrj > d3-6 && iPrj < d3) -% gTMP = (psTile(:,:,iPrj-1).*0.4+... -% psTile(:,:,iPrj)+... -% psTile(:,:,iPrj+1).*0.4)./1.8; -% elseif (iPrj >= 7 && iPrj < 18) -% gTMP = (psTile(:,:,iPrj) + ... -% psTile(:,:,iPrj+1).*0.3)./1.3; -% elseif (iPrj > d3-17 && iPrj <= d3-6) -% gTMP = (psTile(:,:,iPrj-1).*0.3+... -% psTile(:,:,iPrj))./1.3; -% elseif iPrj == d3 -% gTMP = (psTile(:,:,d3-2).*0.25+... -% psTile(:,:,d3-1).*0.55+... -% psTile(:,:,d3))./1.8; -% else -% gTMP = psTile(:,:,iPrj); -% end -% else - -% gTMP = psTile(:,:,iPrj); -% end - - - gTMP = gTMP - BH_movingAverage(gTMP,[nBgPix,nBgPix]); - if (normFactor) - bgSubPS(:,:,iPrj) = gTMP ./ BH_movingRMS(gTMP,floor([nRMSpix,nRMSpix])); - else - bgSubPS(:,:,iPrj) = gTMP; - end - clear gTMP - - if (rotationalAvg) - - rotTMP = bgSubPS(:,:,iPrj); - for i = 0.5:0.5:360 - R = BH_defineMatrix([i,0,0],'Bah','forward'); - ROT1 = R(1).*rot1 + R(4).*rot2; - ROT2 = R(2).*rot1 + R(5).*rot2; - rotTMP = rotTMP + interpn(r1,r2,bgSubPS(:,:,iPrj),ROT1,ROT2,'linear',0); - end - rotBgSubPS(:,:,iPrj) = rotTMP./720; -% rotBgSubPS(:,:,iPrj) = rotBgSubPS(:,:,iPrj) ./ ... -% BH_movingRMS(rotBgSubPS(:,:,iPrj), ... -% [nBgPix,nBgPix]); - clear rotTMP - end - + delete(ppool); + delete(gcp('nocreate')) - - end - + BH_runCtfFind(sprintf('fixedStacks/ctf/%s-PS2',fileName), ... + sprintf('%s_ctf',fileName), ctfParams,TLT) - cccStorage = cell(3,1); - for iRefine = 1:3 - - iRefine - if iRefine == 1 - % Initialize the best defocus from the global estimate for the first iter. - maxDef = zeros(d3,1)+TLT(1,15); - if (flgAstigmatism) - maxAst = zeros(d3,1,'gpuArray')+TLT(1,12); - maxAng = zeros(d3,1,'gpuArray')+TLT(1,13); - else - % Only using rotational average, so don't consider previosuly estimated - % astigmatism. - maxAst = zeros(d3,1,'gpuArray'); - maxAng = zeros(d3,1,'gpuArray'); - end - end + end % do_make_tiles + % exit an fit the PS using CTFFIND4 - for iPrj = 1:d3 - defSearch{iRefine,iPrj} = maxDef(iPrj)-defRange{iRefine}:defInc{iRefine}:maxDef(iPrj)+defRange{iRefine}; - end - cccStorage{iRefine} = zeros(length(defSearch{iRefine,1}),d3); - - - for iDF = 1:length(defSearch{iRefine,1}) - if iRefine == 1 - % On first pass, search range is the same for all projections, so limit - % calcs. - df1 = defSearch{iRefine,iPrj}(iDF) - 0;%maxAst(iPrj); - df2 = defSearch{iRefine,iPrj}(iDF) + 0;%maxAst(iPrj); - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,0],size(radialForCTF{1}), ... - AMPCONT,-1.0); - end - - for iPrj = 1:d3 - - if iRefine > 1 - df1 = defSearch{iRefine,iPrj}(iDF) - maxAst(iPrj); - df2 = defSearch{iRefine,iPrj}(iDF) + maxAst(iPrj); - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,maxAng(iPrj)],size(radialForCTF{1}), ... - AMPCONT,-1.0); - end - - % If not calc astigmatism use rotationally averaged always. - if (iRefine == 1 || ~(flgAstigmatism)) && rotationalAvg - [ iCCC ] = calc_CCC(radialForCTF{1},highCutoff, rotBgSubPS(:,:,iPrj), Hqz,0); - else - [ iCCC ] = calc_CCC(radialForCTF{1},highCutoff, bgSubPS(:,:,iPrj), Hqz,0); - end - cccStorage{iRefine}(iDF,iPrj) = gather(iCCC); - end - - - end - % get the max scores for this iteration per projection - maxDef = zeros(d3,1); - for iPrj = 1:d3 - [~,c]=max(cccStorage{iRefine}(:,iPrj)); - maxDef(iPrj) = defSearch{iRefine,iPrj}(c); - end - - - if (flgAstigmatism) - % If not leave maxAstig and maxAngle set to their initial values from the - % TLT geometry. - for iPrj = 1:d3 - n=1; - iPrj - for iAng = astigAngSearch{iRefine} - for iDelDF = astigDefSearch{iRefine} - iDelDfFull = iDelDF + maxAst(iPrj); - iAngFull = iAng + maxAng(iPrj); - - df1 = maxDef(iPrj) - iDelDfFull; - df2 = maxDef(iPrj) + iDelDfFull; +end - % For values very close to zero, the search range may include - % values |df1| < |df2| which is against convention. - if abs(df1) >= abs(df2) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAngFull],size(radialForCTF{1}), ... - AMPCONT,-1.0); - [ iCCC ] = calc_CCC(radialForCTF{1},highCutoff, bgSubPS(:,:,iPrj), Hqz,1); - %fprintf('%d / %d coarse astigmatism search\n',n,size(initAstigCCC,1)); - else +function [psTile_out,psTile_inv_out,pixelSize] = runAvgTiles(TLT, paddedSize, tileSize, d1,d2, iPrj, overlap, ... + iProjection, evalMask, ... + ddZ, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine) - iCCC = -9999 +DFo = abs(TLT(iPrj,15)); - end +padTileOver = 256; - astigCCC{iRefine}{iPrj}(n,:) = [iAngFull,iDelDfFull,iCCC]; - n = n + 1; - end - end - end +tiltOrigin = ceil((size(iProjection,1)+1)./2); - for iPrj = 1:d3 - [~,c]=max(astigCCC{iRefine}{iPrj}(:,3)); - maxAst(iPrj) = astigCCC{iRefine}{iPrj}(c,2); - maxAng(iPrj) = astigCCC{iRefine}{iPrj}(c,1); - end +oXprj = ceil((size(iProjection,1)+1)./2); +% Don't worry about extending the edges for thickness +half_width = (size(iProjection,1)/2); +halfX = emc_get_origin_index(paddedSize); - maxAst - maxAng - end - end - - +maxEval = (fraction_of_extra_tilt_data + ... + cosd(TLT(iPrj,4)).*(1-fraction_of_extra_tilt_data)) .* half_width; - - +iEvalMask = floor(oXprj-maxEval):ceil(oXprj+maxEval); +% iEvalMask = BH_multi_gridCoordinates([size(iProjection,1),1,1],'Cartesian','GPU',{'none'},0,1,0); -avgCCC=0; -for iPrj = 1:d3 - avgCCC = avgCCC+maxDef(iPrj); -end -avgCCC = avgCCC ./ d3; +psTile = zeros([halfX,paddedSize,1], 'single','gpuArray'); +psTile_inv = zeros([halfX,paddedSize,1], 'single','gpuArray'); +% Since I'm enforcing Y-tilt axis, then this could be dramatically sped up +% by resampling strips along the sampling +bhF1 = fourierTransformer(zeros([paddedSize,paddedSize],'single','gpuArray')); - - for iPrj = 1:d3 +for tilt_sign = [-1,1] + for iOuter = 1+tileSize/2:overlap:d1-tileSize/2 - defAstig = [maxDef(iPrj) - maxAst(iPrj), maxDef(iPrj) + maxAst(iPrj),maxAng(iPrj) ]; - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,defAstig,paddedSize,AMPCONT,-1.0); - - - [ ~, bandpass ] = calc_CCC(radialForCTF{1},highCutoff, bgSubPS(:,:,iPrj), Hqz, 1); - - bgSubPS(:,:,iPrj) = bgSubPS(:,:,iPrj) .* bandpass; - if (rotationalAvg) - rotBgSubPS(:,:,iPrj) = rotBgSubPS(:,:,iPrj) .* bandpass; + if (iOuter < tileSize/2 || iOuter > d1-tileSize/2) + continue; end - end - - - -SAVE_IMG(MRCImage(gather(bgSubPS)),sprintf('fixedStacks/ctf/%s_bgOUT.mrc',fileName)); -clear bgSubPS -if (rotationalAvg) - SAVE_IMG(MRCImage(gather(rotBgSubPS)),sprintf('fixedStacks/ctf/%s_bgOUT_rotAvg.mrc',fileName)); -end -clear rotBgSubPS - -ang = TLT(:,4); -r1 = maxDef(TLT(:,1)); -r2 = fit(ang, r1, 'smoothingSpline'); -r3 = fit(ang, r2(ang), 'smoothingSpline'); - -ast1 = fit(ang,gather(maxAst(TLT(:,1))), 'smoothingSpline'); -ang1 = fit(ang,gather(maxAng(TLT(:,1))), 'smoothingSpline'); + + % Slightly randomize the step size to avoid a Moire like effect that + % presents particulary strongly with a continuous carbon layer. -figure('Visible','off'), ... -plot(ang,r3(ang),'bo',ang,zeros(1,d3)+TLT(1,15),'b--', ... - ang,zeros(1,d3)+TLT(1,15)+defRange{1},'k--',... - ang,zeros(1,d3)+TLT(1,15)-defRange{1},'k--'); - title(sprintf('CTF refine\nmeanDef %03.3f Îŧm\nmeanDefPerTilt %03.3f Îŧm ', TLT(1,15)*10^6,avgCCC*10^6)); - xlabel('Projection'); ylabel('defocus'); - - saveas(gcf,sprintf('fixedStacks/ctf/%s_refine.pdf',fileName), 'pdf'); + iDeltaZ = (iOuter - tiltOrigin)*pixelSize*tilt_sign.*tand(TLT(iPrj,4)); + if any(ismember(iOuter-tileSize/2+1:iOuter+tileSize/2,iEvalMask)) %evalMask(iOuter,paddedSize/2+1) + + % The formulat is 1 + deltaDefocus / defocus, and deltaZ = - deltaDefocus + mag = (1-iDeltaZ./DFo).^0.5; + if ~isfinite(mag) + error('mag is not finite'); + end + + estSize = tileSize(1); + ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); + ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo-iDeltaZ,estSize,TLT(iPrj,19),-1,1); + ctf1 = ctf1(1:estSize/2); + ctf2 = ctf2(1:estSize/2); + firstZero = find(ctf1 > 0, 1, 'first'); + % secondZero= find(ctf1(firstZero:end) < 0 , 1, 'first') + firstZero - 1; + + %fprintf(ftmp,'firstZero %d %2.2f\n',firstZero,estSize/firstZero*pixelSize); + % This range will depend on the size of the field of view. For now, setting manually for Florian's HIV + % data, but will derive a formula to make sure the search is appropriate. Here we expect at most ~ 300 nm + % deltaZ, the strongest difference is at the lowest defocus which is ~ 1500 nm, which gives an estimated mag + % ~ 1.095 + defRange = mag-.1:.001:mag+.1; + nDef = length(defRange); + scoreDef = zeros(nDef,1,'gpuArray'); + for iDef = 1:nDef + ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); + % Larger scalings will have zeros rather than extroplation, so% + % % don't let this influence the score. + % lastZero = find(abs(ci) > 0 , 1, 'last'); + %fprintf(ftmp,'%d %d %d %d',size(ci),size(ctf1)); + scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); + + end + [~,maxCoord] = max(scoreDef); + mag = defRange(maxCoord); + + defRange = mag-.01:.0001:mag+.01; + nDef = length(defRange); + scoreDef = zeros(nDef,1,'gpuArray'); + for iDef = 1:nDef + ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); + scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); + end + [~,maxCoord] = max(scoreDef); + mag = defRange(maxCoord); + + if (testNoRefine) + mag = 1; + end + + reduced_x = floor(tileSize*cosd(TLT(iPrj,4))); + % ---------------+--------------- + % 000000---------+---------000000 + tile_origin_x = emc_get_origin_index(tileSize); + reduced_origin_x = emc_get_origin_index(reduced_x); + zeroed_coords = [1:1+(tile_origin_x-reduced_origin_x),(tile_origin_x+reduced_origin_x):tileSize]; - if (flgAstigmatism) - figure('Visible','off'), ... - plot(ang,ast1(TLT(:,4)).*10^9,'bo',ang,(180/pi).*ang1(TLT(:,4)),'go'); - title('CTF refine astigmatism'); - xlabel('Projection'); ylabel('astig(nm) angle(deg)'); - saveas(gcf,sprintf('fixedStacks/ctf/%s_astig.pdf',fileName), 'pdf'); - end - TLT(:,15) = r3(TLT(:,4)); - if (flgAstigmatism) - TLT(:,12) = ast1(TLT(:,4)); - TLT(:,13) = ang1(TLT(:,4)); - end - fileID = fopen(sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',STACK_PRFX,mapBackIter+1), 'w'); - fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\n'], TLT'); - fclose(fileID); - -end -clear -for i = 1:gpuDeviceCount - gpuDevice(i); -end -end + scaled_size = floor([paddedSize,paddedSize] .* mag); + scaled_size = scaled_size + mod(scaled_size,2); + cut_out_padVal = BH_multi_padVal([tileSize,tileSize],paddedSize.*[1,1]); + tile_padVal_RealSpace = BH_multi_padVal([(paddedSize),paddedSize],[(scaled_size(1)), scaled_size(2)]); + scaled_halfX = emc_get_origin_index(scaled_size(1)) + scaled_paddedX = emc_get_origin_index(paddedSize) + tile_padVal_FourierSpace = ([scaled_halfX, scaled_size(2)] - [scaled_paddedX,paddedSize]) ./ 2; + real_resize_increase = all(tile_padVal_RealSpace >= 0); + fourier_resize_increase = all(tile_padVal_FourierSpace >= 0); + padded_fft = zeros([scaled_halfX, scaled_size(2)], 'single','gpuArray'); + [bhF2] = fourierTransformer(zeros(scaled_size,'single','gpuArray')); + for y = 1+tileSize/2:overlap:d2-tileSize/2 + iTile = gpuArray(iProjection(iOuter-tileSize/2+1:iOuter+tileSize/2,y-tileSize/2+1:y+tileSize/2)); + iTile(zeroed_coords,:) = 0; + iTile = iTile - mean(iTile(:)); + iTile = iTile ./ rms(iTile(:)); + + iTile = BH_padZeros3d(iTile,'fwd',cut_out_padVal,'GPU','singleTaper'); + + % iTile = fftshift(fftn(scaledStrip(:,index_into)));%.*coordShift; + % iTile = gpuArray(scaledStrip(:,y-tileSize/2+1:y+tileSize/2));%.*coordShift; + iTile = bhF1.swapIndexFWD(bhF1.fwdFFT(iTile)); + -function [ iCCC, bandpass ] = calc_CCC(radialForCTF,highCutoff, bgSubPS, Hqz, flgAst) + + if (fourier_resize_increase) + padded_fft = padded_fft .* 0; + padded_fft(1:size(iTile,1),1 + tile_padVal_FourierSpace(2):size(iTile,2) + tile_padVal_FourierSpace(2)) = iTile; + else + padded_fft = iTile(1:scaled_halfX,1 - tile_padVal_FourierSpace(2):size(iTile,2) + tile_padVal_FourierSpace(2)); + end + iTile = real(bhF2.invFFT(bhF2.swapIndexINV(padded_fft))); - ctfSQ = abs(Hqz); - - rV = Hqz(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); - freqVector = radialForCTF(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); - firstZero = find(rV > 0, 1,'first'); - secondZero = find(rV(firstZero+1:end) < 0, 1,'first') + firstZero; + % % Slightly randomize scaling + % if (randi(2,1) == 2) + % scaledSize = ceil(size(iTile) .* mag) + randi(2,1) -1; + % else + % scaledSize = floor(size(iTile) .* mag)+ randi(2,1) -1; + % end + + % tile_padVal = BH_multi_padVal(size(iTile),scaledSize); + % iTile = real(ifftn(ifftshift(BH_padZeros3d(iTile,'fwd',tile_padVal,'GPU','singleTaper', 0)))); - [~,firstMax]=min(abs(rV(firstZero:secondZero-1)- ... - rV(firstZero+1:secondZero))) ; - firstMax = firstMax + firstZero; -% if (flgAst) -% bandpass = ( radialForCTF > freqVector(firstZero) & ... -% radialForCTF < highCutoff & ctfSQ < 3*rms(ctfSQ(:))); -% else -% % % bandpass = ( radialForCTF > freqVector(firstMax) & ... -% % % radialForCTF < highCutoff ); -% end - - lowRes = radialForCTF > freqVector(firstZero) & radialForCTF < freqVector(firstMax); - useRes = radialForCTF > freqVector(firstZero) & radialForCTF < highCutoff; - bandpass = single( useRes ); - bandpass(lowRes) = ctfSQ(lowRes).^4; - - - ctfSQ = ctfSQ .* bandpass; - - bgSubPS = bgSubPS .* bandpass; - - - iCCC = sum(sum((bgSubPS(useRes) .* ... - ctfSQ(useRes)))) ./ ... - ( numel(ctfSQ(useRes)).*... - std2(ctfSQ(useRes)).*... - std2(bgSubPS(useRes)) ); + % Do the final forwardSwap at the end + iTile = bhF1.fwdFFT(BH_padZeros3d(iTile, 'inv', tile_padVal_RealSpace, 'GPU','singleTaper', mean(iTile(:)))); + + % iTile = fftshift(abs(fftn(BH_padZeros3d(iTile, 'fwd', iPadVal, ... + % 'GPU','singleTaper', mean(iTile(:)))))); + + if (tilt_sign == -1) + psTile = psTile + abs(iTile); + else + psTile_inv = psTile_inv + abs(iTile); + end + + + end % loop over y + end % if over eval mask + end % over tiles +end % loop over tilt_sign - -end +psTile = bhF1.swapIndexFWD(psTile); +psTile_inv = bhF1.swapIndexFWD(psTile_inv); +psTile = psTile - mean(psTile(:)); +psTile = psTile ./ rms(psTile(:)); +oob = psTile > 3; +psTile(oob) = 3 + rand([sum(oob(:)),1],'single','gpuArray')./3; -function [psTile,pixelSize] = runAvgTiles(TLT, paddedSize, tileSize, d1,d2, iPrj, overlap, ... - iProjection, evalMask, ... - ddZ, x1, y1, Xnew, Ynew, coordShift, ... - reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine) +psTile_inv = psTile_inv - mean(psTile_inv(:)); +psTile_inv = psTile_inv ./ rms(psTile_inv(:)); +oob = psTile_inv > 3; +psTile_inv(oob) = 3 + rand([sum(oob(:)),1],'single','gpuArray')./3; - DFo = TLT(iPrj,15); - padTileOver = 256; - tmpTile = zeros(paddedSize.*[1,1]+2*padTileOver,'single','gpuArray'); - - tiltOrigin = ceil((size(iProjection,1)+1)./2); - - oXprj = ceil((size(iProjection,1)+1)./2); - % Don't worry about extending the edges for thickness - half_width = (size(iProjection,1)/2); +psTile_out = gather(BH_multi_makeHermitian(psTile, [paddedSize,paddedSize], 1)); +psTile_inv_out = gather(BH_multi_makeHermitian(psTile_inv, [paddedSize,paddedSize], 1)); - maxEval = (fraction_of_extra_tilt_data + ... - cosd(TLT(iPrj,4)).*(1-fraction_of_extra_tilt_data)) .* half_width; - iEvalMask = floor(oXprj-maxEval):ceil(oXprj+maxEval); - - % Since I'm enforcing Y-tilt axis, then this could be dramatically sped up - % by resampling strips along the sampling - for iOuter = 1+tileSize/2:overlap:d1-tileSize/2 - randSize = randi(floor(overlap/2),1); - if (randi(2,1) == 2) - randSize = -1*randSize; - end - i = iOuter + randSize; - if (i < tileSize/2 || i > d1-tileSize/2) - continue; - end - - % Slightly randomize the step size to avoid a Moire like effect that - % presents particulary strongly with a continuous carbon layer. - - iDeltaZ = (i - tiltOrigin)*pixelSize*-1.*tand(TLT(iPrj,4)); - if any(ismember(i-tileSize/2+1:i+tileSize/2,iEvalMask)) %evalMask(i,paddedSize/2+1) - doSplineInterp=1; - - mag = (1+iDeltaZ./DFo).^0.5; - - estSize = 2048; - ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); - ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),iDeltaZ+DFo,estSize,TLT(iPrj,19),-1,1); - ctf1 = ctf1(1:estSize/2); - ctf2 = ctf2(1:estSize/2); - firstZero = find(ctf1 > 0, 1, 'first'); - secondZero= find(ctf1(firstZero:end) < 0 , 1, 'first') + firstZero - 1; - - %fprintf(ftmp,'firstZero %d %2.2f\n',firstZero,estSize/firstZero*pixelSize); - % This range will depend on the size of the field of view. For now, setting manually for Florian's HIV - % data, but will derive a formula to make sure the search is appropriate. Here we expect at most ~ 300 nm - % deltaZ, the strongest difference is at the lowest defocus which is ~ 1500 nm, which gives an estimated mag - % ~ 1.095 - defRange = mag-.1:.001:mag+.1; - nDef = length(defRange); - scoreDef = zeros(nDef,1,'gpuArray'); - for iDef = 1:nDef - ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); - % Larger scalings will have zeros rather than extroplation, so% - % % don't let this influence the score. - % lastZero = find(abs(ci) > 0 , 1, 'last'); - %fprintf(ftmp,'%d %d %d %d',size(ci),size(ctf1)); - scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - - end - [~,maxCoord] = max(scoreDef); - mag = defRange(maxCoord); - - defRange = mag-.01:.0001:mag+.01; - nDef = length(defRange); - scoreDef = zeros(nDef,1,'gpuArray'); - for iDef = 1:nDef - ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); - scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - end - [~,maxCoord] = max(scoreDef); - - mag = defRange(maxCoord); - - -% testNoRefine = 0; - if (testNoRefine) - mag = 1 - end - - - scaledStrip = iProjection(i-tileSize/2+1:i+tileSize/2,:); - - - - - for j = 1+tileSize/2:overlap:d2-tileSize/2 - - iTile = scaledStrip(:,j-tileSize/2+1:j+tileSize/2);%.*coordShift; - if (reScaleRealSpace) - scaledSize = paddedSize; - else - % Slightly randomize scaling - if (randi(2,1) == 2) - scaledSize = ceil(paddedSize .* mag) + randi(2,1) -1; - else - scaledSize = floor(paddedSize .* mag)+ randi(2,1) -1; - end - %scaledSize = floor(paddedSize ./ mag); - end - - [oX,oY] = size(tmpTile); - oX = ceil((oX+1)./2); - oY = ceil((oY+1)./2); - - - iPadVal = BH_multi_padVal(size(iTile),[scaledSize,scaledSize]); - - - oupSize = [floor(scaledSize./2),ceil(scaledSize./2); ... - floor(scaledSize./2),ceil(scaledSize./2)]; - - % Get rid of th fftshift - tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... - oY-oupSize(2,1):oY+oupSize(2,2)-1) = ... - tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... - oY-oupSize(2,1):oY+oupSize(2,2)-1) + ... - fftshift(abs(fftn(BH_padZeros3d(iTile,iPadVal(1,:),iPadVal(2,:), ... - 'GPU','singleTaper', mean(iTile(:)))))); - % Using singleTaper here produces - % a grid like artifact. Test - % switch for EMC functions - - - - - - end - end - end - - - psTile = gather(BH_padZeros3d(tmpTile, [-1,-1].* ... - padTileOver,[-1,-1].*padTileOver,... - 'GPU','single')); - clear tmpTile iProjection ddZ evalMask Xnew Ynew x1 y1 +% psTile = gather(psTile); +% psTile_inv = gather(psTile_inv); +clear tmpTile iProjection ddZ evalMask Xnew Ynew x1 y1 end - + function [minRes] = calcMinResolution(TLT, radialForCTF,Cs,WAVELENGTH,AMPCONT) - meanDef = mean(TLT(:,15)); - meanAst = mean(TLT(:,12)); - meanAng = mean(TLT(:,13)); +meanDef = abs(mean(TLT(:,15))); +meanAst = mean(TLT(:,12)); +meanAng = mean(TLT(:,13)); - df1 = meanDef - meanAst; - df2 = meanDef + meanAst; +df1 = meanDef + meanAst; +df2 = meanDef - meanAst; - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,meanAng],size(radialForCTF{1}), ... - AMPCONT,-1.0); - rV = Hqz(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); - freqVector = radialForCTF{1}(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); - firstZero = find(rV > 0, 1,'first'); +[ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [df1,df2,meanAng],size(radialForCTF{1}), ... + AMPCONT,-1.0); +rV = Hqz(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); +freqVector = radialForCTF{1}(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); +firstZero = find(rV > 0, 1,'first'); - minRes = 1/freqVector(firstZero) * 10^10; +minRes = 1/freqVector(firstZero) * 10^10; end diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 21158b0d..2fdd0a33 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -1,76 +1,63 @@ function [ ] = BH_ctf_Updatefft( PARAMETER_FILE, STACK_PRFX, applyFullorUpdate) -global bh_global_do_2d_fourier_interp; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); flgSkipUpdate = 0; % To avoid accidently masking any failures in subsequent update, clean out % all stacks and reconstructions from the local cache. -try - eucentric_minTilt = pBH.('eucentric_minTilt'); -catch - eucentric_minTilt = 15; -end -try - flgShiftEucentric = pBH.('eucentric_fit') -catch - flgShiftEucentric = 0 -end -try - % Should be negative, but to test. - defShiftSign = pBH.('testFlipSign'); -catch - defShiftSign = -1; -end -try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR; -catch - mapBackIter = 0; -end +% defShiftSign is now handled in BH_parseParameterFile +defShiftSign = emc.defShiftSign; -if isnan(EMC_str2double(STACK_PRFX)) - % It is a name, run here. - nGPUs = 1; - flgParallel = 0; - STACK_LIST = {STACK_PRFX}; - ITER_LIST = {STACK_LIST}; +% Load using wrapper +subTomoMeta = BH_loadSubTomoMeta(emc.('subTomoMeta'), emc.('metadata_format')); +if isfield(subTomoMeta, 'currentTomoCPR') + mapBackIter = subTomoMeta.currentTomoCPR; else - flgParallel = 1; + % mapBackIter is handled in BH_parseParameterFile as fallback + mapBackIter = emc.mapBackIter; +end +% if isnan(EMC_str2double(STACK_PRFX)) +% % It is a name, run here. +% nGPUs = 1; +% flgParallel = 0; +% STACK_LIST = {STACK_PRFX}; +% ITER_LIST = {STACK_LIST}; +% else + flgParallel = 1; + updateCMD = sprintf('%s,%d,TiltAlignment,UpdateTilts,[%d,0,0],STD', ... - PARAMETER_FILE,subTomoMeta.currentCycle, ... - subTomoMeta.currentCycle); - % fprintf('%s/n',updateCMD); - nGPUs = pBH.('nGPUs'); - ITER_LIST = cell(nGPUs,1); - + PARAMETER_FILE,subTomoMeta.currentCycle, ... + subTomoMeta.currentCycle); + % fprintf('%s/n',updateCMD); + nGPUs = emc.('nGPUs'); + nWorkers_per_gpu = 2; + nWorkers = nWorkers_per_gpu*nGPUs; + ITER_LIST = cell(nWorkers,1); + [STACK_LIST, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); clear STACK_LIST_tmp - for iGPU = 1:nGPUs - ITER_LIST{iGPU} = STACK_LIST(iGPU:nGPUs:nTiltSeries); + for iGPU = 1:nWorkers + ITER_LIST{iGPU} = STACK_LIST(iGPU:nWorkers:nTiltSeries); end -end +% end eucShiftsResults = 0; -if flgShiftEucentric +if emc.eucentric_fit eucShiftsResults = cell(size(ITER_LIST)); end % BH_geometryAnalysis(sprintf('%s',PARAMETER_FILE),sprintf('%d',subTomoMeta.currentCycle),'TiltAlignment','UpdateTilts',sprintf('[%d,0,0]',subTomoMeta.currentCycle),'STD') -try - conserveDiskSpace = pBH.('conserveDiskSpace'); -catch - conserveDiskSpace = 0; -end +% conserveDiskSpace is now handled in BH_parseParameterFile +conserveDiskSpace = emc.conserveDiskSpace; try - EMC_parpool(nGPUs); + EMC_parpool(nWorkers); catch delete(gcp('nocreate')); - EMC_parpool(nGPUs); + EMC_parpool(nWorkers); end % Assuming that mapBackIter > 0 since we are updating @@ -82,13 +69,14 @@ % For some reason matlab was geeking out about calling this in the parfor % loop, getting confused about whether it is a variable or a function. -recGeomForThickness = subTomoMeta.reconGeometry; -parfor iGPU = 1:nGPUs -% for iTilt = 1:length(ITER_LIST{iGPU}) - +parfor iGPU = 1:nWorkers +% for iGPU = 1:nGPUs + + % for iTilt = 1:length(ITER_LIST{iGPU}) + if ( flgParallel ) - useGPU = iGPU; - gDev = gpuDevice(useGPU); + useGPU = floor((1+iGPU)/nWorkers_per_gpu); + gpuDevice(useGPU); else useGPU = BH_multi_checkGPU(-1); gDev = gpuDevice(useGPU); @@ -96,512 +84,318 @@ for iTilt = 1:length(ITER_LIST{iGPU}) + STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - - if (mapBackIter) - mapBackPrfx = sprintf('mapBack%d/%s_ali%d_ctf',mapBackIter,STACK_PRFX,mapBackIter) - else - mbEST=''; - end - - -if strcmpi(applyFullorUpdate, 'full') - % Combine old and new transformations and apply as well as erasing beads, - % e.g. go from raw stack to preCTF. - flgSkipErase = 0; - flgApplyFullXform = 1; - PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} - PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); - outputDirectory = 'aliStacks' -elseif strcmpi(applyFullorUpdate, 'fullScale') - % Combine old and new transformations and apply as well as erasing beads, - % e.g. go from raw stack to preCTF. - - % Also remove shifts due to the sample being non-eucentric. This is a - % test, and if it helps, it would be even better to just apply this shift - % to all the subtomos pre-emptivel. - flgSkipErase = 0; - flgApplyFullXform = 1; - PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} - PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); - - outputDirectory = 'aliStacks'; -elseif strcmpi(applyFullorUpdate, 'refine') - - % Don't combine, just use tlt with new defocus values and erase beads - flgSkipErase = 0; - flgApplyFullXform = 1; - PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} - PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); - - - outputDirectory = 'aliStacks'; -elseif strcmpi(applyFullorUpdate,'update') - % Combine old and new transformations, but only apply the new xform and - % don't erase beads. e.g. in inital binned mapback just update the tilt - % files and resample the ctfCorrected stack. - flgSkipErase = 1; - flgApplyFullXform = 0; - PRJ_STACK ={sprintf('ctfStacks/%s_ali%d_ctf.fixed',STACK_PRFX,mapBackIter+flgInitResample)} - PRJ_OUT = {sprintf('%s_ali%d_ctf',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d_ctf',STACK_PRFX,mapBackIter); - - outputDirectory = 'ctfStacks'; -else - - error('applyFullorUpdate should be [full], [fullScale] or [update]') -end - -try - tlt = {sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',STACK_PRFX,mapBackIter+flgInitResample)}; - testLoad = load(tlt{1}); -catch - tlt = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+flgInitResample)}; - testLoad = load(tlt{1}); - -end -tlt_OUT = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1)}; - -eraseStack = sprintf('rm cache/%s_*.fixed',STACK_PRFX); -eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); -% Converte bead diameter to pixels and add a little to be safe. -PIXEL_SIZE = pBH.('PIXEL_SIZE'); -SuperResolution = pBH.('SuperResolution'); - -% Don't apply any fourier cropping of super-res data if only updating, -% as it would already be done. -if strcmpi(applyFullorUpdate,'update') - SuperResolution = 0; -end - -if (SuperResolution) - % Transform the raw images at full sampling then crop the fft to physical - % nyquist - PIXEL_SIZE = 2.* PIXEL_SIZE; -end - - -eraseSigma = 3;%pBH.('beadSigma'); - -eraseRadius = ceil(1.2.*(pBH.('beadDiameter')./PIXEL_SIZE.*0.5)); -flgImodErase = 0 - - % FIXME, this should be stored from previous mask calc and accessed there. - % For now just take based on tomogram (which will be larger than the true specimen thickness) - %THICKNESS = recGeomForThickness.(sprintf('%s_1',STACK_PRFX)); - %THICKNESS = min(10,abs(THICKNESS(1,3)-THICKNESS(2,3)).*PIXEL_SIZE.*10^9); - THICKNESS = 100; -% Assuming all extreme pixels have already been removed from the stack. -%PRJ_STACK = {sprintf('%s_local04_18.mrc',mjIDX)};%,sprintf('%s_local14_18.mrc',mjIDX),sprintf('%s_local24_18.mrc',mjIDX),sprintf('%s_local34_18.mrc',mjIDX)}; -nStacks = length(tlt); -INPUT_CELL = cell(nStacks,7); -TLT_Trans = cell(nStacks,1); - -% killed the loop, clean up later -if exist(tlt{1}, 'file') && exist(PRJ_STACK{1}, 'file') - INPUT_CELL{1,1} = load(tlt{1}); - INPUT_CELL{1,2} = PRJ_STACK{1}; - [pathName,fileName,extension] = fileparts(PRJ_STACK{1}); - if isempty(pathName) - pathName = '.'; - end - [ctfPath,~,~] = fileparts(tlt{1}); - INPUT_CELL{1,3} = pathName; - INPUT_CELL{1,4} = fileName; - INPUT_CELL{1,5} = extension; - INPUT_CELL{1,6} = PRJ_OUT{1}; - INPUT_CELL{1,7} = ctfPath; -else - if ~exist(tlt{1}, 'file') - fprintf('\nignoring %s, because the file is not found.\n', tlt{1}); - end - if ~exist(PRJ_STACK{1}, 'file') - fprintf('\nignoring %s, because the file is not found.\n',PRJ_STACK{1}); - end - -end - - -% killed the loop, clean up later -iStack=1; - - - - iMrcObj = MRCImage(INPUT_CELL{iStack,2},0); - - % The pixel size should be previously set correctly, but if it is not, then we - % must maintain whatever is there in case beads are to be erased. The model - % used for this process depends on the pixel size in the header when it was - % created in IMod alignment. -% [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); -% iPixelHeader = EMC_str2double(iPixelHeader); - iHeader = getHeader(iMrcObj); - iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* (1+abs(SuperResolution)), ... - iHeader.cellDimensionY/iHeader.nY .* (1+abs(SuperResolution)), ... - iHeader.cellDimensionZ/iHeader.nZ]; - - iOriginHeader= [iHeader.xOrigin , ... - iHeader.yOrigin , ... - iHeader.zOrigin ] ./ (1+abs(SuperResolution)); - - d1 = iHeader.nX; d2 = iHeader.nY; d3 = size(INPUT_CELL{iStack,1},1);%iHeader.nZ; - - osX = 1-mod(d1,2); osY = 1-mod(d2,2); - - -if (SuperResolution) - gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,-0.235,'GPU','nyquistHigh'); -else - gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); -end -TLT = INPUT_CELL{iStack,1}; -pathName = INPUT_CELL{iStack,3} -fileName = INPUT_CELL{iStack,4} -extension = INPUT_CELL{iStack,5} - -% Copy with column for defocus = input to CTF correct -% saved as _ctf.tlt - - -nPrjs = size(TLT,1); - - - -% Optionally address magnification changes. -% system(sprintf('mkdir -p %s/recon',INPUT_CELL{i,3})); -system('mkdir -p aliStacks'); - - if (mapBackIter) - fprintf('Combining tranformations\n\n'); - % Load in the mapBack alignment - try - mbEST = load(sprintf('%s.tltxf',mapBackPrfx)); - catch - fprintf('WARNING: did not load %s.tltxf, cannot update alignments',mapBackPrfx) - system(sprintf('cp fixedStacks/ctf/%s_ali1_ctf.tlt fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,STACK_PRFX,mapBackIter+1)); - continue; + + if (mapBackIter) + mapBackPrfx = sprintf('mapBack%d/%s_ali%d_ctf',mapBackIter,STACK_PRFX,mapBackIter) + else + mbEST=''; end - mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); - defShifts = sprintf('%s.defShifts',mapBackPrfx); - if exist(defShifts,'file') - % tomoCPR is now using mexCTF so updated to def > 0 and in Angstrom, - % which are added to the base value. - % ctf 3d is still using orig def < 0 and in SI so convert here - defShifts = load(defShifts) .* (defShiftSign*10^-10); - fprintf('Updating defocus shifts from tomoCPR\n'); + + + if strcmpi(applyFullorUpdate, 'full') + % Combine old and new transformations and apply as well as erasing beads, + % e.g. go from raw stack to preCTF. + flgSkipErase = 0; + flgApplyFullXform = 1; + PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} + PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); + outputDirectory = 'aliStacks' + elseif strcmpi(applyFullorUpdate, 'fullScale') + % Combine old and new transformations and apply as well as erasing beads, + % e.g. go from raw stack to preCTF. + + % Also remove shifts due to the sample being non-eucentric. This is a + % test, and if it helps, it would be even better to just apply this shift + % to all the subtomos pre-emptivel. + flgSkipErase = 0; + flgApplyFullXform = 1; + PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} + PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); + + outputDirectory = 'aliStacks'; + elseif strcmpi(applyFullorUpdate, 'refine') + + % Don't combine, just use tlt with new defocus values and erase beads + flgSkipErase = 0; + flgApplyFullXform = 1; + PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} + PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); + + + outputDirectory = 'aliStacks'; + elseif strcmpi(applyFullorUpdate,'update') + % Combine old and new transformations, but only apply the new xform and + % don't erase beads. e.g. in inital binned mapback just update the tilt + % files and resample the ctfCorrected stack. + flgSkipErase = 1; + flgApplyFullXform = 0; + PRJ_STACK ={sprintf('ctfStacks/%s_ali%d_ctf.fixed',STACK_PRFX,mapBackIter+flgInitResample)} + PRJ_OUT = {sprintf('%s_ali%d_ctf',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d_ctf',STACK_PRFX,mapBackIter); + + outputDirectory = 'ctfStacks'; else - defShifts = 0; - fprintf('Did not find updated defocus estimate from tomoCPR\n'); + error('applyFullorUpdate should be [full], [fullScale] or [update]') end - end - - if ( flgShiftEucentric && mapBackIter ) - toFit = abs(mbTLT) > eucentric_minTilt; - - % For now take the mean, but it would probably be better to fit a line, - % use the Y intercept, and use the deviation from 0 of the slope as a - % measure of quality. - fprintf('\n\nYou suspect a eucentric drift\n\n') - eucShift = fit(mbTLT(toFit),1 .* mbEST(toFit,5) ./ sind(mbTLT(toFit)),'poly1'); - fprintf('\n\nFound a possible eucentric shift of %3.3f\nThe slope (%3.3f) should be close to zero.\n\n',eucShift.p2,eucShift.p1); - -% mbEST(:,5) = mbEST(:,5) - eucShift.*sind(mbTLT); -% mbEST(:,5) - eucShiftsResults{iGPU}{iTilt} = eucShift.p2; - - end - - outputStackName = sprintf('%s/%s%s',outputDirectory,INPUT_CELL{iStack,6},INPUT_CELL{iStack,5}); - oldStackName = sprintf('%s/%s%s',outputDirectory,PRJ_OLD,INPUT_CELL{iStack,5}); - - -try - erase_beads_after_ctf = pBH.('erase_beads_after_ctf'); -catch - erase_beads_after_ctf = false; -end - -if (erase_beads_after_ctf) - flgEraseBeads = 0; -else - if exist(sprintf('fixedStacks/%s.erase',fileName),'file') - flgEraseBeads = 1; - % create and later run a script to erase gold beads using imods - % ccderaser and the present fiducial model. - - else - flgEraseBeads = 0; - end -end - - - - - - - - SIZEOUT = [d1,d2]; - - - - - - tlt_tmp = cell(d3,1); - out_tmp = cell(d3,1); - - - - origOrder = TLT(:,1); - TLT = sortrows(TLT,1); - - - for i = 1:d3 - tlt_tmp{i} = TLT(i,:); - end - - - - if (flgSkipUpdate) - continue - end - - if (SuperResolution) - % Forcing output to odd size. - sizeCropped = floor([d1,d2,d3]./2)-(1-mod(floor([d1,d2,d3]./2),2)); - else - sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); - end - sizeCropped(3) = d3; - - STACK = zeros(sizeCropped,'single'); - samplingMaskStack = zeros(sizeCropped,'single'); - - for i = 1:d3 - - - - - - if (SuperResolution) - % The transform shifts need to be scaled by 2 since the stored values - % are relative to full sampling, while the tomoCPR are relative to - % physical pixel size. - updateScale = 2; - else - updateScale = 1; - end - - if (mapBackIter) - - % Stored in row order as output by imod, st transpose is needed. Inversion - % of the xform is handled in resample2d. - origXF = reshape(tlt_tmp{i}(7:10),2,2)'; - newXF = reshape(mbEST(i,1:4),2,2)'; - - - dXYZ = [(newXF*tlt_tmp{i}(2:3)')' + mbEST(i,5:6).*updateScale , 0]; - if ~isvector(dXYZ) - % In case some implicit expansion were to happen for whatever reason. - error('dXYZ is a matrix and should be a vector'); + tlt = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+flgInitResample)}; + tlt_OUT = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1)}; + + eraseStack = sprintf('rm cache/%s_*.fixed',STACK_PRFX); + eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); + + eraseSigma = 3;%emc.('beadSigma'); + + eraseRadius = ceil(1.2.*(emc.('beadDiameter')./emc.pixel_size_si.*0.5)); + flgImodErase = 0 + + % FIXME, this should be stored from previous mask calc and accessed there. + % For now just take based on tomogram (which will be larger than the true specimen thickness) + THICKNESS = 100; + % Assuming all extreme pixels have already been removed from the stack. + %PRJ_STACK = {sprintf('%s_local04_18.mrc',mjIDX)};%,sprintf('%s_local14_18.mrc',mjIDX),sprintf('%s_local24_18.mrc',mjIDX),sprintf('%s_local34_18.mrc',mjIDX)}; + nStacks = length(tlt); + INPUT_CELL = cell(nStacks,7); + TLT_Trans = cell(nStacks,1); + + % killed the loop, clean up later + if exist(tlt{1}, 'file') && exist(PRJ_STACK{1}, 'file') + INPUT_CELL{1,1} = load(tlt{1}); + INPUT_CELL{1,2} = PRJ_STACK{1}; + [pathName,fileName,extension] = fileparts(PRJ_STACK{1}); + if isempty(pathName) + pathName = '.'; + end + [ctfPath,~,~] = fileparts(tlt{1}); + INPUT_CELL{1,3} = pathName; + INPUT_CELL{1,4} = fileName; + INPUT_CELL{1,5} = extension; + INPUT_CELL{1,6} = PRJ_OUT{1}; + INPUT_CELL{1,7} = ctfPath; + else + if ~exist(tlt{1}, 'file') + error('The file %s is not found.', tlt{1}); + fprintf('\nignoring %s, because the file is not found.\n', tlt{1}); + end + if ~exist(PRJ_STACK{1}, 'file') + fprintf('\nignoring %s, because the file is not found.\n',PRJ_STACK{1}); + end + end - tlt_tmp{i}(2:3) = dXYZ(1:2); - - - combinedXF = reshape((newXF*origXF)',1,4); - tlt_tmp{i}(7:10) = combinedXF; - else - combinedXF = tlt_tmp{i}(7:10); - dXYZ = [tlt_tmp{i}(2:3),0]; - end - - % Now that we are always cropping prior to transforming, reduce the - % scale. Probable should just instruct to fourier crop prior to tilt - % alignment.flgSkipUpdate - dXYZ = dXYZ ./ updateScale; - - - - - - - - - - % Pad the projection prior to xforming in Fourier space. - if (SuperResolution) - - iProjection = single(getVolume(iMrcObj,[],[],tlt_tmp{i}(23),'keep')); - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); - - % Information beyond the physical nyquist should be removed to limit - % aliasing of noise prior tto interpolation. - iProjection = BH_padZeros3d(iProjection,[0,0],[0,0],'GPU','singleTaper',mean(iProjection(:))); - trimVal = BH_multi_padVal(1.*size(iProjection),sizeCropped(1:2)); - - largeOutliersMean= mean(iProjection(:)); - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single','gpuArray'); - - iProjection = real(ifftn(ifftshift(... - BH_padZeros3d(fftshift(... - fftn(iProjection)), ... - trimVal(1,:),trimVal(2,:),... - 'GPU','single')))); - - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),[0,0,0],[0,0],'Bah','GPU','forward',1/2,sizeCropped(1:2)); - sizeODD = size(iProjection)-[osX,osY]; - else - sizeODD = [d1,d2]-[osX,osY]; + % FIXME: removed the loop, clean up later cells etc later + iStack=1; + iMrcObj = MRCImage(INPUT_CELL{iStack,2},0); - % If it is even sized, shift up one pixel so that the origin is in the middle - % of the odd output here we can just read it in this way, unlike super res. - - iProjection = ... - single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep')); - - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); - - largeOutliersMean= mean(iProjection(:)); - - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); - - - - end - - % Because the rotation/scaling and translation are done separately, - % we must use a square transform; otherwise, a rotation angle dependent - % anisotropic distortion (like mag distortion) is introduced. - sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp.*0.25).*max(sizeODD)); + % The pixel size should be previously set correctly, but if it is not, then we + % must maintain whatever is there in case beads are to be erased. The model + % used for this process depends on the pixel size in the header when it was + % created in IMod alignment. + iHeader = getHeader(iMrcObj); + iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... + iHeader.cellDimensionY/iHeader.nY, ... + iHeader.cellDimensionZ/iHeader.nZ]; - padVal = BH_multi_padVal(sizeODD,sizeSQ); - trimVal = BH_multi_padVal(sizeSQ,sizeCropped(1:2)); - - iProjection = iProjection - mean(iProjection(:)); - iProjection = iProjection ./ std(iProjection(:)); - - - if ( SuperResolution ) - iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... - padVal(1,:),padVal(2,:),'GPU','singleTaper'); - else - iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - 'GPU','singleTaper'); - end - - if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); - end + iOriginHeader= [iHeader.xOrigin , ... + iHeader.yOrigin , ... + iHeader.zOrigin ]; + d1 = iHeader.nX; d2 = iHeader.nY; d3 = size(INPUT_CELL{iStack,1},1);%iHeader.nZ; + + osX = 1-mod(d1,2); osY = 1-mod(d2,2); - if (flgApplyFullXform) - % Do the phase shift after rotating - need to invert the scaling since - % we are in reciprocal space - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(combinedXF); - % Assuming stretch and skew are not fit, leave defined for possible - % later consideration. - - - if (bh_global_do_2d_fourier_interp) - if (i == 1) - fprintf('resampling at 2x padding with fourier interp\n'); - end -% combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - combinedInverted = combinedInverted([1,2,4,5]); - - - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); - else - if (i == 1) - fprintf('resampling at 1x padding with linear interp\n'); - end - % Real space, do not invert mag - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); - end - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - - else - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(mbEST(i,1:4)); - % Assuming stretch and skew are not fit, leave defined for possible - % later consideration. + gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); + + TLT = INPUT_CELL{iStack,1}; + pathName = INPUT_CELL{iStack,3}; + fileName = INPUT_CELL{iStack,4}; + extension = INPUT_CELL{iStack,5}; + + + % Optionally address magnification changes. + % system(sprintf('mkdir -p %s/recon',INPUT_CELL{i,3})); + system('mkdir -p aliStacks'); + + if (mapBackIter) + fprintf('Combining tranformations\n\n'); + % Load in the mapBack alignment + skip = false; + try + mbEST = load(sprintf('%s.tltxf',mapBackPrfx)); + catch + error('WARNING: did not load %s.tltxf, cannot update alignments',mapBackPrfx) + system(sprintf('cp fixedStacks/ctf/%s_ali1_ctf.tlt fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,STACK_PRFX,mapBackIter+1)); + continue; + end - % NOTE mag is ignored when the rotation matrix has 4 elements (IMOD) - - if (bh_global_do_2d_fourier_interp) - if (i == 1) - fprintf('resampling at 2x padding with fourier interp\n'); - end -% mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - mbEstInverted = mbEstInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); + mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); + defShifts = sprintf('%s.defShifts',mapBackPrfx); + if exist(defShifts,'file') + % tomoCPR is now using mexCTF so updated to def > 0 and in Angstrom, + % which are added to the base value. + % ctf 3d is still using orig def < 0 and in SI so convert here + defShifts = load(defShifts) .* (defShiftSign*10^-10); + fprintf('Updating defocus shifts from tomoCPR\n'); else - if (i == 1) - fprintf('resampling at 1x padding with linear interp\n'); - end - mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - mbEstInverted = mbEstInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); + defShifts = 0; + fprintf('Did not find updated defocus estimate from tomoCPR\n'); end - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),mbEST(i,1:4),dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - end - + end + + + if ( emc.eucentric_fit && mapBackIter ) + toFit = abs(mbTLT) > emc.eucentric_maxTilt; + error('eucentric fit not implemented') + % For now take the mean, but it would probably be better to fit a line, + % use the Y intercept, and use the deviation from 0 of the slope as a + % measure of quality. + fprintf('\n\nYou suspect a eucentric drift\n\n') + eucShift = fit(mbTLT(toFit),1 .* mbEST(toFit,5) ./ sind(mbTLT(toFit)),'poly1'); + + fprintf('\n\nFound a possible eucentric shift of %3.3f\nThe slope (%3.3f) should be close to zero.\n\n',eucShift.p2,eucShift.p1); + eucShiftsResults{iGPU}{iTilt} = eucShift.p2; + end + + outputStackName = sprintf('%s/%s%s',outputDirectory,INPUT_CELL{iStack,6},INPUT_CELL{iStack,5}); + oldStackName = sprintf('%s/%s%s',outputDirectory,PRJ_OLD,INPUT_CELL{iStack,5}); + + % erase_beads_after_ctf is now handled in BH_parseParameterFile + erase_beads_after_ctf = emc.erase_beads_after_ctf; + + if (erase_beads_after_ctf) + flgEraseBeads = 0; + else + if exist(sprintf('fixedStacks/%s.erase',fileName),'file') + flgEraseBeads = 1; + % create and later run a script to erase gold beads using imods + % ccderaser and the present fiducial model. + + else + flgEraseBeads = 0; + end + end + + tlt_tmp = cell(d3,1); + out_tmp = cell(d3,1); + + origOrder = TLT(:,1); + TLT = sortrows(TLT,1); + + for i = 1:d3 + tlt_tmp{i} = TLT(i,:); + end + + if (flgSkipUpdate) + continue; + end + + sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); + sizeCropped(3) = d3; + + + if (mapBackIter) + tmp_xf = tempname; + tmp_xf_fd = fopen(tmp_xf,"w"); + for i = 1:d3 + fprintf(tmp_xf_fd,"%f %f %f %f %f %f\n",tlt_tmp{i}([7,8,9,10,2,3])); + end + fclose(tmp_xf_fd); + tmp_combined_xf = tempname; + cmd_base = sprintf('xfproduct %s %s.tltxf %s', tmp_xf, mapBackPrfx, tmp_combined_xf); + [ xfprod_err ] = system(sprintf('%s > /dev/null',cmd_base)); + if (xfprod_err) + system(cmd_base); + error('xfprod failed'); + else + mbEST = load(tmp_combined_xf, '-ascii'); + for i = 1:d3 + tlt_tmp{i}([7,8,9,10,2,3]) = mbEST(i,:); + end + end + else + error('ctf update should only be called after tomoCPR (mapBackIter > 0), you can change this with emClarity geometry paramX.m X TiltAlignment SwitchCurrentTomoCpr [mapBackIter,0,0] STD'); + end - STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... - trimVal(1,:),trimVal(2,:),... - 'GPU','single'))); - - iSamplingMask(isnan(iSamplingMask(:))) = 0; - samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); - iSamplingMask = []; -% -% end - - end + base_cmd = sprintf('newstack -mode 12 -meansd 0,1 -xf %s %s %s',tmp_combined_xf,INPUT_CELL{iStack,2},outputStackName); + [ newstack_err ] = system(sprintf('%s > /dev/null',base_cmd)); + if (newstack_err) + system(base_cmd); + error('newstack failed'); + end + samplingMaskStack = ones(sizeCropped,'single'); + SAVE_IMG(samplingMaskStack,{sprintf('%s.samplingMask_pre',outputStackName), 'half'}, iPixelHeader,iOriginHeader); + + base_cmd = sprintf('newstack -mode 12 -fill 0 -xf %s %s.samplingMask_pre %s.samplingMask',tmp_combined_xf,outputStackName,outputStackName); + [ newstack_err ] = system(sprintf('%s > /dev/null',base_cmd)); + if (newstack_err) + system(base_cmd); + error('newstack failed'); + end - + system(sprintf('rm %s.samplingMask_pre',outputStackName)); + system(sprintf('rm %s %s',tmp_xf,tmp_combined_xf)); - - for i= 1:d3 - TLT(i,:) = tlt_tmp{i}; -% STACK(:,:,TLT(i,1)) = out_tmp{i}; - end + for i = 1:d3 + + updateScale = 1; + + if (mapBackIter) + + % % Stored in row order as output by imod, st transpose is needed. Inversion + % % of the xform is handled in resample2d. + % origXF = reshape(tlt_tmp{i}(7:10),2,2)'; + % newXF = reshape(mbEST(i,1:4),2,2)'; + + % dXYZ = [(newXF*tlt_tmp{i}(2:3)')' + mbEST(i,5:6).*updateScale , 0]; + % if ~isvector(dXYZ) + % % In case some implicit expansion were to happen for whatever reason. + % error('dXYZ is a matrix and should be a vector'); + % end + % This is now updated above + dXYZ(1:2) = tlt_tmp{i}(2:3); + combinedXF = tlt_tmp{i}(7:10); + + + % combinedXF = reshape((newXF*origXF)',1,4); + % tlt_tmp{i}(7:10) = combinedXF; + else + combinedXF = tlt_tmp{i}(7:10); + dXYZ = [tlt_tmp{i}(2:3),0]; + end + + % Now that we are always cropping prior to transforming, reduce the + % scale. Probable should just instruct to fourier crop prior to tilt + % alignment.flgSkipUpdate + dXYZ = dXYZ ./ updateScale; + - out_tmp = []; - - if (mapBackIter) - % Update the tilt angles - TLT(:,4) = mbTLT; - if (defShifts) - TLT(:,15) = TLT(:,15) + defShifts; - TLT(:,16) = PIXEL_SIZE; + TLT(i,:) = tlt_tmp{i}; + % STACK(:,:,TLT(i,1)) = out_tmp{i}; end - - + + + out_tmp = []; + + if (mapBackIter) + % Update the tilt angles + TLT(:,4) = mbTLT; + if (defShifts) + TLT(:,15) = TLT(:,15) + defShifts; + TLT(:,16) = emc.pixel_size_si; + end + + % Sort descending along the magnitude of the tilt angles because higher tilts take % longer on CTF correction. If more processor available than projections, % this doesn't affect anything. @@ -614,69 +408,69 @@ sprintf('%s/%s.tlt',INPUT_CELL{iStack,7},INPUT_CELL{iStack,6}) fileID = fopen(tlt_OUT{iStack}, 'w'); fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%5e\t%5e\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%3.2f\n'], TLT'); - - - if ( flgEraseBeads ) - STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); - end - - + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%3.2f\n'], TLT'); + + STACK = gpuArray(OPEN_IMG('single',outputStackName)); + if ( flgEraseBeads ) + STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); + end + + + + fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',THICKNESS, STACK_PRFX); + samplingMaskStack = gpuArray(OPEN_IMG('single',sprintf('%s.samplingMask',outputStackName))); + [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,emc.pixel_size_angstroms,gpuArray(samplingMaskStack)); + SAVE_IMG(STACK,{outputStackName,'half'},iPixelHeader,iOriginHeader); + + xShift= []; yShift = []; scale = []; angleShift = []; + dZ = []; recZ= []; rotMat = []; angX = []; angY = []; + else + + error('This branch is deprecated and should not be reached.'); + if ( flgEraseBeads ) + STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); + end - fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',... - THICKNESS, STACK_PRFX); - - [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,PIXEL_SIZE*10^10,gpuArray(samplingMaskStack)); - SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); - SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName)); - xShift= []; yShift = []; scale = []; angleShift = []; - dZ = []; recZ= []; rotMat = []; angX = []; angY = []; - else - - if ( flgEraseBeads ) - STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); - end - - fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',... - THICKNESS, STACK_PRFX); - - [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,PIXEL_SIZE*10^10,gpuArray(samplingMaskStack)); + THICKNESS, STACK_PRFX); + + [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,emc.pixel_size_angstroms,gpuArray(samplingMaskStack)); SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName),iPixelHeader,iOriginHeader); - - end - if (mapBackIter && conserveDiskSpace) - system(sprintf('rm %s',oldStackName)); - end - - - % Once updated the reconstructions are no longer valid - system(eraseStack); - system(eraseRec); + + end + if (mapBackIter && conserveDiskSpace) + system(sprintf('rm %s',oldStackName)); + end + + + % Once updated the reconstructions are no longer valid + system(eraseStack); + system(eraseRec); end % end of loop over tilts end % end of par for loop -% -if (flgShiftEucentric && mapBackIter) +% +if (emc.eucentric_fit && mapBackIter) + error('eucentric fit not implemented') % Update the sub tomo z coords with an estimate of the shift cycle_to_update = subTomoMeta.('tomoCPR_run_in_cycle')(find(subTomoMeta.('tomoCPR_run_in_cycle')(:,1) == subTomoMeta.currentTomoCPR),2); for iGPU = 1:nGPUs - - for iTilt = 1:length(ITER_LIST{iGPU}) - + + for iTilt = 1:length(ITER_LIST{iGPU}) + STACK_PRFX = ITER_LIST{iGPU}{iTilt}; eucShift = eucShiftsResults{iGPU}{iTilt}; - + % Workaround for partial numbers - need something better. FIXME n_tomos_found = 0; - for jTomo = 1:size(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords,1) %subTomoMeta.mapBackGeometry.(STACK_PRFX).nTomos + for jTomo = 1:size(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords,1) if any(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords(jTomo,:)) - n_tomos_found = n_tomos_found + 1; + n_tomos_found = n_tomos_found + 1; % We might have skipped the update if tomoCPR failed. if isempty(eucShift) fprintf('No updated shifts for %s_%d\n',STACK_PRFX,jTomo); @@ -685,28 +479,29 @@ fprintf('shifting %s_%d\n',STACK_PRFX,jTomo); subTomoMeta.(sprintf('cycle%0.3d',cycle_to_update)).('eucentric_shifts').(sprintf('%s_%d',STACK_PRFX,jTomo)) = [eucShift]; % The tomogram is shifted by the calculated amount -% subTomoMeta.(sprintf('cycle%0.3d',subTomoMeta.currentCycle)).RawAlign.(sprintf('%s_%d',STACK_PRFX,jTomo))(:,13) = ... -% eucShift + subTomoMeta.(sprintf('cycle%0.3d',subTomoMeta.currentCycle)).RawAlign.(sprintf('%s_%d',STACK_PRFX,jTomo))(:,13); + % subTomoMeta.(sprintf('cycle%0.3d',subTomoMeta.currentCycle)).RawAlign.(sprintf('%s_%d',STACK_PRFX,jTomo))(:,13) = ... + % eucShift + subTomoMeta.(sprintf('cycle%0.3d',subTomoMeta.currentCycle)).RawAlign.(sprintf('%s_%d',STACK_PRFX,jTomo))(:,13); end - + end end if (n_tomos_found ~= subTomoMeta.mapBackGeometry.(STACK_PRFX).nTomos) - error('The number of tomos found (%d) does not match (%d) for tomo %s\n', ... - n_tomos_found, subTomoMeta.mapBackGeometry.(STACK_PRFX).nTomos, STACK_PRFX); - end + error('The number of tomos found (%d) does not match (%d) for tomo %s\n', ... + n_tomos_found, subTomoMeta.mapBackGeometry.(STACK_PRFX).nTomos, STACK_PRFX); + end end end - save(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + % Save using wrapper + BH_saveSubTomoMeta(emc.('subTomoMeta'), subTomoMeta); end - + if ( flgParallel ) fprintf('\nAuto updating the tilt geometry\n'); - % BH_geometryAnalysis(updateCMD) - BH_geometryAnalysis(sprintf('%s',PARAMETER_FILE),sprintf('%d',subTomoMeta.currentCycle),'TiltAlignment','UpdateTilts',sprintf('[%d,0,0]',subTomoMeta.currentCycle),'STD'); + % BH_geometryAnalysis(updateCMD) + BH_geometryAnalysis(sprintf('%s',PARAMETER_FILE),sprintf('%d',subTomoMeta.currentCycle),'TiltAlignment','UpdateTilts',sprintf('[%d,0,0]',subTomoMeta.currentCycle),'STD'); else fprintf('\n\nSince you are updating each tilt series manually, you must'); fprintf(' run\nemClarity geometry [param] [cycle] TiltAlignment UpdateTilts [cycle,0,0] STD\n\n'); end - + diff --git a/ctf/BH_runCtfFind.m b/ctf/BH_runCtfFind.m index 1d44e8be..52a6f85a 100644 --- a/ctf/BH_runCtfFind.m +++ b/ctf/BH_runCtfFind.m @@ -1,88 +1,137 @@ -function [ ] = BH_runCtfFind(stackName, tltName, ctfParams, tiltAngles) +function [ ] = BH_runCtfFind(stackNameBaseName, tltNameBaseName, ctfParams, tiltAngles) %Fit the ctf to a background subtracted PS using ctffind4 % CTF params -% PixelSize (Ang) -% KeV +% PixelSize (Ang) +% KeV % CS (mm) % Amplitude Contrast system('mkdir -p fixedStacks/ctf/forCtfFind'); rng('shuffle'); -randPrfx = sprintf('%s_%d',tltName,randi(1e6,[1,1])); +randPrfx = sprintf('%s_%d',tltNameBaseName,randi(1e6,[1,1])); +randPrfx_inv = sprintf('%s_inv',randPrfx); ctfFindPath = getenv('EMC_CTFFIND'); fprintf('%s\n',ctfFindPath);% split the stack up -fullStack = getVolume(MRCImage(stackName)); +fullStack = OPEN_IMG('single', sprintf('%s.mrc',stackNameBaseName)); +fullStack_inv = OPEN_IMG('single', sprintf('%s_inv.mrc',stackNameBaseName)); [d1,d2,d3] = size(fullStack); % FIXME d1 assumed to equal d2 Add check in saving for iPrj = 1:d3 SAVE_IMG(MRCImage(fullStack(:,:,iPrj)),sprintf('fixedStacks/ctf/forCtfFind/%s_%d.mrc',randPrfx,iPrj)); + SAVE_IMG(MRCImage(fullStack_inv(:,:,iPrj)),sprintf('fixedStacks/ctf/forCtfFind/%s_%d.mrc',randPrfx_inv,iPrj)); end % % Check to make sure this hasn't alread been done -% if ~exist(sprintf('fixedStacks/ctf/%s_orig',tltName), 'file') - system(sprintf('mv fixedStacks/ctf/%s fixedStacks/ctf/%s_orig',tltName,tltName)); +% if ~exist(sprintf('fixedStacks/ctf/%s_orig',tltNameBaseName), 'file') +system(sprintf('mv fixedStacks/ctf/%s.tlt fixedStacks/ctf/%s.tlt_orig',tltNameBaseName,tltNameBaseName)); % end - tmpTLT = load(sprintf('fixedStacks/ctf/%s_orig',tltName)); - meanDefocus = mean(tmpTLT(:,15))*-1.0*10^10; - fprintf('Searching around an estimated mean defocus of %3.6f Angstrom\n'); +tmpTLT = load(sprintf('fixedStacks/ctf/%s.tlt_orig',tltNameBaseName)); +meanDefocus = mean(abs(tmpTLT(:,15)))*10^10; +fprintf('Searching around an estimated mean defocus of %3.6f Angstrom\n'); % write the run script, this should link to a distributed version with % special name, but for testing use the beta. -scriptName = sprintf('.%s.sh',randPrfx); -fID = fopen(scriptName,'w'); - -fprintf(fID,'#!/bin/bash\n\n'); -for iPrj = 1:d3 % I want to fit to lower resolution at higher tilts - tltIDX = find(tiltAngles(:,1) == iPrj); - - % put in a line to limit number of cores, or use the threaded version - fprintf(fID,'\n%s --amplitude-spectrum-input << eof &',ctfFindPath); - fprintf(fID,'\nfixedStacks/ctf/forCtfFind/%s_%d.mrc\n',randPrfx,iPrj); - fprintf(fID,'fixedStacks/ctf/forCtfFind/%s_diagnostic_%d.mrc\n',randPrfx,iPrj); - fprintf(fID,'%f\n%f\n%f\n%f\n%d\n%f\n%f\n%d\n%d\n%d\n',ctfParams(1:4), ... - d1,30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)),... - 0.75*meanDefocus,... - 1.25*meanDefocus,... - 25.0); - fprintf(fID,'no\nno\nyes\n500.0\nno\nno\nno\neof\n\n'); -end -fprintf(fID,'wait\n'); -fclose(fID); +score = 0; +score_inv = 0; +for i_run = [1:2] + if (i_run == 1) + % regular + using_prfx = randPrfx; + else + using_prfx = randPrfx_inv; + % inverse + end -system(sprintf('chmod a=wrx %s',scriptName)); + scriptName = sprintf('.%s.sh',using_prfx); + + + fID = fopen(scriptName,'w'); + + fprintf(fID,'#!/bin/bash\n\n'); + for iPrj = 1:d3 + % I want to fit to lower resolution at higher tilts + tltIDX = find(tiltAngles(:,1) == iPrj); + + % put in a line to limit number of cores, or use the threaded version + fprintf(fID,'\n%s --amplitude-spectrum-input << eof &',ctfFindPath); + fprintf(fID,'\nfixedStacks/ctf/forCtfFind/%s_%d.mrc\n',using_prfx,iPrj); + fprintf(fID,'fixedStacks/ctf/forCtfFind/%s_diagnostic_%d.mrc\n',using_prfx,iPrj); + fprintf(fID,'%f\n%f\n%f\n%f\n%d\n%f\n%f\n%d\n%d\n%d\n', ... + ctfParams(1:4), ... + d1, ... + 30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)).^0.4,... + 0.9*meanDefocus,... + 1.1*meanDefocus,... + 25.0); + fprintf(fID,'no\nno\nyes\n500.0\nno\nno\nno\neof\n\n'); + end + fprintf(fID,'wait\n'); + fclose(fID); -[runFail] = system(sprintf('./%s',scriptName)); + system(sprintf('chmod a=wrx %s',scriptName)); -if (runFail) - system(sprintf('cp ./%s tmpFail',scriptName)); - system(sprintf('mv tmpFail ./%s',scriptName)); [runFail] = system(sprintf('./%s',scriptName)); + if (runFail) - error('Tried to run %s twice and failed\n',scriptName); + system(sprintf('cp ./%s tmpFail',scriptName)); + system(sprintf('mv tmpFail ./%s',scriptName)); + [runFail] = system(sprintf('./%s',scriptName)); + if (runFail) + error('Tried to run %s twice and failed\n',scriptName); + end end -end -% will this wait for return? -baseName = sprintf('fixedStacks/ctf/forCtfFind/%s_diagnostic_',randPrfx); -tmpName = sprintf('fixedStacks/ctf/forCtfFind/%s_tmp',randPrfx); -system(sprintf('newstack %s?.mrc %s??.mrc %s_full.st',baseName,baseName,baseName)); -system(sprintf('rm %s?.mrc %s??.mrc',baseName,baseName)); -system(sprintf('rm -f %s',tmpName)); + % will this wait for return? -for iPrj = 1:d3 - - system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10,3.141592/180*$4,-1*(($2+$3)/2)*10^-10 }'' >> %s', baseName,iPrj,tmpName)); - -end + baseName = sprintf('fixedStacks/ctf/forCtfFind/%s_diagnostic_',using_prfx); + tmpName = sprintf('fixedStacks/ctf/forCtfFind/%s_tmp',using_prfx); + if (i_run == 1) + using_tltName = sprintf('%s.tlt',tltNameBaseName); + else + using_tltName = sprintf('%s_inv.tlt',tltNameBaseName); + end -% TODO ground truth to confirm orientation of astigmatism + system(sprintf('newstack %s?.mrc %s??.mrc %sfull.st',baseName,baseName,baseName)); + system(sprintf('rm %s?.mrc %s??.mrc',baseName,baseName)); + % system(sprintf('rm -f %s?.mrc %s??.mrc',using_prfx,using_prfx)); + system(sprintf('rm -f %s',tmpName)); + for iPrj = 1:d3 + % 2024 Jan, finally make switch to record positive for underfocus as is used internally. + system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10, 3.1415926535/180.0*$4, 1*(($2+$3)/2)*10^-10, $6 }'' >> %s', baseName,iPrj,tmpName)); + + end + % TODO ground truth to confirm orientation of astigmatism + + system(sprintf('awk ''FNR==NR{a[FNR]=$1;b[FNR]=$2;c[FNR]=$3 ;next}{ print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,a[$1],b[$1],$14,c[$1],$16,$17,$18,$19,$20,$21,$22,$23}'' %s fixedStacks/ctf/%s.tlt_orig > fixedStacks/ctf/%s',tmpName,tltNameBaseName,using_tltName)); + + a = importdata(tmpName); + if (i_run == 1) + % regular + score = mean(a(:,4)); + else + % inverse + score_inv = mean(a(:,4)); + end +end % loop on reg/inv + +% Save the scores in +fprintf('Found an average score: %3.6f and an average inverted hand score: %3.6f for tilt %s\n',score,score_inv, tltNameBaseName); +if (score_inv > score) + fprintf('It looks like your handedness is inverted based on tiles.\n'); +end + +% Clean up the input slices (the stacks are still at fixedStacks/ctf/...PS-2.mrc) +for iPrj = 1:d3 + system(sprintf('rm -f fixedStacks/ctf/forCtfFind/%s_%d.mrc',randPrfx,iPrj)); + system(sprintf('rm -f fixedStacks/ctf/forCtfFind/%s_%d.mrc',randPrfx_inv,iPrj)); +end -system(sprintf('awk ''FNR==NR{a[FNR]=$1;b[FNR]=$2;c[FNR]=$3 ;next}{ print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,a[$1],b[$1],$14,c[$1],$16,$17,$18,$19,$20,$21,$22,$23}'' %s fixedStacks/ctf/%s_orig > fixedStacks/ctf/%s',tmpName,tltName,tltName)); +end % function \ No newline at end of file diff --git a/ctf/EMC_ctf_refine_from_star.m b/ctf/EMC_ctf_refine_from_star.m new file mode 100644 index 00000000..f9735801 --- /dev/null +++ b/ctf/EMC_ctf_refine_from_star.m @@ -0,0 +1,896 @@ +function [] = EMC_ctf_refine_from_star(star_file_path, stack_file_path, ... + reference_volume_path, output_star_path, varargin) +% EMC_ctf_refine_from_star - Standalone CTF refinement from star file + external reference. +% +% Refines per-tilt defocus, astigmatism, and per-particle Z offsets using +% ADAM-based optimization. Reference projections are generated by rotating +% a 3D reference volume and projecting along Z. +% +% INPUTS: +% star_file_path - Input star file (30-column, from BH_to_cisTEM_mapBack) +% stack_file_path - Particle stack MRC file +% reference_volume_path - 3D reference volume MRC file +% output_star_path - Output refined star file path +% varargin - Name-value option pairs: +% 'defocus_search_range' - Defocus search range in Angstroms (default: 5000) +% 'maximum_iterations' - Max ADAM iterations (default: 15) +% 'lowpass_cutoff' - Lowpass cutoff in Angstroms (default: 10) +% 'astigmatism_angle_range' - Max astigmatism angle change in radians (default: pi/4) +% 'z_offset_bound_factor' - Z offset bound multiplier (default: 5) +% 'shift_sigma' - Gaussian penalty sigma for X/Y shifts in Angstroms (default: 5) +% 'highpass_cutoff' - Highpass cutoff in Angstroms (default: 400) +% 'use_phase_compensated_correlation' - Phase-doubled correlation (default: false) +% 'debug_tilt_list' - Comma-separated tilt names to process (default: '' = all) +% 'exit_after_n_tilts' - Process only first N tilt groups, 0=all (default: 0) +% 'verbose_timing' - Print per-section timing (default: false) +% 'debug_print' - Print all option values after parsing (default: false) +% 'gpu_ids' - GPU device IDs (1-indexed), comma-separated string +% e.g. '1,2,5' for 3 GPUs. Default '1' (single GPU) +% 'workers_per_gpu' - Parallel workers per GPU (default: 3) +% +% BEHAVIORS TO WATCH: +% - ADAM score_history should increase monotonically after warmup; oscillation = lr too high +% - delta_z should cluster near 0; bimodal = sign error +% - Defocus corrections > 2000 A = failed refinement or wrong conventions +% - GPU memory with >500 particles per tilt group + +%% ===== Stage A: Parse inputs and star file ===== + +% Parse name-value options +opts = parse_options(varargin); + +fprintf('=== EMC_ctf_refine_from_star ===\n'); +fprintf(' Star file: %s\n', star_file_path); +fprintf(' Stack file: %s\n', stack_file_path); +fprintf(' Reference vol: %s\n', reference_volume_path); +fprintf(' Output: %s\n', output_star_path); + +if opts.debug_print + print_struct_fields('Options', opts); +end + +[particles, header_lines] = parse_star_file(star_file_path); +n_total_particles = length(particles); +fprintf(' Parsed %d particles from star file\n', n_total_particles); + +%% ===== Stage B: Read stack header (CPU only — file handles created per worker) ===== + +stack_mrc_tmp = MRCImage(stack_file_path, 0); +stack_header = getHeader(stack_mrc_tmp); +tile_size = [stack_header.nX, stack_header.nY]; +fprintf(' Stack tile size: [%d, %d], %d slices\n', tile_size(1), tile_size(2), stack_header.nZ); +clear stack_mrc_tmp; + +% Check reference volume size (CPU read) +ref_vol_tmp = single(OPEN_IMG('single', reference_volume_path)); +ref_vol_size = size(ref_vol_tmp); +fprintf(' Reference volume size: [%d, %d, %d]\n', ref_vol_size(1), ref_vol_size(2), ref_vol_size(3)); +clear ref_vol_tmp; + +%% ===== Stage C: Compute CTFSIZE and padding (CPU-only, small arrays) ===== + +CTFSIZE = BH_multi_iterator([2.*tile_size, 1], 'fourier'); +CTFSIZE = CTFSIZE(1:2); +padCTF = BH_multi_padVal(tile_size, CTFSIZE); +ctfOrigin = emc_get_origin_index(CTFSIZE); +fprintf(' tile_size=[%d,%d] -> CTFSIZE=[%d,%d] (FFT-friendly, ~2x)\n', ... + tile_size(1), tile_size(2), CTFSIZE(1), CTFSIZE(2)); + +%% ===== Stage D: Group particles by tilt, extract per-particle arrays ===== + +[unique_tilt_names, tilt_group_indices, tilt_angles_per_group] = group_particles_by_tilt(particles); +n_tilt_groups = length(unique_tilt_names); +fprintf(' %d tilt groups\n', n_tilt_groups); + +% Get microscope parameters from first particle (constant across stack) +pixel_size_angstroms = particles(1).pixel_size; +voltage_kv = particles(1).voltage_kv; +voltage_volts = voltage_kv * 1e3; +wavelength_angstroms = 12.2643 / sqrt(voltage_volts * (1 + voltage_volts * 0.978466e-6)); +cs_mm = particles(1).cs_mm; +amplitude_contrast = particles(1).amplitude_contrast; + +% Extract per-particle data into plain arrays (parfor can't index into struct arrays) +all_positions = [particles.position_in_stack]'; +all_psi = [particles.psi]'; +all_theta = [particles.theta]'; +all_phi = [particles.phi]'; +all_x_shift = [particles.x_shift]'; +all_y_shift = [particles.y_shift]'; +all_df1 = [particles.defocus_1]'; +all_df2 = [particles.defocus_2]'; +all_df_angle = [particles.defocus_angle]'; + +% Pre-compute per-tilt-group member lists and particle counts +tilt_group_members = cell(n_tilt_groups, 1); +particles_per_group = zeros(n_tilt_groups, 1); +for g = 1:n_tilt_groups + tilt_group_members{g} = find(tilt_group_indices == g); + particles_per_group(g) = length(tilt_group_members{g}); +end + +%% ===== Stage E: Setup parallel workers ===== + +% Parse GPU IDs (may be string, numeric, or logical if parser converted '1' to true) +if ischar(opts.gpu_ids) || isstring(opts.gpu_ids) + gpu_ids = EMC_str2double(opts.gpu_ids); +else + gpu_ids = double(opts.gpu_ids); % double() handles logical (true→1) and numeric +end +if isempty(gpu_ids) || (isscalar(gpu_ids) && gpu_ids == 0) + gpu_ids = [1]; +end +n_gpus = length(gpu_ids); +n_workers = n_gpus * opts.workers_per_gpu; + +% Filter to specific tilt groups by name for debugging +if ~isempty(opts.debug_tilt_list) + debug_names = strtrim(strsplit(opts.debug_tilt_list, ',')); + keep_indices = []; + for di = 1:length(debug_names) + pattern = debug_names{di}; + if pattern(end) == '*' + % Prefix match: 'H68_1_label_81_*' matches all tilts in that series + prefix = pattern(1:end-1); + match = find(strncmp(unique_tilt_names, prefix, length(prefix))); + else + match = find(strcmp(unique_tilt_names, pattern)); + end + if ~isempty(match) + keep_indices = [keep_indices, match]; %#ok + else + fprintf(' [WARNING] debug_tilt_list: pattern "%s" matched no tilts, skipping\n', pattern); + end + end + keep_indices = unique(keep_indices); % deduplicate in case patterns overlap + unique_tilt_names = unique_tilt_names(keep_indices); + tilt_angles_per_group = tilt_angles_per_group(keep_indices); + tilt_group_members = tilt_group_members(keep_indices); + particles_per_group = particles_per_group(keep_indices); + n_tilt_groups = length(keep_indices); + fprintf(' [DEBUG] debug_tilt_list: processing %d tilt groups: %s\n', ... + n_tilt_groups, strjoin(unique_tilt_names, ', ')); +end + +% Limit tilt groups for testing (0 = process all) +if opts.exit_after_n_tilts > 0 + n_tilt_groups = min(n_tilt_groups, opts.exit_after_n_tilts); + fprintf(' [DEBUG] Limiting to %d tilt groups (exit_after_n_tilts=%d)\n', ... + n_tilt_groups, opts.exit_after_n_tilts); +end + +fprintf(' GPUs: [%s], %d workers (%d per GPU)\n', ... + num2str(gpu_ids), n_workers, opts.workers_per_gpu); + +% Load-balanced assignment: sort tilt groups by descending particle count, +% then round-robin assign to workers so each gets similar total work +[~, sort_order] = sort(particles_per_group, 'descend'); +worker_assignments = cell(n_workers, 1); +for i = 1:n_tilt_groups + worker_idx = mod(i - 1, n_workers) + 1; + worker_assignments{worker_idx} = [worker_assignments{worker_idx}, sort_order(i)]; +end + +% Report load balance +for w = 1:n_workers + n_tilts_w = length(worker_assignments{w}); + n_particles_w = sum(particles_per_group(worker_assignments{w})); + fprintf(' Worker %d: %d tilts, %d particles, GPU %d\n', ... + w, n_tilts_w, n_particles_w, gpu_ids(mod(w - 1, n_gpus) + 1)); +end + +% Build refinement options struct (CPU scalars, safe to broadcast) +refinement_options = struct(); +refinement_options.defocus_search_range = opts.defocus_search_range; +refinement_options.maximum_iterations = opts.maximum_iterations; +refinement_options.CTFSIZE = CTFSIZE; +refinement_options.use_phase_compensated_correlation = opts.use_phase_compensated_correlation; +refinement_options.lowpass_cutoff = opts.lowpass_cutoff; +refinement_options.highpass_cutoff = opts.highpass_cutoff; +refinement_options.astigmatism_angle_range = opts.astigmatism_angle_range; +refinement_options.z_offset_bound_factor = opts.z_offset_bound_factor; +refinement_options.peak_search_radius = floor(CTFSIZE ./ 4); +refinement_options.maximum_xy_shift = max(floor(CTFSIZE ./ 4)); +refinement_options.minimum_global_iterations = opts.minimum_global_iterations; +refinement_options.global_only = opts.global_only; +refinement_options.shift_sigma = opts.shift_sigma; +refinement_options.verbose_timing = opts.verbose_timing; + +if opts.debug_print + print_struct_fields('Refinement Options (passed to EMC_refine_tilt_ctf)', refinement_options); +end + +EMC_parpool(n_workers); + +%% ===== Stage F: Parallel refinement ===== + +% TODO: Tilt-dependent occupancy scoring (removed to enable parallelization) +% +% The intent: CC scores drop as a function of tilt angle due to increased +% specimen thickness / foreshortening. We expect this to follow approximately +% cos^alpha(tilt_angle). If a whole tilt has anomalously low scores relative +% to this model, it likely means that tilt image is garbage and all its +% particles should get occupancy=0. +% +% What was here: +% - Process tilts in ascending |tilt_angle| order +% - Use low-tilt (~0 deg) median score as baseline +% - Accumulate (angle, median_score) pairs across tilts +% - Fit alpha from log(score_ratio) = alpha * log(cos(angle)) +% - Set score_threshold = expected_score * 0.3 for each tilt +% - Reject particles below threshold (occupancy=0) +% +% To re-enable: run refinement in parallel (as below), then apply the scoring +% model as a sequential post-processing step over the collected results. +% Sort results by |tilt_angle|, compute baseline from low-tilt groups, +% fit cos^alpha model, apply threshold to high-tilt groups. + +worker_results = cell(n_workers, 1); + +% Progress tracking via DataQueue — workers send updates, client accumulates +if opts.enable_progress + progress_queue = parallel.pool.DataQueue; + update_progress(struct('total', n_tilt_groups)); + afterEach(progress_queue, @(msg) update_progress(msg)); +else + progress_queue = []; +end + +parfor iWorker = 1:n_workers % revert parfor + fprintf(' [W%d] CUDA_VISIBLE_DEVICES=%s\n', iWorker, getenv('CUDA_VISIBLE_DEVICES')); + % GPU assignment: round-robin across parent-specified gpu_ids + % (matches BH_average3d / BH_alignRaw3d pattern — don't query gpuDeviceCount inside workers) + local_gpu_id = gpu_ids(mod(iWorker - 1, n_gpus) + 1); + gpuDevice(local_gpu_id); + + verbose_timing = opts.verbose_timing; + t_worker_start = tic; + % Each worker creates ALL GPU objects fresh — no shared GPU state + local_soft_mask = create_2d_soft_mask(tile_size, 7); + local_ctf_mask = gpuArray(BH_mask3d('sphere', CTFSIZE, ctfOrigin - 7, [0,0], '2d')); + local_ref_vol = gpuArray(single(OPEN_IMG('single', reference_volume_path))); + local_ref_interp = interpolator(local_ref_vol, [0,0,0], [0,0,0], 'SPIDER', 'inv', 'C1'); + local_stack_mrc = MRCImage(stack_file_path, 0); + if verbose_timing, fprintf(' [W%d][T] GPU setup: %.3f s\n', iWorker, toc(t_worker_start)); end + + my_tilts = worker_assignments{iWorker}; + local_results = cell(length(my_tilts), 1); + + for j = 1:length(my_tilts) + tilt_idx = my_tilts(j); + member_indices = tilt_group_members{tilt_idx}; + current_tilt_angle = tilt_angles_per_group(tilt_idx); + current_tilt_name = unique_tilt_names{tilt_idx}; + n_particles_this_tilt = length(member_indices); + + t_tilt_start = tic; + fprintf(' [W%d] (%d/%d tilts, ~%.0f%%) Refining %s (angle %.1f deg, %d particles)...\n', ... + iWorker, j, length(my_tilts), 100 * j / length(my_tilts), ... + current_tilt_name, current_tilt_angle, n_particles_this_tilt); + + % Determine slice range for batch loading + t_io = tic; + slice_indices = all_positions(member_indices); + first_slice = min(slice_indices); + last_slice = max(slice_indices); + assert(last_slice - first_slice + 1 == n_particles_this_tilt, ... + 'Tilt %s: slices not consecutive (range %d-%d for %d particles)', ... + current_tilt_name, first_slice, last_slice, n_particles_this_tilt); + + % Batch-load data tiles from stack + tilt_data = single(OPEN_IMG('single', local_stack_mrc, [], [], [first_slice, last_slice], 'keep')); + if verbose_timing, fprintf(' [W%d][T] IO load: %.3f s\n', iWorker, toc(t_io)); end + + % Build data and reference tile cell arrays + data_tiles = cell(n_particles_this_tilt, 1); + ref_tiles = cell(n_particles_this_tilt, 1); + initial_shifts = zeros(n_particles_this_tilt, 2); + + ctf_params_for_tilt = struct(); + ctf_params_for_tilt.defocus_mean = zeros(n_particles_this_tilt, 1); + ctf_params_for_tilt.half_astigmatism = zeros(n_particles_this_tilt, 1); + ctf_params_for_tilt.astigmatism_angle = zeros(n_particles_this_tilt, 1); + ctf_params_for_tilt.pixel_size_angstroms = pixel_size_angstroms; + ctf_params_for_tilt.wavelength_angstroms = wavelength_angstroms; + ctf_params_for_tilt.spherical_aberration_mm = cs_mm; + ctf_params_for_tilt.amplitude_contrast = amplitude_contrast; + ctf_params_for_tilt.tilt_angle_degrees = current_tilt_angle; + + t_tile_prep = tic; + for particle_index = 1:n_particles_this_tilt + idx = member_indices(particle_index); + local_slice = all_positions(idx) - first_slice + 1; + + % Tile preparation (reference: BH_synthetic_mapBack.m lines 1527-1543) + data_tile = local_soft_mask .* gpuArray(tilt_data(:,:,local_slice)); + data_tile = data_tile - mean(data_tile(:)); + data_tile = data_tile ./ rms(data_tile(:)); + data_tiles{particle_index} = local_ctf_mask .* BH_padZeros3d(data_tile, 'fwd', padCTF, 'GPU', 'singleTaper'); + + % Reference projection: [phi, theta, psi] convention (permutation C) + angles = [all_phi(idx), all_theta(idx), all_psi(idx)]; + rotated_vol = local_ref_interp.interp3d(angles, [0,0,0], 'SPIDER', 'inv', 'C1'); + ref_projection = sum(rotated_vol, 3); + ref_tile = local_soft_mask .* center_crop_or_pad(ref_projection, tile_size); + ref_tile = ref_tile - mean(ref_tile(:)); + ref_tile = ref_tile ./ rms(ref_tile(:)); + ref_tiles{particle_index} = local_ctf_mask .* BH_padZeros3d(ref_tile, 'fwd', padCTF, 'GPU', 'singleTaper'); + + initial_shifts(particle_index, :) = [all_x_shift(idx) / pixel_size_angstroms, ... + all_y_shift(idx) / pixel_size_angstroms]; + + ctf_params_for_tilt.defocus_mean(particle_index) = (all_df1(idx) + all_df2(idx)) / 2; + ctf_params_for_tilt.half_astigmatism(particle_index) = (all_df1(idx) - all_df2(idx)) / 2; + ctf_params_for_tilt.astigmatism_angle(particle_index) = all_df_angle(idx) * pi / 180; + end + if verbose_timing, fprintf(' [W%d][T] tile prep (%d particles): %.3f s\n', iWorker, n_particles_this_tilt, toc(t_tile_prep)); end + + tilt_data = []; %#ok + + % Run ADAM refinement + t_refine = tic; + tilt_results = EMC_refine_tilt_ctf(data_tiles, ref_tiles, ctf_params_for_tilt, ... + initial_shifts, refinement_options); + if verbose_timing, fprintf(' [W%d][T] refinement: %.3f s (%d iters)\n', iWorker, toc(t_refine), length(tilt_results.score_history)); end + + % Flag parameters that hit their search bounds + df_range = refinement_options.defocus_search_range; + astig_bound = df_range / 2; + angle_bound = refinement_options.astigmatism_angle_range; + bound_tol = 0.99; % within 1% of bound = saturated + flags = ''; + if abs(tilt_results.delta_defocus_tilt) >= bound_tol * df_range + flags = [flags ' DEFOCUS_SAT']; + end + if abs(tilt_results.delta_half_astigmatism) >= bound_tol * astig_bound + flags = [flags ' ASTIG_SAT']; + end + if abs(tilt_results.delta_astigmatism_angle) >= bound_tol * angle_bound + flags = [flags ' ANGLE_SAT']; + end + n_iters = length(tilt_results.score_history); + score_mean = mean(tilt_results.per_particle_scores); + score_std = std(tilt_results.per_particle_scores); + initial_avg_defocus = mean(ctf_params_for_tilt.defocus_mean); + score_initial = tilt_results.score_history(1) / n_particles_this_tilt; + score_final = tilt_results.score_history(end) / n_particles_this_tilt; + if score_final ~= 0 + score_change_pct = 100 * (score_final - score_initial) / score_final; + else + score_change_pct = 0; + end + fprintf(' [W%d] %s | angle=%.1f | %d particles | %d iters | initDF=%.0f dDF=%.1f dAstig=%.1f dAngle=%.1f deg | score=%.4f+/-%.4f dScore=%.1f%% | conv=%d', ... + iWorker, current_tilt_name, current_tilt_angle, n_particles_this_tilt, n_iters, ... + initial_avg_defocus, tilt_results.delta_defocus_tilt, tilt_results.delta_half_astigmatism, ... + tilt_results.delta_astigmatism_angle * 180/pi, score_mean, score_std, score_change_pct, tilt_results.converged); + if ~isempty(flags) + fprintf(' |%s', flags); + end + if verbose_timing, fprintf(' | %.3fs', toc(t_tilt_start)); end + fprintf('\n'); + + local_results{j} = struct( ... + 'tilt_idx', tilt_idx, ... + 'member_indices', member_indices, ... + 'tilt_angle', current_tilt_angle, ... + 'tilt_name', current_tilt_name, ... + 'n_particles', n_particles_this_tilt, ... + 'tilt_results', tilt_results); + + if ~isempty(progress_queue) + send(progress_queue, struct('saturated', ~isempty(flags))); + end + end + + worker_results{iWorker} = local_results; +end + +% Clean up parallel pool and re-initialize GPUs (matches BH_average3d / BH_alignRaw3d pattern) +delete(gcp('nocreate')); +for iGPU = 1:n_gpus + gpuDevice(gpu_ids(iGPU)); +end + +%% ===== Stage G: Unpack parallel results ===== + +% Initialize from originals — unrefined particles keep their input values +refined_defocus_1 = all_df1; +refined_defocus_2 = all_df2; +refined_astigmatism_angle = all_df_angle; +refined_shift_x = all_x_shift; +refined_shift_y = all_y_shift; +refined_scores = zeros(n_total_particles, 1); +refined_occupancy = 100 * ones(n_total_particles, 1); +processed_mask = false(n_total_particles, 1); +for iWorker = 1:n_workers + for j = 1:length(worker_results{iWorker}) + r = worker_results{iWorker}{j}; + tilt_results = r.tilt_results; + member_indices = r.member_indices; + current_tilt_angle = r.tilt_angle; + + for particle_index = 1:r.n_particles + idx = member_indices(particle_index); + + particle_dz = tilt_results.delta_z(particle_index); + defocus_correction = tilt_results.delta_defocus_tilt + particle_dz * cosd(current_tilt_angle); + + defocus_mean = (all_df1(idx) + all_df2(idx)) / 2; + half_astig = (all_df1(idx) - all_df2(idx)) / 2; + + refined_defocus_1(idx) = defocus_mean + half_astig + ... + tilt_results.delta_half_astigmatism + defocus_correction; + refined_defocus_2(idx) = defocus_mean - half_astig - ... + tilt_results.delta_half_astigmatism + defocus_correction; + refined_astigmatism_angle(idx) = (all_df_angle(idx) * pi/180 + tilt_results.delta_astigmatism_angle) * 180 / pi; + refined_shift_x(idx) = tilt_results.shift_x(particle_index) * pixel_size_angstroms; + refined_shift_y(idx) = tilt_results.shift_y(particle_index) * pixel_size_angstroms; + refined_scores(idx) = tilt_results.per_particle_scores(particle_index); + refined_occupancy(idx) = 100; + processed_mask(idx) = true; + end + end +end + +%% ===== Summary statistics ===== +n_processed = sum(processed_mask); +n_unprocessed = n_total_particles - n_processed; +n_tilts_total = 0; +n_tilts_df_sat = 0; +n_tilts_astig_sat = 0; +n_tilts_angle_sat = 0; +n_tilts_any_sat = 0; +bound_tol = 0.99; +df_range = refinement_options.defocus_search_range; +angle_range = refinement_options.astigmatism_angle_range; +for iW = 1:n_workers + for j = 1:length(worker_results{iW}) + n_tilts_total = n_tilts_total + 1; + tr = worker_results{iW}{j}.tilt_results; + is_df = abs(tr.delta_defocus_tilt) >= bound_tol * df_range; + is_astig = abs(tr.delta_half_astigmatism) >= bound_tol * df_range / 2; + is_angle = abs(tr.delta_astigmatism_angle) >= bound_tol * angle_range; + n_tilts_df_sat = n_tilts_df_sat + is_df; + n_tilts_astig_sat = n_tilts_astig_sat + is_astig; + n_tilts_angle_sat = n_tilts_angle_sat + is_angle; + n_tilts_any_sat = n_tilts_any_sat + (is_df || is_astig || is_angle); + end +end + +fprintf('\n=== Refinement Summary ===\n'); +fprintf(' Tilt groups: %d processed\n', n_tilts_total); +fprintf(' Particles processed: %d / %d (%.1f%%)\n', n_processed, n_total_particles, ... + 100 * n_processed / max(n_total_particles, 1)); +fprintf(' Particles unprocessed: %d / %d (%.1f%%)\n', n_unprocessed, n_total_particles, ... + 100 * n_unprocessed / max(n_total_particles, 1)); +fprintf(' Saturation: %d / %d tilts (%.1f%%) hit any bound\n', ... + n_tilts_any_sat, n_tilts_total, 100 * n_tilts_any_sat / max(n_tilts_total, 1)); +fprintf(' Defocus: %d (%.1f%%)\n', n_tilts_df_sat, 100 * n_tilts_df_sat / max(n_tilts_total, 1)); +fprintf(' Astigmatism: %d (%.1f%%)\n', n_tilts_astig_sat, 100 * n_tilts_astig_sat / max(n_tilts_total, 1)); +fprintf(' Angle: %d (%.1f%%)\n', n_tilts_angle_sat, 100 * n_tilts_angle_sat / max(n_tilts_total, 1)); + +if n_processed > 0 + defocus_corrections = refined_defocus_1(processed_mask) - all_df1(processed_mask); + fprintf(' Defocus corrections (processed only): mean=%.1f A, std=%.1f A\n', ... + mean(defocus_corrections), std(defocus_corrections)); +end + +if n_unprocessed > 0 + fprintf(' WARNING: %d particles have zeros (unprocessed or failed)\n', n_unprocessed); +end + +%% ===== Per-tilt diagnostic log ===== +% Save a tab-delimited file with per-tilt diagnostics for post-processing +% analysis (score outlier detection, saturation filtering, occupancy decisions). + +n_tilts_processed = 0; +for iW = 1:n_workers + n_tilts_processed = n_tilts_processed + length(worker_results{iW}); +end + +diag = struct(); +diag.tilt_name = cell(n_tilts_processed, 1); +diag.tilt_angle = zeros(n_tilts_processed, 1); +diag.n_particles = zeros(n_tilts_processed, 1); +diag.n_iters = zeros(n_tilts_processed, 1); +diag.converged = false(n_tilts_processed, 1); +diag.delta_df = zeros(n_tilts_processed, 1); +diag.delta_astig = zeros(n_tilts_processed, 1); +diag.delta_angle = zeros(n_tilts_processed, 1); +diag.score_mean = zeros(n_tilts_processed, 1); +diag.score_std = zeros(n_tilts_processed, 1); +diag.score_min = zeros(n_tilts_processed, 1); +diag.score_max = zeros(n_tilts_processed, 1); +diag.df_sat = false(n_tilts_processed, 1); +diag.astig_sat = false(n_tilts_processed, 1); +diag.angle_sat = false(n_tilts_processed, 1); +diag.score_change_pct = zeros(n_tilts_processed, 1); + +bound_tol = 0.99; +df_range = refinement_options.defocus_search_range; +angle_range = refinement_options.astigmatism_angle_range; + +row = 0; +for iW = 1:n_workers + for j = 1:length(worker_results{iW}) + row = row + 1; + r = worker_results{iW}{j}; + tr = r.tilt_results; + diag.tilt_name{row} = r.tilt_name; + diag.tilt_angle(row) = r.tilt_angle; + diag.n_particles(row) = r.n_particles; + diag.n_iters(row) = length(tr.score_history); + diag.converged(row) = tr.converged; + diag.delta_df(row) = tr.delta_defocus_tilt; + diag.delta_astig(row) = tr.delta_half_astigmatism; + diag.delta_angle(row) = tr.delta_astigmatism_angle; + scores = tr.per_particle_scores; + diag.score_mean(row) = mean(scores); + diag.score_std(row) = std(scores); + diag.score_min(row) = min(scores); + diag.score_max(row) = max(scores); + diag.df_sat(row) = abs(tr.delta_defocus_tilt) >= bound_tol * df_range; + diag.astig_sat(row) = abs(tr.delta_half_astigmatism) >= bound_tol * df_range / 2; + diag.angle_sat(row) = abs(tr.delta_astigmatism_angle) >= bound_tol * angle_range; + s_init = tr.score_history(1) / r.n_particles; + s_final = tr.score_history(end) / r.n_particles; + if s_final ~= 0 + diag.score_change_pct(row) = 100 * (s_final - s_init) / s_final; + end + end +end + +[out_dir, out_base, ~] = fileparts(output_star_path); +if isempty(out_dir), out_dir = '.'; end +diag_path = fullfile(out_dir, [out_base '_diagnostics.txt']); +fid = fopen(diag_path, 'w'); +fprintf(fid, 'tilt_name\ttilt_angle\tn_particles\tn_iters\tconverged\tdelta_df\tdelta_astig\tdelta_angle_deg\tscore_mean\tscore_std\tscore_min\tscore_max\tscore_change_pct\tdf_sat\tastig_sat\tangle_sat\n'); +for row = 1:n_tilts_processed + fprintf(fid, '%s\t%.2f\t%d\t%d\t%d\t%.1f\t%.1f\t%.1f\t%.6f\t%.6f\t%.6f\t%.6f\t%.1f\t%d\t%d\t%d\n', ... + diag.tilt_name{row}, diag.tilt_angle(row), diag.n_particles(row), ... + diag.n_iters(row), diag.converged(row), diag.delta_df(row), ... + diag.delta_astig(row), diag.delta_angle(row) * 180/pi, diag.score_mean(row), ... + diag.score_std(row), diag.score_min(row), diag.score_max(row), ... + diag.score_change_pct(row), diag.df_sat(row), diag.astig_sat(row), diag.angle_sat(row)); +end +fclose(fid); +fprintf(' Diagnostics: %s (%d tilt groups)\n', diag_path, n_tilts_processed); + +%% ===== Stage F: Write refined star file ===== + +fprintf('Writing refined star file: %s\n', output_star_path); +write_refined_star_file(star_file_path, output_star_path, particles, ... + refined_defocus_1, refined_defocus_2, refined_astigmatism_angle, ... + refined_shift_x, refined_shift_y, refined_scores, refined_occupancy); + +fprintf('=== CTF refinement complete ===\n'); +fprintf(' Input: %s (%d particles)\n', star_file_path, n_total_particles); +fprintf(' Output: %s\n', output_star_path); + +end % EMC_ctf_refine_from_star + + +%% ===== Local Functions ===== + +function opts = parse_options(args) +% Parse name-value option pairs from varargin cell array. + opts.defocus_search_range = 5000; + opts.maximum_iterations = 15; + opts.lowpass_cutoff = 10; + opts.astigmatism_angle_range = pi/4; + opts.z_offset_bound_factor = 5; + opts.minimum_global_iterations = 3; % iterations with only per-tilt params before per-particle delta_z + opts.global_only = false; % if true, only optimize per-tilt params (no per-particle delta_z) + opts.shift_sigma = 5.0; % Gaussian penalty sigma for X/Y shifts (Angstroms) + opts.highpass_cutoff = 400; % highpass cutoff in Angstroms for bandpass filter + opts.use_phase_compensated_correlation = false; % phase-doubled correlation (C * C/(|C| + eps)) + opts.debug_tilt_list = ''; % comma-separated tilt names to process (empty = all) + opts.exit_after_n_tilts = 0; % 0 = process all, N = process first N tilt groups + opts.verbose_timing = false; + opts.enable_progress = true; % DataQueue progress tracking during parfor + opts.debug_print = false; % print all option values after parsing + opts.gpu_ids = '1'; + opts.workers_per_gpu = 3; + + i = 1; + while i <= length(args) + if ischar(args{i}) || isstring(args{i}) + key = char(args{i}); + if i + 1 <= length(args) + val = args{i+1}; + if ischar(val) || isstring(val) + val_str = lower(char(val)); + if strcmp(val_str, 'true') || strcmp(val_str, '1') + val = true; + elseif strcmp(val_str, 'false') || strcmp(val_str, '0') + val = false; + else + num_val = str2double(val); + if ~isnan(num_val) + val = num_val; + else + % Handle vector syntax like '[1,2,5]' or '1,2,5' + num_val = str2num(char(args{i+1})); %#ok + if ~isempty(num_val) + val = num_val; + end + % Otherwise keep val as the original string + end + end + end + if isfield(opts, key) + opts.(key) = val; + else + fprintf('Warning: unknown option "%s", ignoring\n', key); + end + i = i + 2; + else + fprintf('Warning: option "%s" has no value, ignoring\n', key); + i = i + 1; + end + else + i = i + 1; + end + end +end + + +function update_progress(msg) +% DataQueue callback: accumulate progress from parfor workers. + persistent n_done n_sat n_total; + if ischar(msg) && strcmp(msg, 'reset') + n_done = 0; n_sat = 0; n_total = 0; + return; + end + if ischar(msg) && strcmp(msg, 'set_total') + return; + end + if isstruct(msg) && isfield(msg, 'total') + n_done = 0; n_sat = 0; n_total = msg.total; + return; + end + if isempty(n_done), n_done = 0; n_sat = 0; n_total = 0; end + n_done = n_done + 1; + n_sat = n_sat + msg.saturated; + fprintf(' [PROGRESS] %d / %d tilts (%.0f%%) | %d saturated (%.0f%%)\n', ... + n_done, n_total, 100 * n_done / max(n_total, 1), ... + n_sat, 100 * n_sat / max(n_done, 1)); +end + + +function print_struct_fields(label, s) +% Print all fields of a struct with type-aware formatting. + fprintf('\n --- %s ---\n', label); + fields = fieldnames(s); + for i = 1:length(fields) + val = s.(fields{i}); + if islogical(val) + fprintf(' %-40s %s\n', fields{i}, mat2str(val)); + elseif isnumeric(val) + fprintf(' %-40s %s\n', fields{i}, mat2str(val, 6)); + elseif ischar(val) || isstring(val) + fprintf(' %-40s %s\n', fields{i}, char(val)); + else + fprintf(' %-40s [%s]\n', fields{i}, class(val)); + end + end + fprintf(' ---\n\n'); +end + + +function [particles, header_lines] = parse_star_file(path) +% Parse a 30-column star file into a struct array. + fh = fopen(path, 'r'); + if fh == -1 + error('Cannot open star file: %s', path); + end + + header_lines = {}; + data_lines = {}; + + while ~feof(fh) + line = fgetl(fh); + if isempty(strtrim(line)) + header_lines{end+1} = line; %#ok + continue; + end + first_char = strtrim(line); + first_char = first_char(1); + if first_char == '#' || first_char == '_' || ... + startsWith(strtrim(line), 'data_') || startsWith(strtrim(line), 'loop_') + header_lines{end+1} = line; %#ok + else + data_lines{end+1} = line; %#ok + end + end + fclose(fh); + + n = length(data_lines); + particles = struct('position_in_stack', cell(1,n), ... + 'psi', cell(1,n), 'theta', cell(1,n), 'phi', cell(1,n), ... + 'x_shift', cell(1,n), 'y_shift', cell(1,n), ... + 'defocus_1', cell(1,n), 'defocus_2', cell(1,n), 'defocus_angle', cell(1,n), ... + 'phase_shift', cell(1,n), 'occupancy', cell(1,n), ... + 'logp', cell(1,n), 'sigma', cell(1,n), 'score', cell(1,n), ... + 'score_change', cell(1,n), 'pixel_size', cell(1,n), ... + 'voltage_kv', cell(1,n), 'cs_mm', cell(1,n), 'amplitude_contrast', cell(1,n), ... + 'beam_tilt_x', cell(1,n), 'beam_tilt_y', cell(1,n), ... + 'image_shift_x', cell(1,n), 'image_shift_y', cell(1,n), ... + 'best_2d_class', cell(1,n), 'beam_tilt_group', cell(1,n), ... + 'particle_group', cell(1,n), 'pre_exposure', cell(1,n), ... + 'total_exposure', cell(1,n), ... + 'original_image_filename', cell(1,n), 'tilt_angle', cell(1,n)); + + for i = 1:n + tokens = strsplit(strtrim(data_lines{i})); + if length(tokens) < 29 + error('Star file line %d has only %d tokens (expected >= 29)', i, length(tokens)); + end + + particles(i).position_in_stack = str2double(tokens{1}); + particles(i).psi = str2double(tokens{2}); + particles(i).theta = str2double(tokens{3}); + particles(i).phi = str2double(tokens{4}); + particles(i).x_shift = str2double(tokens{5}); + particles(i).y_shift = str2double(tokens{6}); + particles(i).defocus_1 = str2double(tokens{7}); + particles(i).defocus_2 = str2double(tokens{8}); + particles(i).defocus_angle = str2double(tokens{9}); + particles(i).phase_shift = str2double(tokens{10}); + particles(i).occupancy = str2double(tokens{11}); + particles(i).logp = str2double(tokens{12}); + particles(i).sigma = str2double(tokens{13}); + particles(i).score = str2double(tokens{14}); + particles(i).score_change = str2double(tokens{15}); + particles(i).pixel_size = str2double(tokens{16}); + particles(i).voltage_kv = str2double(tokens{17}); + particles(i).cs_mm = str2double(tokens{18}); + particles(i).amplitude_contrast = str2double(tokens{19}); + particles(i).beam_tilt_x = str2double(tokens{20}); + particles(i).beam_tilt_y = str2double(tokens{21}); + particles(i).image_shift_x = str2double(tokens{22}); + particles(i).image_shift_y = str2double(tokens{23}); + particles(i).best_2d_class = str2double(tokens{24}); + particles(i).beam_tilt_group = str2double(tokens{25}); + particles(i).particle_group = str2double(tokens{26}); + particles(i).pre_exposure = str2double(tokens{27}); + particles(i).total_exposure = str2double(tokens{28}); + particles(i).original_image_filename = tokens{29}; + + % Column 30: tilt angle (stored as _cisTEMOriginalXPosition) + if length(tokens) >= 30 + particles(i).tilt_angle = str2double(tokens{30}); + else + particles(i).tilt_angle = 0; + fprintf('Warning: particle %d missing tilt angle (col 30), defaulting to 0\n', i); + end + end +end + + +function mask = create_2d_soft_mask(tile_size, taper_width) +% Create 2D soft-edge cylindrical mask with cosine taper. + radius = floor(min(tile_size) ./ 2) - taper_width; + origin = emc_get_origin_index(tile_size); + [gx, gy] = ndgrid(1:tile_size(1), 1:tile_size(2)); + dist = sqrt((gx - origin(1)).^2 + (gy - origin(2)).^2); + + mask = ones(tile_size, 'single'); + taper_region = (dist > radius) & (dist <= radius + taper_width); + mask(taper_region) = 0.5 .* (1 + cos(pi .* (dist(taper_region) - radius) ./ taper_width)); + mask(dist > radius + taper_width) = 0; + mask = gpuArray(mask); +end + + +function [unique_names, group_indices, angles_per_group] = group_particles_by_tilt(particles) +% Group particles by tilt image filename and extract tilt angle per group. +% +% Returns: +% unique_names - cell array of unique tilt image filenames +% group_indices - Nx1 array mapping each particle to its group index +% angles_per_group - Mx1 array of tilt angles (from first particle in each group) + + all_tilt_names = {particles.original_image_filename}; + [unique_names, ~, group_indices] = unique(all_tilt_names); + n_groups = length(unique_names); + + angles_per_group = zeros(n_groups, 1); + for g = 1:n_groups + members = find(group_indices == g); + angles_per_group(g) = particles(members(1)).tilt_angle; + end +end + + +function output = center_crop_or_pad(input, target_size) +% Center-crop or center-pad a 2D array to match target_size. + input_size = size(input); + + if all(input_size(1:2) == target_size) + output = input; + return; + end + + % Determine output origin and input origin + output = zeros(target_size, 'like', input); + input_origin = emc_get_origin_index(input_size(1:2)); + output_origin = emc_get_origin_index(target_size); + + % Compute overlapping region + % In input coordinates + in_start = max(1, input_origin - output_origin + 1); + in_end = min(input_size(1:2), input_origin + (target_size - output_origin)); + + % Corresponding output coordinates + out_start = max(1, output_origin - input_origin + 1); + out_end = out_start + (in_end - in_start); + + output(out_start(1):out_end(1), out_start(2):out_end(2)) = ... + input(in_start(1):in_end(1), in_start(2):in_end(2)); +end + + +function write_refined_star_file(input_path, output_path, particles, ... + refined_df1, refined_df2, refined_ast_angle, ... + refined_sx, refined_sy, refined_scores, refined_occ) +% Write refined star file by reading original and replacing refined columns. + + fh_in = fopen(input_path, 'r'); + if fh_in == -1 + error('Cannot open input star file for reading: %s', input_path); + end + fh_out = fopen(output_path, 'w'); + if fh_out == -1 + fclose(fh_in); + error('Cannot open output star file for writing: %s', output_path); + end + + particle_idx = 0; + + while ~feof(fh_in) + line = fgetl(fh_in); + if line == -1 + break; + end + + trimmed = strtrim(line); + if isempty(trimmed) + fprintf(fh_out, '%s\n', line); + continue; + end + + % Check if this is a data line (starts with a number) + first_char = trimmed(1); + if first_char >= '0' && first_char <= '9' + tokens = strsplit(trimmed); + stack_pos = str2double(tokens{1}); + if ~isnan(stack_pos) && stack_pos >= 1 && particle_idx < length(refined_df1) + particle_idx = particle_idx + 1; + + % Replace columns 5-6 (shifts), 7-9 (defocus), 11 (occupancy), 14 (score) + tokens{5} = sprintf('%9.2f', refined_sx(particle_idx)); + tokens{6} = sprintf('%9.2f', refined_sy(particle_idx)); + tokens{7} = sprintf('%8.1f', refined_df1(particle_idx)); + tokens{8} = sprintf('%8.1f', refined_df2(particle_idx)); + tokens{9} = sprintf('%7.2f', refined_ast_angle(particle_idx)); + tokens{11} = sprintf('%5i', refined_occ(particle_idx)); + tokens{14} = sprintf('%10.4f', refined_scores(particle_idx)); + % cisTEM expects filenames in single quotes + if length(tokens) >= 29 && tokens{29}(1) ~= '''' + tokens{29} = ['''' tokens{29} '''']; + end + fprintf(fh_out, '%s\n', strjoin(tokens, ' ')); + continue; + end + end + + % Header or comment line - pass through unchanged + fprintf(fh_out, '%s\n', line); + end + + fclose(fh_in); + fclose(fh_out); + + fprintf(' Wrote %d refined particle records\n', particle_idx); +end diff --git a/ctf/EMC_tlt_get_projection_values.m b/ctf/EMC_tlt_get_projection_values.m new file mode 100644 index 00000000..5a67fdf9 --- /dev/null +++ b/ctf/EMC_tlt_get_projection_values.m @@ -0,0 +1,96 @@ +function [values] = EMC_tlt_get_projection_values(tlt_data, column_spec) +%EMC_tlt_get_projection_values Extract TLT values indexed by projection position +% +% values = EMC_tlt_get_projection_values(tlt_data, column_spec) +% +% Extracts values from a TLT matrix, sorted and indexed by projection +% position (column 1). This ensures values align with stack slice order. +% +% Input: +% tlt_data - Loaded TLT matrix (n_tilts x 23 columns) +% column_spec - Either: +% 'exposure' - Calculate per-projection exposure from cumulative dose +% numeric - Column index(es) to extract directly +% +% Output: +% values - Values indexed by projection position (1 to n_tilts) +% For 'exposure': n_tilts x 1 per-projection dose (e-/A^2) +% For numeric: n_tilts x length(column_spec) +% +% Examples: +% % Get per-projection exposure +% exposure = EMC_tlt_get_projection_values(TLT, 'exposure'); +% +% % Get defocus values (column 15) +% defocus = EMC_tlt_get_projection_values(TLT, 15); +% +% % Get multiple columns: defocus, astig_mag, astig_angle +% ctf_params = EMC_tlt_get_projection_values(TLT, [15, 12, 13]); +% +% TLT Column Reference: +% 1 - Projection index +% 4 - Tilt angle (degrees) +% 11 - Cumulative dose (e-/A^2) +% 12 - Astigmatism magnitude (meters) +% 13 - Astigmatism angle (radians) +% 15 - Defocus (meters) +% 16 - Pixel size (meters) +% 17 - Cs (meters) +% 18 - Wavelength (meters) +% 19 - Amplitude contrast (0-1) +% +% See also: EMC_generate_projections, EMC_setup_synthetic_project + +% Validate input +if isempty(tlt_data) + error('EMC_tlt_get_projection_values:EmptyInput', 'TLT data is empty'); +end + +n_tilts = size(tlt_data, 1); + +% Sort by projection index (column 1) to ensure correct ordering +tlt_sorted = sortrows(tlt_data, 1); + +% Handle column_spec +if ischar(column_spec) || isstring(column_spec) + switch lower(column_spec) + case 'exposure' + % Calculate per-projection exposure from cumulative dose (column 11) + cumulative_dose = tlt_sorted(:, 11); + values = zeros(n_tilts, 1); + + for j = 1:n_tilts + this_cumulative = cumulative_dose(j); + % Find doses smaller than this one (earlier in exposure order) + earlier_doses = cumulative_dose(cumulative_dose < this_cumulative); + if isempty(earlier_doses) + % This is the first exposure + values(j) = this_cumulative; + else + % Previous exposure is the max of all earlier doses + values(j) = this_cumulative - max(earlier_doses); + end + end + + otherwise + error('EMC_tlt_get_projection_values:UnknownSpec', ... + 'Unknown column_spec: %s. Use ''exposure'' or numeric column indices.', ... + column_spec); + end + +elseif isnumeric(column_spec) + % Direct column extraction + max_col = size(tlt_sorted, 2); + if any(column_spec < 1) || any(column_spec > max_col) + error('EMC_tlt_get_projection_values:InvalidColumn', ... + 'Column index out of range. TLT has %d columns.', max_col); + end + + values = tlt_sorted(:, column_spec); + +else + error('EMC_tlt_get_projection_values:InvalidSpec', ... + 'column_spec must be a string (''exposure'') or numeric column indices.'); +end + +end diff --git a/docs/LINTING_MIGRATION_GUIDE.md b/docs/LINTING_MIGRATION_GUIDE.md new file mode 100644 index 00000000..331bc46d --- /dev/null +++ b/docs/LINTING_MIGRATION_GUIDE.md @@ -0,0 +1,215 @@ +# Development Setup Guide: New Linting Tools + +This guide covers the transition from black/isort/flake8/mypy to ruff/pyright/bandit/safety. + +## Quick Start + +1. **Install the new tools:** + ```bash + pip install -e .[dev] # Installs ruff, pyright, bandit, safety + ``` + +2. **Run the migration script:** + ```bash + python migrate_to_ruff.py + ``` + +3. **Update pre-commit hooks:** + ```bash + pre-commit install + pre-commit run --all-files + ``` + +## Tool Overview + +### Ruff (Replaces: black + isort + flake8 + many plugins) +- **Purpose**: Linting, formatting, and import sorting in one tool +- **Speed**: 10-100x faster than the old tools +- **Configuration**: `pyproject.toml` under `[tool.ruff]` + +**Common commands:** +```bash +# Lint and auto-fix +ruff check python/ --fix + +# Format code +ruff format python/ + +# Check specific rules +ruff check python/ --select F,E,W + +# Show what would be fixed +ruff check python/ --diff +``` + +### Pyright (Replaces: mypy) +- **Purpose**: Static type checking +- **Speed**: Faster than mypy, better editor integration +- **Configuration**: `pyproject.toml` under `[tool.pyright]` + +**Common commands:** +```bash +# Type check all files +pyright python/ + +# Type check specific files +pyright python/metaData/emc_parameter_converter.py + +# Watch mode for development +pyright --watch python/ +``` + +### Bandit (Security linting) +- **Purpose**: Find common security issues +- **Configuration**: `pyproject.toml` under `[tool.bandit]` + +**Common commands:** +```bash +# Security scan +bandit -r python/ -c pyproject.toml + +# Generate detailed report +bandit -r python/ -c pyproject.toml -f json -o security-report.json +``` + +### Safety (Dependency vulnerability scanning) +- **Purpose**: Check for known vulnerabilities in dependencies +- **Usage**: Scans installed packages + +**Common commands:** +```bash +# Check for vulnerabilities +safety check + +# Generate report +safety check --json --output safety-report.json +``` + +## Editor Configuration + +### VS Code +Add to your `settings.json`: +```json +{ + "python.linting.enabled": true, + "python.linting.ruffEnabled": true, + "python.formatting.provider": "none", + "python.analysis.typeCheckingMode": "basic", + "ruff.enable": true, + "ruff.organizeImports": true, + "ruff.fixAll": true, + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.fixAll.ruff": true, + "source.organizeImports.ruff": true + } +} +``` + +### PyCharm/IntelliJ +1. Install the Ruff plugin +2. Configure Ruff as the formatter and linter +3. Enable Pyright or configure the built-in type checker + +## Rule Configuration + +The new setup enables comprehensive checking: + +- **E, W, F**: Standard Python style and syntax +- **I**: Import sorting (replaces isort) +- **UP**: Modern Python syntax (pyupgrade equivalent) +- **B**: Bug detection (flake8-bugbear) +- **N**: Naming conventions +- **C90**: Complexity checking +- **PIE, SIM**: Code simplification +- **T20**: Print statement detection +- **PD**: Pandas best practices +- **PL**: Pylint-style checks +- **RUF**: Ruff-specific rules +- **D**: Docstring style + +### Ignoring Rules + +To ignore specific rules temporarily: +```python +# ruff: noqa: E501 # Line too long +long_line = "This is a very long line that exceeds the limit but is necessary" + +# ruff: noqa # Ignore all rules for this line +problematic_code() +``` + +For permanent ignores, update `pyproject.toml`: +```toml +[tool.ruff.lint] +ignore = ["E501", "D101"] # Ignore line length and missing docstrings +``` + +## Pre-commit Integration + +The new pre-commit hooks run automatically on commit: + +1. **Ruff linting** with auto-fixes +2. **Ruff formatting** +3. **Pyright type checking** +4. **Bandit security scanning** +5. **Basic file checks** (trailing whitespace, etc.) + +To run manually: +```bash +pre-commit run --all-files +``` + +To skip hooks temporarily: +```bash +git commit -m "message" --no-verify +``` + +## Migration Notes + +### From Black/isort +- Ruff's formatter is compatible with Black +- Import sorting behavior matches isort with `profile = "black"` +- Line length defaults to 88 characters (Black's default) + +### From Flake8 +- Most flake8 rules are included in Ruff's default set +- Plugin functionality (bugbear, etc.) is built into Ruff +- Custom ignore lists have been migrated to `pyproject.toml` + +### From MyPy +- Pyright provides similar type checking with better performance +- Some mypy-specific annotations may need adjustment +- Pyright is more strict about some type issues + +## Troubleshooting + +### Common Issues + +1. **Too many lint errors**: Use `ruff check --fix` to auto-fix many issues +2. **Type checking failures**: Add type annotations gradually +3. **Import order conflicts**: Ruff handles this automatically now +4. **Performance in large repos**: Ruff is designed for speed + +### Getting Help + +- **Ruff docs**: https://docs.astral.sh/ruff/ +- **Pyright docs**: https://microsoft.github.io/pyright/ +- **Bandit docs**: https://bandit.readthedocs.io/ + +### Reverting Changes + +If you need to temporarily revert: +1. Checkout the old configuration files +2. Reinstall old tools: `pip install black isort flake8 mypy` +3. Run `pre-commit clean && pre-commit install` + +## Performance Comparison + +The new setup is significantly faster: + +- **Ruff**: ~100x faster than flake8, ~10x faster than black +- **Pyright**: ~5-10x faster than mypy +- **Overall**: Developer feedback loop improved dramatically + +This speed improvement makes it practical to run comprehensive checks on every save/commit. diff --git a/docs/context/MAIN_CONTEXT.md b/docs/context/MAIN_CONTEXT.md new file mode 100644 index 00000000..f54010e2 --- /dev/null +++ b/docs/context/MAIN_CONTEXT.md @@ -0,0 +1,91 @@ +# emClarity Main Context File + +This file aggregates key development context for AI assistants working on emClarity. + +## Quick Start for AI Assistants + +Read these files for complete context: + +- `../PYTHON_STYLE_GUIDE.md` - Essential Python coding standards +- `../emClarity_Tutorial.md` - Project overview and usage +- `copilot-instructions.md` - AI assistant behavioral guidelines and critical rules +- `python_conversion_instructions.md` - MATLAB to Python conversion guidelines +- `GUI_IMPLEMENTATION_SUMMARY.md` - GUI architecture +- `agent_notes.md` - Accumulated development insights + +## Key Development Rules Summary + +### Critical Rules (from copilot-instructions.md) + +- **Never replace real panels/widgets with dummy versions** without user approval +- **Never alter production database** - always work on copies +- **All temporary files must go in /tmp/copilot-test/** - never in project directories +- **All development scripts (testing, migration, etc.) must go in /tmp/agent-tmp/** - never in project directories +- Start with simplest solutions and explain if scope needs expansion + +### Python Development + +- Use Black + isort with pyproject.toml configuration +- Follow PEP 8 with 88-character line length +- Use type hints consistently +- Prefer f-strings over .format() or % formatting + +### MATLAB to Python Conversion (from python_conversion_instructions.md) + +- Mirror directory structure: `metaData/BH_file.m` → `python/metaData/emc_file.py` +- Use `emc_` prefix for Python modules +- Create unit tests in `python/folderName/tests/` +- Update README.md files and agent_notes.md +- For CUDA: use `extern "C"` wrapper, prefer int to uint +- **Always use Fortran-style (column-major) CuPy arrays** - construct with `order='F'` +- **Never use ambiguous terms like rows/columns** - use nx/ny/nz consistently +- **Wrap library call outputs with ensure_f()** to catch memory layout issues + +### Git Workflow + +- Work on feature branches (like ctf3d_work) +- Ensure CI passes before merging +- Use descriptive commit messages + +### AI Assistant Guidelines + +- Always run linting checks locally before committing +- Test changes comprehensively +- Read full file context before making edits +- Use proper tool selection (replace_string_in_file vs edit_notebook_file) +- **Execute independent operations in parallel** - invoke relevant tools simultaneously for efficiency +- **Create general-purpose solutions** - implement code that handles all valid inputs, not just test cases +- **Avoid hard-coding values** - solutions should be robust, maintainable, and extendable +- **Focus on algorithm correctness** - understand requirements before implementation +- **Raise concerns about incorrect tests** - if tests seem wrong, communicate this clearly + +### Implementation Principles + +- **Algorithm first, tests second** - Tests verify correctness but don't define the solution +- **Focus on maintainability** - Write clean, well-documented code following established patterns +- **Design for the general case** - Solutions should handle edge cases and unusual inputs +- **Apply software design principles** - Use proper abstractions, separation of concerns +- **Report infeasibility** - If a task seems unreasonable or has contradictory requirements, communicate this +- **Maximize parallel operations** - When gathering context or performing multiple edits, batch operations + +## File Locations + +Key files are organized as: + +```text +emClarity/ +├── PYTHON_STYLE_GUIDE.md # Python coding standards +├── .clang-format # C++ formatting +├── pyproject.toml # Python tool configuration +├── docs/ +│ ├── emClarity_Tutorial.md # Main documentation +│ └── context/ # This directory +│ ├── MAIN_CONTEXT.md # This file +│ ├── copilot-instructions.md # AI assistant rules +│ ├── python_conversion_instructions.md # MATLAB→Python guidelines +│ ├── GUI_IMPLEMENTATION_SUMMARY.md +│ └── agent_notes.md +└── python/gui/prompts/ # AI assistant context files +``` + +Reference this file as: `#file:docs/context/MAIN_CONTEXT.md` diff --git a/docs/context/README.md b/docs/context/README.md new file mode 100644 index 00000000..b16caedd --- /dev/null +++ b/docs/context/README.md @@ -0,0 +1,34 @@ +# emClarity Development Context + +This directory contains key documentation and guidelines for working with emClarity. Reference this when working with AI assistants or new team members. + +## Core Documentation Files + +### Development Guidelines + +- `../PYTHON_STYLE_GUIDE.md` - Python coding standards and examples +- `../.clang-format` - C++ formatting configuration +- `copilot-instructions.md` - AI assistant behavioral guidelines and critical rules +- `python_conversion_instructions.md` - Guidelines for converting MATLAB to Python + +### Project Documentation + +- `../emClarity_Tutorial.md` - Main tutorial and usage guide +- `GUI_IMPLEMENTATION_SUMMARY.md` - GUI architecture and design decisions +- `agent_notes.md` - Accumulated notes from development sessions + +## Quick Reference for AI Assistants + +Instead of specifying individual files, you can reference: + +```markdown +#file:docs/context/MAIN_CONTEXT.md +``` + +This will provide the AI assistant with pointers to all relevant context files, and they can then read the specific files they need for the current task. + +## Usage Pattern + +1. Start conversations by referencing this README +2. AI assistant reads this index and determines which specific files to load +3. Reduces repetitive file specifications while maintaining access to all context diff --git a/docs/context/python_conversion_instructions.md b/docs/context/python_conversion_instructions.md new file mode 100644 index 00000000..34b5da09 --- /dev/null +++ b/docs/context/python_conversion_instructions.md @@ -0,0 +1,429 @@ +# Some basic rules for GH agent for converting emClarity to python + +## Structure + +- mirror the directory structure of the main project, eg, if we are adapting metaData/BH_parseParameter.m, we should create a corresponding Python file at python/metaData/emc_parseParameter.py + +- update a README.md file in the corresponding directory to explain the purpose and usage of the new Python module. + +- add unit tests for the new Python module, at python/folderName/tests/ + +- update an overall README at python/docs/agent_notes.md that includes useful information about the conversion process, any challenges encountered, and other details that will help future agent work on the project. + +## Temporary File Management + +- **All temporary files must go in /tmp/copilot-test/**: Never create temporary test files, demo data, star files, CSV files, or experimental files directly in the project directories. Always use `/tmp/copilot-test/` for any temporary files during development and testing. This keeps the project clean and prevents accidental commits of temporary data. + +- **Clean up after testing**: Remove temporary files from `/tmp/copilot-test/` when testing is complete, or document their purpose if they need to be preserved. + +## Naming Conventions + +- Use `emc_` prefix for Python modules (e.g., `BH_parseParameterFile.m` → `emc_parameter_converter.py`) +- Use descriptive names that reflect functionality +- Follow Python naming conventions (snake_case for functions and variables) + +## Formatting + +- when working on cuda, please use the guidlines from the file #python/cuda_ops/.clang-format +- CuPy requires names to not be mangled, so clean entry points for kernels must include extern "C" + + +## Image and Array conventions + +- For multi-dimensional arrays, emClarity refers to fastest dimension as X, the second as Y and the third as Z. +- Indexing in emClarity matlab files is started from 1 (vs 0 in python and cuda) + +## CUDA Integration Architecture + +- prefer int to uint where possible. + +### Pattern for CUDA-Accelerated Operations + +Following the established pattern in `python/cuda_ops/`, CUDA operations should use: + +1. **File Naming Convention**: `emc_module_name.cu` and `emc_module_name.py` +2. **CuPy RawModule Integration**: Load custom CUDA kernels via `cp.RawModule(code=cuda_source)` +3. **External C Linkage**: Use `extern "C" { }` wrapper in CUDA files for Python visibility +4. **Memory Management**: Support both CuPy arrays and raw GPU pointers +5. **Error Handling**: Comprehensive validation and CUDA error checking + +### CUDA Development Workflow + +1. **CUDA Kernel** (`.cu` file): + ```cuda + extern "C" { + __global__ void cuda_operation_name(const float* input, float* output, int n) { + // Kernel implementation + } + } + ``` + +2. **Python Wrapper** (`.py` file): + ```python + class CudaOperations: + def __init__(self): + self._load_cuda_kernels() + + def operation_name(self, input_array): + # CuPy interface with grid/block calculation + return result_array + ``` + +3. **Testing**: Compare against NumPy/CuPy reference implementations + +### Current CUDA Status + +**✅ Working Operations** (September 3, 2025): +- Array addition, scalar multiplication, 2D transpose +- CuPy RawModule integration established +- Comprehensive test framework + +**🔧 In Development**: +- 3D array transpose (index mapping issues) +- Performance optimization vs CuPy built-ins + +This CUDA architecture will replace MATLAB MEX files (like `mexFFT.cu`) with Python-wrapped implementations providing better memory management and integration. + +## Recent Completions + +### ✅ Parameter Management (September 3, 2025) + +**Converted**: `metaData/BH_parseParameterFile.m` → `python/metaData/emc_parameter_converter.py` + +**Key improvements**: +- Modern JSON format replacing MATLAB syntax +- Clear unit names (angstroms, mm, kV) instead of scientific notation +- Structured parameter organization (system, microscope, ctf, etc.) +- Full bidirectional conversion with backward compatibility +- Comprehensive JSON schema validation +- 100% test coverage with round-trip validation + +**Files created**: +- `python/metaData/emc_parameter_converter.py` - Main converter +- `python/metaData/tests/test_emc_parameter_converter.py` - Unit tests +- `python/metaData/README.md` - Module documentation +- `python/metaData/__init__.py` - Package initialization +- `python/docs/agent_notes.md` - Conversion tracking + +This conversion serves as the template for future module conversions. + +## Key Learnings from Development Sessions + +### CUDA Development Challenges & Solutions (September 3, 2025) + +**Challenge: CuPy Header File Inclusion** +- **Issue**: CuPy's RawModule cannot directly include external header files (`#include "emc_cuda_utils.cuh"`) +- **Solution**: Dynamically inline header content during Python wrapper compilation +- **Learning**: Always inline utility functions rather than relying on separate header includes for CuPy + +**Challenge: Vector Return Types vs Reference Parameters** +- **Issue**: Initial utility functions used reference parameters (`void get_2d_idx(int& x, int& y)`) +- **Improvement**: Switched to CUDA vector return types (`int2 get_2d_idx()`) +- **Learning**: CUDA vector types (`int2`, `int3`) provide cleaner, more idiomatic code + +**Challenge: emClarity Dimension Convention Mapping** +- **Issue**: Multiple iterations needed to correctly implement 3D transpose following emClarity conventions +- **Solution**: X=fastest, Y=second, Z=slowest dimension ordering with row-major memory layout +- **Learning**: Always validate indexing patterns against NumPy/CuPy references for correctness + +**Challenge: CUDA Utility Function Architecture** +- **Issue**: Code duplication across multiple CUDA kernels for indexing operations +- **Solution**: Created comprehensive utility header with inline device functions +- **Learning**: Invest in reusable utilities early to maintain code quality and consistency + +### Class-Based Architecture Patterns (September 3, 2025) + +**Challenge: fourierTransformer.m Pattern Translation** +- **Issue**: Translating MATLAB object-oriented patterns to Python while maintaining efficiency +- **Solution**: Created PaddedArray class following established patterns: + - Persistent memory management with `use_once` parameter + - CPU/GPU switching methods (`to_cpu()`, `to_gpu()`) + - Direct array access via `get_stored_array_reference()` + - Memory monitoring with `get_memory_info()` +- **Learning**: Follow existing MATLAB patterns closely for consistency and user familiarity + +**Challenge: Memory Safety in Array References** +- **Issue**: Python array references can become invalid after operations like `zero_stored_array()` +- **Solution**: Clear documentation and examples about reference lifetime management +- **Learning**: Provide explicit warnings and usage examples for memory-unsafe operations + +**Challenge: Performance Optimization Strategy** +- **Issue**: Custom CUDA kernels sometimes performed worse than CuPy built-ins +- **Solution**: Use custom kernels only when necessary; leverage CuPy for standard operations +- **Learning**: Don't optimize prematurely - profile and compare against established libraries + +### Development Workflow Improvements + +**Communication Pattern: Incremental Testing** +- **Effective**: Breaking complex implementations into testable components +- **Example**: CUDA utilities → basic operations → complex class implementation +- **Learning**: Always validate each layer before building the next + +**Communication Pattern: Real-world Usage Examples** +- **Effective**: Creating practical examples following established patterns (fourierTransformer.m) +- **Example**: PaddedArray examples showing single-use vs persistent patterns +- **Learning**: Concrete usage examples clarify requirements better than abstract descriptions + +**Communication Pattern: Performance Validation** +- **Effective**: Benchmarking against existing implementations to validate improvements +- **Example**: 3.9x speedup for persistent PaddedArray vs original function +- **Learning**: Always include performance comparisons to justify architectural decisions + +### File Organization Best Practices + +**Pattern: Comprehensive Documentation Structure** +- **Effective**: Multiple documentation types for different audiences: + - `README.md`: Quick start and overview + - `README_utilities.md`: Detailed API reference + - `IMPLEMENTATION_SUMMARY.md`: Technical implementation details + - `*_examples.py`: Practical usage demonstrations +- **Learning**: Over-document rather than under-document for complex systems + +**Pattern: Test-Driven Development** +- **Effective**: Creating comprehensive test suites before finalizing implementation +- **Example**: 15+ test scenarios covering correctness, performance, memory management +- **Learning**: Extensive testing catches edge cases and validates design decisions + +### General Implementation Principles + +**Pattern: General-Purpose Solution Design** + +- **Effective**: Implementing algorithms that work for all valid inputs, not just test cases +- **Example**: Proper domain handling in FFT functions beyond the specific test dimensions +- **Learning**: Test cases validate correctness but shouldn't dictate implementation scope + +**Pattern: Parallel Operation Execution** + +- **Effective**: Invoking multiple independent tools simultaneously for efficiency +- **Example**: Gathering context from multiple files while processing algorithm requirements +- **Learning**: Batch operations when possible to reduce development time + +**Pattern: Robust Implementation Principles** + +- **Effective**: Writing code that handles edge cases and unusual inputs properly +- **Example**: Parameter validation with clear error messages before computation +- **Learning**: Always consider what could go wrong beyond the happy path + +These learnings should guide future conversion sessions to minimize iteration cycles and improve code quality. + +## Session-Specific Learnings (September 3, 2025) + +### Key Insights from BH_runAutoAlign → EMC_runAutoAlign Conversion + +**Effective Pattern: Function Rename + Integration Architecture Shift** + +**Challenge**: Converting MATLAB function that relied on external shell scripts (`emC_autoAlign`, `emC_findBeads`) +- **Initial Approach**: Direct translation maintaining external script dependencies +- **Evolution**: Integrated shell script functionality directly into Python implementation +- **Final Result**: Simplified interface with better error handling and no external dependencies + +**Key Decision Points**: +1. **External vs Integrated**: Initially kept external script paths, then realized integration would provide better user experience +2. **Function Signature Evolution**: Removed `run_path` and `find_beads_path` parameters, simplifying the interface +3. **Shell Script Translation**: Implemented `_run_integrated_patch_tracking()` and `_run_integrated_bead_finding()` as native Python methods + +**Learning**: When converting functions with external dependencies, consider whether those dependencies can be integrated for a cleaner Python API. + +--- + +**Communication Pattern: Incremental Refactoring with User Input** + +**Effective Cycle**: +1. **Initial Conversion**: Direct MATLAB-to-Python translation maintaining original architecture +2. **User Feedback**: "Rather than have this separate functionality, let's make them methods" +3. **Architecture Shift**: Integrated external shell scripts as Python methods +4. **Naming Convention**: Applied consistent `emc_` prefix throughout + +**Impact**: This pattern caught a major architectural improvement opportunity that would have been missed with a pure translation approach. + +**Learning**: Always present initial conversion for user review before finalizing - users often have insights about better integration patterns. + +--- + +**Technical Challenge: Complex Shell Script Integration** + +**Problem**: The `emC_autoAlign` shell script contained complex bash logic with: +- Multi-level binning loops with mathematical calculations +- Conditional iteration logic based on alignment quality +- External tool coordination (newstack, tiltxcorr, tiltalign, etc.) +- File management and cleanup + +**Solution Strategy**: +1. **Analyze shell script structure**: Identified main loops and decision points +2. **Translate bash constructs**: Converted shell math and loops to Python equivalents +3. **Subprocess management**: Maintained calls to IMOD tools with proper error handling +4. **Helper function decomposition**: Split complex logic into `_run_first_iteration_alignment()`, `_run_subsequent_iteration_alignment()`, etc. + +**Key Code Pattern**: +```python +# Bash: for iBin in $(seq $binHigh $binInc $binLow) +# Python: +bin_sequence = list(range(binning_params['bin_high'], + binning_params['bin_low'] + binning_params['bin_inc'], + binning_params['bin_inc'])) +for i_bin in bin_sequence: + # Process each binning level +``` + +**Learning**: Complex shell scripts can be successfully translated to Python with careful analysis of control flow and proper helper function decomposition. + +--- + +**File Management Anti-Pattern: Temporary File Proliferation** + +**Problem Observed**: Throughout the session, multiple temporary files were created in project directories: +- Test files in root directory (`test_*.py`, `test_*.m`) +- Demo data in `python/metaData/` (`demo_*.py`, `*.png`, `*.csv`) +- Star file test data (`test_star_data/` directories) + +**Solution Implemented**: +- **Cleanup Rule**: All temporary files must go in `/tmp/copilot-test/` +- **Documentation**: Added rule to both instruction files +- **Retroactive Cleanup**: Removed all temporary files from project directories + +**Learning**: Establish temporary file management rules early in development to prevent project bloat and ensure clean git history. + +--- + +**Testing Strategy: Comprehensive Validation with Renamed Functions** + +**Challenge**: After renaming functions and changing signatures, all references needed updating: +- Test files importing old function names +- Command line interface changes +- Documentation updates + +**Effective Approach**: +1. **Update function definition first** +2. **Update imports systematically** +3. **Test immediately after each change** +4. **Update CLI and help text** +5. **Validate with comprehensive test suite** + +**Key Success**: The test suite caught all reference errors immediately, preventing runtime failures. + +**Learning**: When making breaking changes like function renames, update and test incrementally rather than changing everything at once. + +--- + +**Documentation Pattern: Multiple Documentation Types for Complex Changes** + +**Effective Pattern**: Created multiple documentation artifacts: +- `FUNCTION_RENAME_NOTES.md`: Specific to the naming convention change +- `EMC_INTEGRATION_COMPLETE.md`: Comprehensive implementation summary +- `CONVERSION_COMPLETE.md`: Original completion documentation +- Updated `README.md` files: User-facing documentation + +**Benefit**: Different audiences (users, developers, future AI sessions) each get appropriate level of detail. + +**Learning**: For significant architectural changes, create multiple documentation types rather than trying to fit everything in one document. + +--- + +## Session-Specific Learnings (September 4, 2025) + +### CUDA Memory Layout Debugging: From Confusion to Clarity + +**Challenge**: 2D transpose operation producing scrambled output instead of correct transposition + +**Problem Diagnosis Cycle**: +1. **Initial Symptoms**: Array elements appearing in wrong positions after transpose +2. **First Investigation**: Suspected kernel indexing logic errors +3. **Memory Layout Realization**: Discovered mixing of Fortran-contiguous vs C-contiguous arrays +4. **Architecture Decision**: Standardized on C-contiguous arrays for new CUDA operations +5. **API Unification**: Consolidated multiple indexing functions into overloaded `get_linear_index()` + +**Key Technical Insights**: +- **Memory Layout Consistency**: Mixing F-order and C-order arrays creates subtle bugs that are hard to debug +- **Kernel Parameter Order**: Function signature must match call-site parameter order exactly +- **Indexing Unification**: Single overloaded function (`get_linear_index`) reduces errors vs multiple named functions + +**Effective Debugging Pattern**: +```python +# Instead of separate functions: +# index_2d(), index_3d(), index_2d_fortran() +# Use single overloaded function: +get_linear_index(int2 coords, int nx) # 2D +get_linear_index(int3 coords, int2 dims) # 3D +``` + +**Learning**: When debugging CUDA indexing issues, verify memory layout consistency first before investigating algorithmic logic. + +--- + +**Communication Pattern: Iterative Refinement with Clear Decision Points** + +**Effective Cycle Observed**: +1. **Initial Problem**: "2D transpose giving incorrect results" +2. **Multiple Hypothesis Testing**: Fortran vs C layouts, kernel logic, parameter order +3. **Decisive Pivot**: "Let's standardize on C-contiguous for simplicity" +4. **Systematic Refactor**: API unification, documentation updates, test validation +5. **Verification**: Full test suite execution confirming fix + +**Communication Success**: User provided clear direction when multiple approaches were explored ("go back to C-contiguous"), preventing extended exploration of dead ends. + +**Learning**: When facing complex technical issues, test multiple hypotheses but establish clear decision criteria to avoid analysis paralysis. + +--- + +**Error Diagnostics Enhancement Pattern** + +**Challenge**: Cryptic error messages from `ensure_c()` function made debugging difficult + +**Evolution**: +1. **Original**: Generic "copying required" message +2. **Enhancement Request**: "Can we make ensure_c print the line number?" +3. **Implementation**: Added caller introspection using `inspect` module +4. **Result**: Rich error messages with file:line:function context + +**Technical Solution**: +```python +# Before: RuntimeError: ensure_c: input was not C-contiguous... +# After: RuntimeError: ensure_c: input was not C-contiguous... | at /path/file.py:166 in add_arrays +``` + +**Learning**: When debugging tools produce unclear errors, enhance diagnostics immediately rather than working around them - the time investment pays off quickly. + +--- + +**Test-Driven Validation Strategy** + +**Effective Pattern**: +1. **Isolated Test First**: Ran single transpose test to verify core fix +2. **Full Suite Second**: Ran complete test suite to catch integration issues +3. **Systematic Fix**: Addressed test failures by allowing copies where appropriate +4. **Final Validation**: Re-ran full suite to confirm no regressions + +**Key Success**: Isolated testing caught the core issue quickly, while full testing revealed integration problems that needed different solutions. + +**Learning**: Use both focused and comprehensive testing - focused tests for debugging specific issues, comprehensive tests for validating system integration. + +--- + +**Dependency Management Learning** + +**Challenge**: Missing dependencies (CuPy, fastrlock, psutil, joblib) caused setup issues + +**Solution Pattern**: +1. **Detection**: Found missing deps through virtual environment analysis +2. **Root Cause**: Setup scripts incomplete for CUDA workflow requirements +3. **Comprehensive Fix**: Updated both setup-dev.sh and pyproject.toml +4. **Validation**: Verified dependencies align with actual usage patterns + +**Key Insight**: CUDA workflows have specific dependency requirements (fastrlock for CuPy performance, psutil for memory monitoring) that aren't obvious from basic testing. + +**Learning**: Regularly audit actual vs declared dependencies, especially for GPU computing stacks where performance dependencies matter. + +--- + +**Documentation Enhancement During Development** + +**Effective Pattern**: Updated multiple documentation files in parallel with code changes: +- `MAIN_CONTEXT.md`: Added implementation principles and parallel operation guidance +- `python_conversion_instructions.md`: Added general-purpose solution patterns +- README files: Updated API references for unified indexing functions + +**Success**: Documentation updates prevented future confusion about design decisions and coding patterns. + +**Learning**: Document architectural decisions immediately while the rationale is fresh, rather than deferring to later cleanup phases. + +```` diff --git a/docs/emClarity-tutorial-V1-5-3-10.pdf b/docs/emClarity-tutorial-V1-5-3-10.pdf new file mode 100644 index 00000000..21032f1c Binary files /dev/null and b/docs/emClarity-tutorial-V1-5-3-10.pdf differ diff --git a/docs/exampleParametersAndRunScript/runTutorial.sh b/docs/exampleParametersAndRunScript/runTutorial.sh deleted file mode 100755 index 7e66e71e..00000000 --- a/docs/exampleParametersAndRunScript/runTutorial.sh +++ /dev/null @@ -1,96 +0,0 @@ -skipThis=1 -runThis=1 - -# -# - - - -if [[ ${skipThis} -eq 0 ]] ; then - exit - -fi # - emClarity init param0.m ; [[ $? -ne 0 ]] && exit - - emClarity ctf update param0.m ; [[ $? -ne 0 ]] && exit - - emClarity ctf 3d param0.m ; [[ $? -ne 0 ]] && exit - - - - for i in 0 1 2 ; do - if [[ $i -eq 0 ]] ; then - ST='NoAlignment' - else - ST='RawAlignment' - fi - - if [[ $i -ge 0 ]] ; then emClarity avg param${i}.m ${i} $ST ; fi; [[ $? -ne 0 ]] && exit - - if [[ $i -ge 0 ]] ; then emClarity alignRaw param${i}.m ${i}; fi; [[ $? -ne 0 ]] && exit - - done - - - - emClarity removeDuplicates param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity tomoCPR param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity ctf update param$((${i}+1)).m ; [[ $? -ne 0 ]] && exit - - emClarity ctf 3d param$((${i}+1)).m; [[ $? -ne 0 ]] && exit - - - - for i in 3 4 5 ; do - - if [[ $i -ge 3 ]] ; then emClarity avg param${i}.m ${i} RawAlignment ; fi; [[ $? -ne 0 ]] && exit - - if [[ $i -ge 3 ]] ; then emClarity alignRaw param${i}.m ${i} ; fi; [[ $? -ne 0 ]] && exit - - done - - - - emClarity removeDuplicates param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity tomoCPR param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity ctf update param$((${i}+1)).m ; [[ $? -ne 0 ]] && exit - - emClarity ctf 3d param$((${i}+1)).m; [[ $? -ne 0 ]] && exit - - - - - for i in 6 7 8 ; do - if [[ $i -ge 6 ]] ; then emClarity avg param${i}.m ${i} RawAlignment ; fi; [[ $? -ne 0 ]] && exit - - if [[ $i -ge 6 ]] ; then emClarity alignRaw param${i}.m ${i} ; fi; [[ $? -ne 0 ]] && exit - - done - - emClarity removeDuplicates param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity tomoCPR param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity ctf update param$((${i}+1)).m ; [[ $? -ne 0 ]] && exit - - emClarity ctf 3d param$((${i}+1)).m ; [[ $? -ne 0 ]] && exit - - - - for i in 9 10 11 12 ; do - - if [[ $i -ge 9 ]] ; then emClarity avg param${i}.m ${i} RawAlignment ; fi; [[ $? -ne 0 ]] && exit - - if [[ $i -ge 9 ]] ; then emClarity alignRaw param${i}.m ${i} ; fi; [[ $? -ne 0 ]] && exit - - done - - - emClarity avg param13.m 13 RawAlignment; [[ $? -ne 0 ]] && exit - emClarity avg param13.m 13 FinalAlignment; [[ $? -ne 0 ]] && exit - # fi # end of skipThis -#fi # end of runTHis diff --git a/docs/gen_param.m b/docs/gen_param.m new file mode 100644 index 00000000..608d1056 --- /dev/null +++ b/docs/gen_param.m @@ -0,0 +1,275 @@ +% This is a comment +% Inline comments will break the parser. + + +% String to name the structure that contains all of the metadata, projectName +subTomoMeta=full_enchilada_2_1_branch_5 + +save_mapback_classes=1 +tomoCPR_n_particles_minimum=1 + +measure_noise_variance=0 + +fastScratchDisk=ram + +nGPUs=4 +nCpuCores=16 +n_tilt_workers=4 + +refine_defocus_cisTEM=0 +rerun_refinement_cisTEM=0 + +phakePhasePlate=0 +flgQualityWeight=0 + +flgMultiRefAlignment=0 +updateClassByBestReferenceScore=0 + +max_ctf3dDepth=100e-9 +% Do not whiten (1), but apply ctf weiner filter (3) with additive term +whitenPS=[0,0,0.5] +diameter_fraction_for_local_stats=0.9 +test_local=1 +scale_mip=0 + + +nPeaks=1 +symmetry=C12 +doHelical=0 + +Pca_refineKmeans=1 + +% if > 1 use this many subtomos in the avg +% if < 1 use this fraction in the avg +%ccc_cutoff=0.6 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%% Mask parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% The particle radius in x,y,z Angstrom, smallest value to contain particle. +% For particles in a lattice, neighboring particles can be used in alignment +% by specifying a larger mask size, but this paramter must correspond to your +% target, a cetral hexamer of capsid proteins for example. + +% For TM +particleRadius=[180,180,150] +particleMass=3.2 + +Ali_mType=cylinder +Cls_mType=cylinder + + +% For special cases where repeated motifs are present which might cause one +% subtomo to drift to a neighbor. This allows a larger alignment mask to be used +% for the rotational search (Ali_m...) but limits the translational peak search. +Peak_mType=cylinder +%Peak_mRadius=[210,210,320] +Peak_mRadius=[40,40,40] +% mask radius and center - and center in Angstrom. Mask size is determined +% large enough to contain delocalized signal, proper apodization, and to +% avoid wraparound error in cross-correlation. +% mask radius and center - and center in Angstrom. Mask size is determined +% large enough to contain delocalized signal, proper apodization, and to +% avoid wraparound error in cross-correlation. +Ali_mRadius=[220,220,164] +%Ali_mRadius=[220,220,200] +Ali_mCenter=[0,0,0] +Cls_mRadius=[220,220,164] +Cls_mCenter=[ 0,0,0 ] + + +% Sampling rate +Ali_samplingRate=3 +Cls_samplingRate=3 + +move_reference_by_com=0 + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%% Tomo-constrained projection refinement parameters %%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +% I advise to avoid using this experimental feature for now. +tomoCprDefocusRefine=0 +tomoCprDefocusRange=500e-9; +tomoCprDefocusStep=20e-9; + +% By default the patch size is calculated based on the number of available fiducials and the +% mass. To limit the number of local areas, set this to something other than zero. +tomoCPR_target_n_patches_x_y=[3,4] +tomoCPR_random_subset=0 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% subTomogram alignment %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +Raw_className=0 +% Second row specifies C1 symmetry +Raw_classes_odd=[0;12.*ones(1,1)] +Raw_classes_eve=[0;12.*ones(1,1)] + +% replicate the in plane angles at each (CX) symmetry position +symmetry_constrained_search=0 +Raw_angleSearch=[0,0,180,3] +print_alignment_stats=1 +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% Template matching parameters %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +Tmp_bandpass=[0.01,1200,25] +Tmp_samplingRate=5 +Tmp_threshold=1500 +Tmp_angleSearch=[180,12,180,12] + +Tmp_targetSize=[512,512,768] + +Tmp_half_precision=0 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%% Class reference %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + + +Cls_className=49 +Cls_classes_odd=[1:64;12.*ones(1,64)] +Cls_classes_eve=[1:64;12.*ones(1,64)] + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%% FSC Paramters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% On/Off anisotropic SSNR calc +flgCones=0 +fsc_shape_mask=0 +% B-factor applied to weighted averages and refs. Should be < 20. Can be a vector +% where the 2:end positions generate independent maps at that sharpening +% when avg paramN.m N FinalAlignment is run. + +Fsc_bfactor=10 + +% For very tightly packed subTomos set to 1 to avoid mixing halfsets +% form overlaping peripheral density. fscGoldSplitOnTomos=0 +fscGoldSplitOnTomos=0 +% Default to doing an alignment between class halfs before calculating FSC +% This should be deprecated as the halves converge to each other. +fscWithChimera=0 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% Classification Paramters %%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% Constrain to the asymmetric unit +% Only for Cx and cylinder masks right now. +Pca_constrain_symmetry=1 +flgPcaShapeMask=0 +% On/Off classification. This must be on when "avg paramN.m N RawAlignment" +% is run at the begining of a cycle where classification is to be run. +flgClassify=1 + +% List of different cluster sizes to try, eg [3;4] +Pca_clusters=[49,64] + +% Maximum number of eigenvalues/vectors to save +Pca_maxEigs=64 + +% Different resolution bands to run PCA on. Not all need to be used for subsequent +% clustering. (Angstrom) + +pcaScaleSpace=[21,42,84]; + + +% Different ranges of coefficients to use in the clustering. At times, the +% missing wedge can be a strong feature, such that ignoring the first few +% eigen values can be usefule. [2:40 ; 6;40 ; 10:40] +% Each row must have the same number of entries, and there must be a row +% for each scale space, even if it is all zeros. + +% NOTE: if using multi_refalignment, this must match the number of references +Pca_coeffs=[3:48;3:48;3:48;3:48]; +Pca_bandpass=[0.01,1200,20]; + + + + +% The number of subtomos to process at once before pulling tempDataMatrix off +% the gpu and into main memory. +PcaGpuPull=5000 +Pca_randSubset=0 + + + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%% Parameters for CTF all si (meters, volts)%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + + +%%%%%%%%%% Microscope parameters %%%%%%%%%% + +% Of the data saved in fixed stacks - MUST match header +PIXEL_SIZE=2.50e-10 +% Currently any super-resolution data is cropped in Fourier Space after alignment +% allowing for finer sampling when interpolating the stacks, while then +% filtering out noise due to aliasing. +SuperResolution=0 +% Spherical abberation +Cs=2.7e-3 +% Accelerating voltage +VOLTAGE=300e3 +% Percent amplitude contrast +AMPCONT=0.04 + +% search range - generally safe to test a wide range +defEstimate=3.5e-6 +defWindow=1.75e-6 +% The PS is considered from the lower resolution inflection point +% past the first zero to this cutoff resolution +defCutOff=6e-10 + +% Total dose in electron/A^2, assumed constant rate +CUM_e_DOSE=180 +% Gold fiducial diameter +beadDiameter=10e-9 + + +oneOverCosineDose=0 +startingAngle=0 +startingDirection=pos +doseSymmetricIncrement=3 +% The reported value is 4.86 but I'm scaling this down to 60% of that +doseAtMinTilt=2.9 + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%% Advanced Parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% default is 1.5, may need to shrink for large objects, may need to increase for periodic objects. +scaleCalcSize=1.3 + +autoAli_min_sampling_rate=10 +autoAli_max_sampling_rate=4 +autoAli_patch_size_factor=4 +autoAli_patch_overlap=0.6 +autoAli_max_resolution=18 +autoAli_refine_on_beads=0 +autoAli_iterations_per_bin=2 +autoAli_patch_tracking_border=64 +autoAli_n_iters_no_rotation=2 +autoAli_max_shift_in_angstroms=300 +autoAli_max_shift_factor=1 + +ctf_tile_overlap=4 + + + diff --git a/docs/gui-testing-guide.md b/docs/gui-testing-guide.md new file mode 100644 index 00000000..95cea028 --- /dev/null +++ b/docs/gui-testing-guide.md @@ -0,0 +1,265 @@ +# emClarity GUI Testing Guide + +**Phase 0 Status**: All 16 tasks complete | 35 E2E tests passing | 27 backend unit tests passing + +--- + +## Quick Start + +You need **two terminals** — one for the backend, one for the frontend. + +### Terminal 1: Backend (FastAPI) + +```bash +cd /workspaces/cisTEMx +pip install -r backend/requirements.txt # first time only +python -m uvicorn backend.main:app --reload --port 8000 +``` + +Verify: `curl http://localhost:8000/api/health` should return `{"status":"ok","service":"emClarity backend"}` + +### Terminal 2: Frontend (React + Vite) + +```bash +cd /workspaces/cisTEMx/frontend +npm install # first time only +npm run dev +``` + +Open **http://localhost:5173** in your browser. + +--- + +## What to Test: Page-by-Page Walkthrough + +### 1. Project Page (`/`) + +The landing page. Create and manage emClarity projects. + +| What to check | Expected behavior | +|---|---| +| "New Project" form renders | Name field, directory picker, create button | +| Create a project | Fill in name + directory path, submit → project appears in list | +| Project state badge | Shows `UNINITIALIZED` for new projects | +| Select a project | Clicking a project sets it as the active project (shown in header) | + +**API calls to watch**: `POST /api/v1/projects`, `GET /api/v1/projects/{id}` + +### 2. Parameters Page (`/parameters`) + +Edit the 160+ emClarity processing parameters. + +| What to check | Expected behavior | +|---|---| +| Tabs render by category | Microscope, Hardware, CTF, Alignment, Classification, etc. | +| Parameter form fields | Each param shows name, input control, default value, description | +| Type-appropriate inputs | Numeric params get number inputs, booleans get toggles, strings get text | +| Validation feedback | Out-of-range values show error messages | +| Required params marked | Required parameters visually distinguished | + +**API calls to watch**: `GET /api/v1/parameters/schema`, `POST /api/v1/parameters/validate` + +### 3. Tilt-Series Page (`/tilt-series`) + +Sortable/filterable data table for managing tilt-series. + +| What to check | Expected behavior | +|---|---| +| Table renders | Columns for name, status, defocus, tilt range, etc. | +| Column sorting | Click headers to sort ascending/descending | +| Column filtering | Filter controls narrow displayed rows | +| Status badges | Color-coded status indicators per tilt-series | + +**API calls to watch**: `GET /api/v1/projects/{id}/tilt-series` + +### 4. Workflow Page (`/workflow`) + +Visual pipeline stepper showing the emClarity processing pipeline. + +| What to check | Expected behavior | +|---|---| +| State machine visualization | Shows pipeline stages as a stepper/flowchart | +| Current state highlighted | Active state visually distinct | +| Available commands | Only valid next commands are enabled | +| Disabled commands | Out-of-order commands are grayed out | +| Run command | Clicking an available command submits a job | + +**Pipeline stages** (in order): +``` +UNINITIALIZED → TILT_ALIGNED → CTF_ESTIMATED → RECONSTRUCTED → +PARTICLES_PICKED → INITIALIZED → CYCLE_0_AVG → CYCLE_N_ALIGNED → +CYCLE_N_AVG → PROCESSING → EXPORT → DONE +``` + +**API calls to watch**: `GET /api/v1/workflow/state-machine`, `GET /api/v1/workflow/{project_id}/available-commands`, `POST /api/v1/workflow/{project_id}/run` + +### 5. Jobs Page (`/jobs`) + +Monitor running and completed jobs. + +| What to check | Expected behavior | +|---|---| +| Job list renders | Shows all jobs with status badges | +| Status badges | PENDING (gray), RUNNING (blue), COMPLETED (green), FAILED (red), CANCELLED (yellow) | +| Job details | Click a job to see command, timestamps, log output | +| Log viewer | Displays job log content | +| Cancel button | Running jobs show a cancel button | +| Sort order | Newest jobs appear first | + +**API calls to watch**: `GET /api/v1/jobs`, `GET /api/v1/jobs/{id}`, `GET /api/v1/jobs/{id}/log`, `DELETE /api/v1/jobs/{id}` + +### 6. Results Page (`/results`) + +View reconstruction quality metrics. + +| What to check | Expected behavior | +|---|---| +| FSC curves | Fourier Shell Correlation plot renders (recharts) | +| 0.143 threshold line | Resolution threshold displayed on FSC plot | +| Particle statistics | Particle count, distribution info | +| System info panel | CPU cores, RAM, GPU info displayed | + +### 7. Utilities Page (`/utilities`) + +System diagnostics and helper operations. + +| What to check | Expected behavior | +|---|---| +| System check | Runs `emClarity check` and displays output | +| Mask creator | Form for mask creation parameters | +| Volume rescaler | Input for target pixel size | +| Geometry operations | Dropdown with operations: RemoveClasses, RemoveFraction, RemoveLowScoringParticles, RestoreParticles, PrintGeometry | + +**API calls to watch**: `POST /api/v1/utilities/check`, `POST /api/v1/utilities/mask`, `POST /api/v1/utilities/rescale`, `POST /api/v1/utilities/geometry` + +### 8. Layout & Navigation (all pages) + +| What to check | Expected behavior | +|---|---| +| Sidebar navigation | Links to all 7 pages, highlights active page | +| Header | Shows active project name and state badge | +| Responsive layout | Sidebar collapses on narrow screens | +| Settings panel | Accessible from header or sidebar | +| Error boundary | Broken components show error UI, not white screen | +| Loading states | Spinners shown while API calls are in flight | + +--- + +## Running Automated Tests + +### E2E Tests (require backend running on port 8000) + +```bash +cd /workspaces/cisTEMx + +# Run all 35 tests +python -m pytest tests/ -v + +# Run specific test file +python -m pytest tests/test_parameter_schema.py -v +python -m pytest tests/test_parameter_validation.py -v +python -m pytest tests/test_project_management.py -v +python -m pytest tests/test_workflow_state.py -v +python -m pytest tests/test_system_info.py -v +python -m pytest tests/test_job_management.py -v +``` + +### Backend Unit Tests (no server needed) + +```bash +cd /workspaces/cisTEMx + +# Run all 27 tests +python -m pytest backend/tests/ -v + +# With coverage +python -m pytest backend/tests/ --cov=backend --cov-report=term-missing +``` + +### Frontend Tests + +```bash +cd /workspaces/cisTEMx/frontend + +# Run once +npm run test + +# Watch mode (re-runs on file change) +npm run test:watch + +# TypeScript compilation check +npm run typecheck + +# Lint +npm run lint +``` + +--- + +## API Quick Reference + +Base URL: `http://localhost:8000` + +| Endpoint | Method | Purpose | +|---|---|---| +| `/api/health` | GET | Health check | +| `/api/v1/parameters/schema` | GET | All 160 parameter definitions | +| `/api/v1/parameters/validate` | POST | Validate parameter values | +| `/api/v1/parameters/file/{path}` | GET | Parse a MATLAB param.m file | +| `/api/v1/parameters/file` | POST | Write a MATLAB param.m file | +| `/api/v1/projects` | POST | Create new project | +| `/api/v1/projects/{id}` | GET | Get project details | +| `/api/v1/projects/{id}` | DELETE | Delete project | +| `/api/v1/projects/{id}/tilt-series` | GET | List tilt-series for project | +| `/api/v1/workflow/state-machine` | GET | Full state machine definition | +| `/api/v1/workflow/{project_id}/available-commands` | GET | Commands available in current state | +| `/api/v1/workflow/{project_id}/run` | POST | Execute a pipeline command | +| `/api/v1/jobs` | GET | List all jobs | +| `/api/v1/jobs/{id}` | GET | Job status and details | +| `/api/v1/jobs/{id}/log` | GET | Job log output | +| `/api/v1/jobs/{id}` | DELETE | Cancel a running job | +| `/api/system/info` | GET | CPU, RAM, GPU info | +| `/api/v1/utilities/check` | POST | Run emClarity system check | +| `/api/v1/utilities/mask` | POST | Create mask | +| `/api/v1/utilities/rescale` | POST | Rescale volume | +| `/api/v1/utilities/geometry` | POST | Geometry operations | + +FastAPI auto-docs available at: **http://localhost:8000/docs** (Swagger UI) and **http://localhost:8000/redoc** + +--- + +## Troubleshooting + +| Problem | Fix | +|---|---| +| Frontend shows network errors | Confirm backend is running on port 8000 | +| CORS errors in browser console | Backend CORS is configured for `localhost:5173` — make sure frontend is on that port | +| `npm install` fails | Delete `frontend/node_modules` and `frontend/package-lock.json`, retry | +| `pip install` fails | Try `pip install --upgrade pip` first, then retry | +| E2E tests fail with connection refused | Start the backend server before running E2E tests | +| TypeScript errors | Run `npm run typecheck` to see specific errors | +| Blank page in browser | Check browser console (F12) for JavaScript errors | +| Port 8000 already in use | `lsof -i :8000` to find the process, or use `--port 8001` | +| Port 5173 already in use | Vite will auto-increment to 5174; update CORS in `backend/main.py` if needed | + +--- + +## Tech Stack Summary + +| Layer | Technology | Version | +|---|---|---| +| Frontend framework | React | 19 | +| Language | TypeScript | 5.9 | +| Build tool | Vite | 8 | +| Styling | Tailwind CSS | 4 | +| Routing | React Router DOM | 7 | +| Server state | TanStack React Query | 5 | +| Data tables | TanStack React Table | 8 | +| Forms | React Hook Form + Zod | 7 / 4 | +| Charts | Recharts | 3 | +| Icons | Lucide React | latest | +| Backend framework | FastAPI | 0.100+ | +| Data validation | Pydantic | 2.0+ | +| ASGI server | Uvicorn | 0.23+ | +| Frontend tests | Vitest + Testing Library | 4 / 16 | +| Backend tests | pytest | latest | diff --git a/docs/param_aug2025.m b/docs/param_aug2025.m new file mode 100644 index 00000000..f6e82a6e --- /dev/null +++ b/docs/param_aug2025.m @@ -0,0 +1,275 @@ +% This is a comment +% Inline comments will break the parser. + + +% String to name the structure that contains all of the metadata, projectName +subTomoMeta=full_enchilada_2_1_branch_10 + +save_mapback_classes=1 +tomoCPR_n_particles_minimum=1 + +measure_noise_variance=0 + +fastScratchDisk=ram + +nGPUs=4 +nCpuCores=16 +n_tilt_workers=4 + +refine_defocus_cisTEM=0 +rerun_refinement_cisTEM=0 + +phakePhasePlate=0 +flgQualityWeight=0 + +flgMultiRefAlignment=0 +updateClassByBestReferenceScore=0 + +max_ctf3dDepth=100e-9 +% Do not whiten (1), but apply ctf weiner filter (3) with additive term +whitenPS=[0,0,0.5] +diameter_fraction_for_local_stats=0.9 +test_local=1 +scale_mip=0 + + +nPeaks=1 +symmetry=C12 +doHelical=0 + +Pca_refineKmeans=1 + +% if > 1 use this many subtomos in the avg +% if < 1 use this fraction in the avg +%ccc_cutoff=0.6 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%% Mask parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% The particle radius in x,y,z Angstrom, smallest value to contain particle. +% For particles in a lattice, neighboring particles can be used in alignment +% by specifying a larger mask size, but this paramter must correspond to your +% target, a cetral hexamer of capsid proteins for example. + +% For TM +particleRadius=[180,180,150] +particleMass=3.2 + +Ali_mType=cylinder +Cls_mType=cylinder + + +% For special cases where repeated motifs are present which might cause one +% subtomo to drift to a neighbor. This allows a larger alignment mask to be used +% for the rotational search (Ali_m...) but limits the translational peak search. +Peak_mType=cylinder +%Peak_mRadius=[210,210,320] +Peak_mRadius=[40,40,40] +% mask radius and center - and center in Angstrom. Mask size is determined +% large enough to contain delocalized signal, proper apodization, and to +% avoid wraparound error in cross-correlation. +% mask radius and center - and center in Angstrom. Mask size is determined +% large enough to contain delocalized signal, proper apodization, and to +% avoid wraparound error in cross-correlation. +Ali_mRadius=[220,220,164] +%Ali_mRadius=[220,220,200] +Ali_mCenter=[0,0,0] +Cls_mRadius=[220,220,164] +Cls_mCenter=[ 0,0,0 ] + + +% Sampling rate +Ali_samplingRate=3 +Cls_samplingRate=3 + +move_reference_by_com=0 + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%% Tomo-constrained projection refinement parameters %%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +% I advise to avoid using this experimental feature for now. +tomoCprDefocusRefine=0 +tomoCprDefocusRange=500e-9; +tomoCprDefocusStep=20e-9; + +% By default the patch size is calculated based on the number of available fiducials and the +% mass. To limit the number of local areas, set this to something other than zero. +tomoCPR_target_n_patches_x_y=[3,4] +tomoCPR_random_subset=400 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% subTomogram alignment %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +Raw_className=0 +% Second row specifies C1 symmetry +Raw_classes_odd=[0;12.*ones(1,1)] +Raw_classes_eve=[0;12.*ones(1,1)] + +% replicate the in plane angles at each (CX) symmetry position +symmetry_constrained_search=0 +Raw_angleSearch=[0,0,3,3] +print_alignment_stats=1 +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% Template matching parameters %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +Tmp_bandpass=[0.01,1200,25] +Tmp_samplingRate=5 +Tmp_threshold=1500 +Tmp_angleSearch=[180,12,180,12] + +Tmp_targetSize=[512,512,768] + +Tmp_half_precision=0 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%% Class reference %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + + +Cls_className=9 +Cls_classes_odd=[1:9;12.*ones(1,9)] +Cls_classes_eve=[1:9;12.*ones(1,9)] + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%% FSC Paramters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% On/Off anisotropic SSNR calc +flgCones=0 +fsc_shape_mask=1 +% B-factor applied to weighted averages and refs. Should be < 20. Can be a vector +% where the 2:end positions generate independent maps at that sharpening +% when avg paramN.m N FinalAlignment is run. + +Fsc_bfactor=10 + +% For very tightly packed subTomos set to 1 to avoid mixing halfsets +% form overlaping peripheral density. fscGoldSplitOnTomos=0 +fscGoldSplitOnTomos=0 +% Default to doing an alignment between class halfs before calculating FSC +% This should be deprecated as the halves converge to each other. +fscWithChimera=0 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% Classification Paramters %%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% Constrain to the asymmetric unit +% Only for Cx and cylinder masks right now. +Pca_constrain_symmetry=1 +flgPcaShapeMask=0 +% On/Off classification. This must be on when "avg paramN.m N RawAlignment" +% is run at the begining of a cycle where classification is to be run. +flgClassify=0 + +% List of different cluster sizes to try, eg [3;4] +Pca_clusters=[9,16] + +% Maximum number of eigenvalues/vectors to save +Pca_maxEigs=64 + +% Different resolution bands to run PCA on. Not all need to be used for subsequent +% clustering. (Angstrom) + +pcaScaleSpace=[18,32,64] + + +% Different ranges of coefficients to use in the clustering. At times, the +% missing wedge can be a strong feature, such that ignoring the first few +% eigen values can be usefule. [2:40 ; 6;40 ; 10:40] +% Each row must have the same number of entries, and there must be a row +% for each scale space, even if it is all zeros. + +% NOTE: if using multi_refalignment, this must match the number of references +Pca_coeffs=[3:48;3:48;3:48;3:48]; +Pca_bandpass=[0.01,1200,18] + + + + +% The number of subtomos to process at once before pulling tempDataMatrix off +% the gpu and into main memory. +PcaGpuPull=5000 +Pca_randSubset=0 + + + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%% Parameters for CTF all si (meters, volts)%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + + +%%%%%%%%%% Microscope parameters %%%%%%%%%% + +% Of the data saved in fixed stacks - MUST match header +PIXEL_SIZE=2.50e-10 +% Currently any super-resolution data is cropped in Fourier Space after alignment +% allowing for finer sampling when interpolating the stacks, while then +% filtering out noise due to aliasing. +SuperResolution=0 +% Spherical abberation +Cs=2.7e-3 +% Accelerating voltage +VOLTAGE=300e3 +% Percent amplitude contrast +AMPCONT=0.04 + +% search range - generally safe to test a wide range +defEstimate=3.5e-6 +defWindow=1.75e-6 +% The PS is considered from the lower resolution inflection point +% past the first zero to this cutoff resolution +defCutOff=6e-10 + +% Total dose in electron/A^2, assumed constant rate +CUM_e_DOSE=180 +% Gold fiducial diameter +beadDiameter=10e-9 + + +oneOverCosineDose=0 +startingAngle=0 +startingDirection=pos +doseSymmetricIncrement=3 +% The reported value is 4.86 but I'm scaling this down to 60% of that +doseAtMinTilt=2.9 + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%% Advanced Parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% default is 1.5, may need to shrink for large objects, may need to increase for periodic objects. +scaleCalcSize=1.3 + +autoAli_min_sampling_rate=10 +autoAli_max_sampling_rate=4 +autoAli_patch_size_factor=4 +autoAli_patch_overlap=0.6 +autoAli_max_resolution=18 +autoAli_refine_on_beads=0 +autoAli_iterations_per_bin=2 +autoAli_patch_tracking_border=64 +autoAli_n_iters_no_rotation=2 +autoAli_max_shift_in_angstroms=300 +autoAli_max_shift_factor=1 + +ctf_tile_overlap=4 + + + diff --git a/emClarity_Tutorial.md b/emClarity_Tutorial.md new file mode 100644 index 00000000..6c1b0714 --- /dev/null +++ b/emClarity_Tutorial.md @@ -0,0 +1,580 @@ +# emClarity v1.5.3.10 Tutorial + +## Table of Contents + +1. [How to use this guide](#how-to-use-this-guide) +2. [The project directory](#the-project-directory) +3. [Get your data ready](#get-your-data-ready) +4. [Workflow](#workflow) +5. [Initial tilt-series alignment](#initial-tilt-series-alignment) +6. [Defocus estimate](#defocus-estimate) +7. [Select the sub-regions](#select-the-sub-regions) +8. [Picking](#picking) +9. [Initialize the project](#initialize-the-project) +10. [CTF 3D](#ctf-3d) +11. [Averaging](#averaging) +12. [Alignment](#alignment) +13. [TomoCPR](#tomocpr) +14. [Classification](#classification) +15. [Final map](#final-map) +16. [Algorithms](#algorithms) + +--- + +## How to use this guide + +### Run the jobs + +Our main objective in writing this tutorial is to help you get started using **emClarity** for processing sub-tomogram data as quickly as possible. To begin, we will not introduce all of the methods that **emClarity** puts at your disposal, instead focusing on core features and concepts. If at any point you are confused, or something seems to not work as you expect, you might find more information on the [wiki](https://github.com/bHimes/emClarity/wiki); feel free to also search the mailing list archive, or post new questions to the community forum, hosted on [google groups](https://groups.google.com/forum/#!forum/emclarity), should you have any questions you cannot resolve on your own. + +> **Tip**: To display every procedure available, run `emClarity help` from the command line. + +### Algorithms + +The **emClarity** source code is available on [github](https://github.com/bHimes/emClarity/tree/LTS_version_1_5_0), and we encourage you to go through the code to look at the algorithms directly. Because **emClarity** is frequently being updated, this can be a great way to see what's going on behind the scenes. + +Section [Algorithms](#algorithms) contains descriptions of the algorithms for each section presented in this tutorial. Please keep in mind that these are simplified descriptions of what **emClarity** is actually doing, as we often don't mention the details that were implemented to make the code more efficient. + +### Installation and Requirements + +Information about the [installation](https://github.com/bHimes/emClarity/wiki/Installation) and the software and hardware [requirements](https://github.com/bHimes/emClarity/wiki/Requirements) is available [online](https://github.com/bHimes/emClarity/wiki). + +### Parameter files + +**emClarity** is currently using a parameter file to manage inputs. You can find an example [here](https://github.com/bHimes/emClarity/blob/master/docs/exampleParametersAndRunScript/param0.m). + +### System parameters + +Your parameter files should have the following parameters: + +| Parameter | Description | +|-----------|-------------| +| `nGPUs` * | The number of visible GPUs. **emClarity** will try to use them in parallel as much as possible. If this number doesn't correspond to the actual number of GPUs available, **emClarity** will ask you to either adjust this number to match the number of GPUs, or modify the environment variable `CUDA_VISIBLE_DEVICE` to make some GPUs invisible to **MATLAB**. | +| `nCpuCores` * | The maximum number of processes to run simultaneously. In most **emClarity** programs, the number of processes launched in parallel on a single GPU is equal to `nCpuCores`/`nGPUs`. If your devices run out of memory, it is likely that you will have to decrease the number of processes per device, thus decreasing this parameter. | +| `fastScratchDisk` * | Path of the optional temporary cache directory, used by `ctf 3d` and `tomoCPR`. This directory is only temporary and is moved back inside the project directory at the end of the execution. We recommend setting this to the fastest storage you have available. If left empty, `ctf 3d` and `tomoCPR` will use directly the project cache directory. | + +*Required parameters are marked with * + +--- + +## The project directory + +**emClarity** should be run from the "project directory", referred to as ``. Every output will be saved in this directory, ignoring the temporary cache set by the `fastScratchDisk` parameter. As we go along, we will present in more detail each sub-directory and their content. + +### Directory Structure + +- **``**: Contains every input file and input directories **emClarity** needs and every outputs. As most of the **emClarity** programs are project based, you should run **emClarity** from this directory. + +- **`/rawData`**: **(User created)** Contains the original raw tilt-series data (`*.mrc`, `*.st`) and associated files (`*.rawtlt`) that will be used as input for `autoAlign`. This is where you should place your downloaded or collected tilt-series before starting the workflow. + +- **`/fixedStacks`**: **(Created by `autoAlign` or user)** Contains the raw (not aligned) tilt-series (`*.fixed`) and the initial tilt-series alignment files (`*.xf`, `*.tlt` and optionally `*.local` and `*.erase`). This directory is automatically created and populated by `emClarity autoAlign`, or manually created if you're starting with pre-aligned data from **ETomo**. + +- **`/fixedStacks/ctf`**: Created by `ctf estimate` and updated after tilt-series refinements by `ctf update`. Contains the radial averages (`*_psRadial1.pdf`) and stretched power spectrum (`*_PS2.mrc`) computed by `ctf estimate`, as well as the tilt-series metadata (`*_ctf.tlt`), used throughout the entire workflow and containing in particular the dose-scheme and defocus estimate of each view. + +- **`/aliStacks`**: Created by `ctf estimate` and updated after tilt-series refinement by `ctf update`. Contains the aligned, bead-erased tilt-series. These stacks are mostly used by `ctf 3d` to compute the tomograms at different binning. + +- **`/cache`**: Created and updated by **emClarity** when needed, usually during `ctf 3d`. Store any stack or reconstruction for the current binning. If a reconstruction (`*.rec`) is present at the current binning, `ctf 3d` will skip its reconstruction. + +- **`/convmap`**: When creating a project with `init`, **emClarity** will look in this directory to grab outputs from `templateSearch`. If you pick your particles with **emClarity**, the content of this directory is generated by `templateSearch`. + +- **`/recon`**: Holds the information for each reconstructed sub-region in a given tilt-series. The `*_recon.coords` files are read into the metadata created by `init` and is used whenever a tomogram is made or whenever the coordinates of a sub-region is needed. + +- **`/`**: **emClarity** does not directly use this directory, but it is used by `recScript2.sh` to define the sub-regions boundaries and create `/recon`. + +- **`/FSC`**: Created and updated during subtomogram averaging. Contains the spherical and conical FSCs for each cycle (`*fsc_GLD.txt` and `*fsc_GLD.pdf`), as well as the Figure-Of-Merit used for filtering (`*cRef_GLD.pdf`) and the CTF-corrected volume used for FSC calculations. + +- **`/alignResume`**: Contains the results of the subtomogram alignments, for each cycle. **emClarity** will look at this directory before aligning the particles from a given sub-region. If the results for this sub-region, at the current cycle, are already saved, it will skip the alignment. + +--- + +## Get your data ready + +In this tutorial, we will use the apoferritin tomography dataset deposited on [EMPIAR-11273](https://www.ebi.ac.uk/pdbe/emdb/empiar/entry/11273/). You should be able to get a sub-3Å map from this tutorial. Apoferritin is an excellent choice for learning subtomogram averaging due to its high octahedral symmetry, which makes processing faster and typically yields higher resolution results. + +### Tutorial Dataset + +| Aspect | Description | +|--------|-------------| +| **Sample** | Apoferritin (octahedral symmetry, ~12nm diameter) | +| **Tilt-series count** | 100 tilt-series (TS_12 to TS_123) | +| **Tilt-scheme** | Hagen dose-symmetric, Âą48°, 3° increment, 115.5e/Ã…Â˛ total exposure | +| **Instruments** | Krios at 300kV, Gatan K3 camera, 0.729Å/pix calibrated pixel size | +| **Defocus range** | -1 to -3 Îŧm | +| **Expected result** | Sub-3Å reconstruction | + +> **Note**: This dataset uses EER format movies that need to be converted to tilt-series. The high symmetry of apoferritin (octahedral) makes it ideal for tutorial purposes as it processes much faster than asymmetric particles like ribosomes. With 100 tilt-series, this provides excellent statistics for high-resolution reconstruction. + +### Setting up the project directory + +Before starting, create your project directory and the initial `rawData` subdirectory: + +```bash +# Create your project directory +mkdir -p /path/to/your/project +cd /path/to/your/project + +# Create rawData directory for your input tilt-series +mkdir rawData + +# For EMPIAR-11273, the data is available in EER format and needs to be converted +# The dataset contains 100 tilt-series (TS_12 to TS_123) +# Copy or link the converted tilt-series to rawData/ +# Each tilt-series should be named like TS_012.st, TS_013.st, etc. +``` + +The `fixedStacks` directory will be automatically created by `emClarity autoAlign` in the next step. If you're starting with pre-aligned data from **ETomo**, you would manually create and populate the `fixedStacks` directory instead. + +> **Data preparation note**: The EMPIAR-11273 dataset contains EER movies that need motion correction and tilt-series generation. Use tools like RELION's `relion_convert_to_tiff` or IMOD's `alignframes` to convert the EER data to tilt-series stacks before starting the emClarity workflow. + +--- + +## Workflow + +You will often find that it is much easier to organize every **emClarity** calls into one script. This script has two main purposes. First, it keeps track of the jobs that have been run (you can also find this information into the `logFile` directory). This is often useful to visualize the global picture and it might help you to remember how you got your final reconstruction. Second, it is a script, so you can use it directly to run **emClarity**, making the workflow much simpler. + +### Example Workflow Script + +```bash +#!/bin/bash + +# Simple function to stop on *most* errors +check_error() { + sleep 2 + if tail -n 30 ./logFile/emClarity.logfile |\ + grep -q "Error in emClarity" ; then + echo "Critical error found. Stopping the script." + exit + else + echo "No error detected. Continue." + fi +} + +# Change binning with tomoCPR +run_transition_tomoCPR() { + emClarity removeDuplicates param${i}.m ${i}; check_error + emClarity tomoCPR param${i}.m ${i}; check_error + emClarity ctf update param$((${i}+1)).m; check_error + emClarity ctf 3d param$((${i}+1)).m; check_error +} + +# Basic alignment cycle +run_avg_and_align() { + emClarity avg param${i}.m ${i} RawAlignment; check_error + emClarity alignRaw param${i}.m ${i}; check_error +} + +# autoAlign +# ctf estimate +# templateSearch + +# Create metadata and reconstruct the tomograms +emClarity init param0.m; check_error +emClarity ctf 3d param0.m; check_error + +# First reconstruction - check if that looks OK. +emClarity avg param0.m 0 RawAlignment; check_error +emClarity alignRaw param0.m 0; check_error + +# Bin 3 +for i in 1 2 3 4; do run_avg_and_align; done + +# Run tomoCPR at bin3 using cycle 4 and then switch to bin2 +run_transition_tomoCPR + +# Bin 2 +for i in 5 6 7 8 9; do run_avg_and_align; done + +# Run tomoCPR at bin2 using cycle 9 and then switch to bin1 +run_transition_tomoCPR + +# Bin 1 +for i in 10 11 12 13 14; do run_avg_and_align; done + +# Last cycle: merge the datasets +emClarity avg param15.m 15 RawAlignment; check_error +emClarity avg param15.m 15 FinalAlignment; check_error +emClarity reconstruct param15.m 15; +``` + +This example doesn't have a classification, but as explained in the classification section, classifications are encapsulated in their own cycles, so you can run them anytime you want between two cycles. + +In our experience, it is usually good practice to keep a close eye on how the half-maps and FSC evolves throughout the workflow, specially before deciding to change the sampling. Moreover, the tilt-series refinement is completely optional and you can simply change the binning by running `ctf 3d`, as opposed to `run_transition_tomoCPR`. + +> **Tip**: It is best practice to work the whole way through the workflow with the smallest data-set as possible and once you checked that everything holds, then process your full data. The same approach may be taken with this tutorial; it should be possible to obtain a low-resolution but recognizable 70S ribosome with only two or three of the tilt-series. + +--- + +## Initial tilt-series alignment + +### Objectives + +The first step of the workflow consists into finding an initial alignment for the raw tilt-series, that is the tilt, rotation and shift for each image within the series. After the alignment, the tilt-axis must be parallel to the y-axis. This alignment can be refined later on using the particles positions (tomoCPR section). + +### With emClarity + +**emClarity** can align the tilt-series for you using its `autoAlign` procedure. This procedure is based on the **IMOD** programs **tilt** and **tiltalign** and offers an automatic way of aligning tilt-series, with or without gold beads. + +#### Run + +As with every **emClarity** programs, you should run the next commands in the project directory. The `autoAlign` routine has the following signature: + +``` +emClarity autoAlign +``` + +Where: +- `` is the name of the parameter file +- `` is the tilt-series to align (e.g. `tilt1.st`) +- `` is a text file containing the raw tilt-angles (e.g. `tilt1.rawtlt`), in degrees +- `` is the image rotation (tilt-axis angle from the vertical), in degrees, as specified in **ETomo** + +For example, to run `autoAlign` on the first tilt-series of the tutorial: + +``` +emClarity autoAlign param.m TS_012.st TS_012.rawtlt 0 +``` + +For this apoferritin dataset, you may need to determine the appropriate rotation angle. You can check the first few tilt-series manually or use a small rotation angle since the data acquisition was well-controlled. + +#### Outputs + +**emClarity** creates and organizes the necessary files it needs to run the next step of the workflow. The goal here is to check whether or not the alignment is good enough to start with and the easiest way is to look at `fixedStacks/_3dfind.ali` or `fixedStacks/_binX.ali`. + +If you are familiar with **ETomo**, then you can of course also look at the log files saved in `emC_autoAlign_`. For instance, to visually check the fiducial beads: + +``` +3dmod \ +emC_autoAlign_/_X_3dfind.ali \ +emC_autoAlign_/_X_fitbyResid_X.fid +``` + +### With ETomo + +If you don't want to use `autoAlign`, we do recommend using the (fiducial) alignment procedure from the **ETomo** pipeline. One powerful option of this pipeline is to be able to solve for a series of local alignments using subsets of fiducial points, which can then be used by **emClarity**, via the IMOD **tilt** program, to reconstruct the tomograms. + +For each tilt-series, **emClarity** needs: + +- **`.fixed`**: the raw (not aligned) tilt-series. These should not be exposure-filtered nor phase flipped. + +- **`.xf`**: the file with alignment transforms to apply to the `.fixed` stacks. This file should contain one line per view, each with a linear transformation specified by six numbers. + +- **`.tlt`**: the file with the solved tilt angles. One line per view, angles in degrees. + +- **(optional) `.local`**: the file of local alignments. This file is similar to the `.xf` file, but contains one transformation per view and per patch. + +- **(optional) `.erase`**: the file with the coordinates (in pixel) of the fiducial beads to erase before ctf estimation. + +These files should be copied to `/fixedStacks`. + +> **Tip**: You don't necessarily need to copy the tilt-series to the `fixedStacks` directory; use soft links: `ln -s <...>/.mrc <...>/fixedStacks/.fixed` + +--- + +## Defocus estimate + +### Objectives + +There are two main objectives. First, create the aligned, optionally bead-erased, weighted stacks. Weighted refers to the per-view weighting applied by **emClarity** to take into account the frequency dependent drop in SNR due to radiation damage, an isotropic drop in SNR due to increased thickness with the tilt-angle causing inelastic scattering losses and optionally also for the cosine dose-scheme, also referred as Saxton scheme. These stacks will be then used to compute the tomograms at later stages. The second objective is to estimate the defocus of each view of the stack (two defoci and the astigmatism angle, per view). + +### Run + +The `ctf estimate` routine has the following signature: + +``` +emClarity ctf estimate +``` + +`` is the name of the parameter file (e.g. `param_ctf.m`), and `` is the base-name of the tilt-series in `/fixedStacks` you wish to process. + +For example, to run `ctf estimate` on the first tilt-series of the tutorial: + +``` +emClarity ctf estimate param_ctf.m TS_012 +``` + +If you have many tilt-series and you don't want to run all of them individually, you can do the following: + +```bash +#!/bin/bash +for stack in fixedStacks/*.fixed; do + prefix=${stack#fixedStacks/} + emClarity ctf estimate param_ctf.m ${prefix%.fixed} +done +``` + +For the apoferritin dataset, you generally won't need to remove specific images, but if needed, `ctf estimate` can remove images from the stack. For instance, to remove the first view: + +``` +emClarity ctf estimate param_ctf.m TS_012 1 +``` + +### Outputs + +You should make sure the tilt-series "looks aligned" and the average defocus (at the tilt axis) was correctly estimated. The best way to check: + +1. Open `aliStacks/_ali1.fixed` and go through the views. The views should be aligned to the tilt-axis, which must be parallel to the Y axis (so vertical if you use **3dmod**). If an `*.erase` file was available for this tilt-series, the beads should be removed. + +2. Open `fixedStacks/ctf/_ali1_psRadial_1.pdf` and check that the theoretical CTF estimate (green) matches the radial average of the power spectrum of the tilt-series (black). Note that the amplitude doesn't matter here, the phase on the other hand, does. + +3. If they don't match, it is likely that you will need to adjust the `defEstimate` and `defWindow` parameters. Open `fixedStacks/ctf/*_ccFIT.pdf`, which plots the cross-correlation score as a function of defocus. There is often an obvious correct peak, smoothly rising and falling. If you don't see this peak, try to change the sampled defoci with `defEstimate` Âą `defWindow` and re-run `ctf estimate`. + +--- + +## Algorithms + +### Naming conventions + +There is a lot of things to cover and it is often easier to use abbreviations (CTF, FSC, CCC, etc.) and symbols to refer to something. + +Indexes are subscripts, e.g. the p-th subtomogram is referred as **s**_p. This works with multiple indexes, e.g. the p-th subtomogram rotated by the r-th rotation is referred as **s**_p,r. Labels are subscripts as well, e.g. if we want to specify that the subtomograms are in the reference frame, we would write **s**_ref. + +### Euler angles conventions + +The Ά, θ, Έ Euler angles used by **emClarity** describe a z-x-z active intrinsic rotations of the particles coordinate system. That is to say, to switch the particles from the microscope frame to the reference frame, the basis vectors of the subtomograms are rotated (positive anti-clockwise) around z, the new x, and the new z axis. + +The microscope frame defines the coordinate system of the microscope, where the electron beam is the z axis. When the subtomograms are extracted from their tomogram, they are in the microscope frame. The reference frame is the coordinate system attached to the reconstruction, i.e. the subtomogram average and is usually set during the particle picking. + +### Linear transformations in Fourier space + +Linear transformations are often applied in Fourier space directly. It might be useful to write down a few useful properties of the Fourier transforms. + +- **Shift**: Shifting an image in real space is equivalent to applying a complex phase shift to its frequency spectrum +- **Magnification**: Magnifying an image by a factor a is equivalent to magnifying its frequency spectrum by 1/a +- **Rotation**: Rotating an image by an angle Θ in real space is the same as rotating its frequency spectrum by the same angle Θ + +--- + +## Select the sub-regions + +The purpose of this step is to define sub-regions within each tilt-series that will be reconstructed as tomograms. This is typically done using IMOD's reconstruction scripts to create appropriate coordinate files that define the boundaries of each sub-region. + +--- + +## Picking + +### Objectives + +It's time to pick the particles, i.e. the subtomograms. There are many ways to pick particles, but they usually all rely on the tomograms. Each particle can be described by its x, y, z coordinates and Ά, θ, Έ Euler angles. **emClarity** has a template matching routine that can pick the subtomograms for you, but it requires a template. + +### Run + +#### Preparing the template + +Before running `templateSearch`, you need to prepare a template. This template should have the same pixel size as the `PIXEL_SIZE` parameter. It doesn't need to be low-pass filter, as **emClarity** will do it internally. If you want to re-scale a map, you can run: + +```bash +emClarity rescale +``` + +`` and `` are the name of your template and the output name of the re-scaled template, respectively. `` is the pixel size of your template and `` is the desired pixel size. `` can be "GPU" or "cpu". + +> **Note**: For this tutorial, you'll need an apoferritin template. Apoferritin templates are readily available from the PDB (e.g. PDB ID: 2FHA) or EMDB, and can be filtered to the appropriate resolution for initial template matching. + +#### Generating the tomograms + +The tomograms use for the template matching are CTF multiplied. To generate them, simply run: + +```bash +emClarity ctf 3d templateSearch +``` + +This will generate a tomogram for every subregion defined in the `recon/*.coords` files. + +#### Template matching + +The `templateSearch` routine has the following signature: + +```bash +emClarity templateSearch