From 82beb307b4f163ba8546bd94f3ac495e80caddba Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 21 Dec 2023 12:54:03 -0500 Subject: [PATCH 001/151] whole buncha shit --- .gitignore | 2 + alignment/BH_alignRaw3d_v2.m | 85 ++-- alignment/BH_templateSearch3d_2.m | 748 ++++++++++++++++------------- coordinates/BH_defineMatrix.m | 4 +- ctf/BH_ctfCalc.m | 3 + ctf/BH_ctf_Correct.m | 2 +- ctf/BH_ctf_Correct3d.m | 14 +- ctf/BH_ctf_Updatefft.m | 2 +- masking/BH_weightMaskMex.m | 5 +- metaData/BH_rawAlignmentsApply.m | 4 +- metaData/EMC_parseParameterFile.py | 30 ++ metaData/EMC_str2double.py | 21 + metaData/EMC_str2double_test.py | 21 + metaData/test_tm.py | 17 + mexFiles/include/core_headers.cuh | 4 +- mexFiles/mexCompile.m | 7 +- mexFiles/mexFFT.cu | 49 +- mexFiles/mexSF3D.cu | 5 +- mexFiles/utils/ctf.cu | 21 +- statistics/BH_fscGold_class.m | 132 +++-- statistics/BH_multi_cRef_Vnorm.m | 1 - statistics/BH_pcaPub.m | 50 +- synthetic/BH_synthetic_mapBack.m | 1 + testScripts/fourierTransformer.m | 23 +- transformations/BH_average3d.m | 23 +- 25 files changed, 773 insertions(+), 501 deletions(-) create mode 100644 metaData/EMC_parseParameterFile.py create mode 100644 metaData/EMC_str2double.py create mode 100644 metaData/EMC_str2double_test.py create mode 100644 metaData/test_tm.py diff --git a/.gitignore b/.gitignore index 7d947cf8..2aa87e89 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ testScripts/mccExcludedFiles.log testScripts/readme.txt testScripts/requiredMCRProducts.txt testScripts/unresolvedSymbols.txt + +__pycache__/ \ No newline at end of file diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 1d935aff..9c70efc4 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -87,6 +87,17 @@ flgCutOutVolumes=0 end +try + tmpVal = pBH.('whitenPS'); + if (numel(tmpVal) == 3) + wiener_constant = tmpVal(3); + else + error('flgWhitenPS should be a 3 element vector'); + end +catch + wiener_constant = 0.0; +end + % TODO decide on a "reasonable" padding based on expected shifts. try CUTPADDING = subTomoMeta.('CUTPADDING') @@ -288,12 +299,12 @@ tiltList = masterTM.tiltGeometry; ctfGroupList = masterTM.('ctfGroupSize'); -% Sort the list by number of active subtomos to improve parallelism -sortedTomoList = zeros(nTomograms,1); -for iTomo = 1:nTomograms - sortedTomoList(iTomo) = sum(geometry.(tomoList{iTomo})(:,26)~=-9999); -end -[~, sortedTomoIDX] = sort(sortedTomoList,'descend') +% % Sort the list by number of active subtomos to improve parallelism +% sortedTomoList = zeros(nTomograms,1); +% for iTomo = 1:nTomograms +% sortedTomoList(iTomo) = sum(geometry.(tomoList{iTomo})(:,26)~=-9999); +% end +% [~, sortedTomoIDX] = sort(sortedTomoList,'descend') % mask defines area for angular search, peakRADIUS restricts translational @@ -322,15 +333,13 @@ limitToOne = pBH.('nCpuCores'); end - -nParProcesses = 0; -iterList = {}; +[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); if ( flgReverseOrder ) - fprintf('nCpuCores is %d\n', limitToOne); - [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); - for iParProc = 1:nParProcesses - iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' - end + % fprintf('nCpuCores is %d\n', limitToOne); + % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); + % for iParProc = 1:nParProcesses + % iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' + % end % Flip the order for reverse processing on a second machine. This will also disable saving of % of the metadata so there aren't conflicts. for iParProc = 1:nParProcesses @@ -338,32 +347,42 @@ end elseif ( flgStartThird ) - fprintf('nCpuCores is %d\n', limitToOne); - [ nParProcesses, iterList_full] = BH_multi_parallelJobs(nTomograms,nGPUs*cycle_denominator, sizeCalc(1),limitToOne*cycle_denominator); + % fprintf('nCpuCores is %d\n', limitToOne); + % [ nParProcesses, iterList_full] = BH_multi_parallelJobs(nTomograms,nGPUs*cycle_denominator, sizeCalc(1),limitToOne*cycle_denominator); - for iParProc = 1:nParProcesses - iterList_full{iParProc} = sortedTomoIDX(iterList_full{iParProc})'; - end + % for iParProc = 1:nParProcesses + % iterList_full{iParProc} = sortedTomoIDX(iterList_full{iParProc})'; + % end - % Need to scale this back down - nParProcesses = limitToOne; + % % Need to scale this back down + % nParProcesses = limitToOne; % Shift to start at one third through to process on a third machine. This will also disable saving of % of the metadata so there aren't conflicts. - iterList = {}; + % iterList = {}; + % for iParProc = 1:nParProcesses + % idx = cycle_numerator + (iParProc-1)*cycle_denominator; + % if (idx <= length(iterList_full)) + % iterList{iParProc} = iterList_full{idx}; + % end + % end for iParProc = 1:nParProcesses - idx = cycle_numerator + (iParProc-1)*cycle_denominator; - if (idx <= length(iterList_full)) - iterList{iParProc} = iterList_full{idx}; - end + % Note the use of floor is more like ceiling here (rounds away from + % zero) + nParts = ceil(length(iterList{iParProc}) ./ cycle_denominator); + fIDX = 1+(cycle_numerator - 1)*nParts; + lIDX = min(cycle_numerator*nParts,length(iterList{iParProc})); + iterList{iParProc} = iterList{iParProc}(fIDX:lIDX); end + else - fprintf('nCpuCores is %d\n', limitToOne); - [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); - for iParProc = 1:nParProcesses - iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' - end + % error('not supported run config'); + % fprintf('nCpuCores is %d\n', limitToOne); + % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); + % for iParProc = 1:nParProcesses + % iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' + % end end @@ -1090,9 +1109,7 @@ make_SF3D = false; if use_v2_SF3D % For now excluding the soften weight. - [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, ... - center,reconGeometry); - + [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); imgWdgInterpolator = ''; % The unshifted mask is kept in texture mem until no longer % needed diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index bef2d17d..f6ff478a 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -37,19 +37,19 @@ pBH = BH_parseParameterFile(PARAMETER_FILE); +% Currently hardcoded to always expect a tomogram constructed with ctf correction +% using emClarity ctf3d paramN.m templateSearch +use_ctf3d_templateSearch=true; if ctf3dNoSubTomoMeta mapBackIter = 0; - shouldBeCTF = 1; else try load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR - % clear subTomoMeta - % Make sure we get a CTF corrected stack - shouldBeCTF = 1 catch + % TODO: is there a better check on whether we are using ctf3d templateSearch vs ctf 3d? mapBackIter = 0; - shouldBeCTF = -1; + shouldBeCTF = false; end end samplingRate = pBH.('Tmp_samplingRate'); @@ -79,6 +79,17 @@ expand_lines = ''; end +try + tmpVal = pBH.('whitenPS'); + if (numel(tmpVal) == 3) + wiener_constant = tmpVal(3); + else + error('flgWhitenPS should be a 3 element vector'); + end +catch + wiener_constant = 0.0; +end + peakThreshold = pBH.('Tmp_threshold'); @@ -179,24 +190,25 @@ stats_diameter_fraction = 1 end -sum_of_x = []; -sum_of_x2 = []; + +mean_r2 = 0; +mean_r_mask = 0; +reference_mask = []; try - rescale_mip = pBH.('rescale_mip'); + scale_mip = pBH.('scale_mip'); catch - rescale_mip = false; + scale_mip = false; end % Limit to the first zero if we are NOT using the CTF rec -if (shouldBeCTF ~= 1) +if ~( use_ctf3d_templateSearch ) TLT = load(sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tomoName,mapBackIter+1)); def = mean(-1.*TLT(:,15))*10^6; %TODO if you switch to POSITIVEDEFOCUS this will be wrong firstZero = -0.2*def^2 +5.2*def +11; - + % FIXME: if you have a ctf 3d tomo why limit to the firsts zero? % Take the lower of firstZero lowResCut or Nyquist bp_vals(3) = max(bp_vals(3), firstZero); fprintf('\nUsing max (%f) of specified resolution cutoff of %f and first ctf zero %f Angstrom\n',bp_vals(3), wantedCut, firstZero); - end if pixelSize*2 > bp_vals(3) @@ -236,13 +248,11 @@ % For wedgeMask particleThickness = latticeRadius(3); -% [ tomogram ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... -% reconCoords, mapBackIter, samplingRate,... -% shouldBeCTF*gpuIDX, reconScaling,1,'','ctf'); -[ tomogram ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... + +[ tomogram, reconGeometry ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... reconCoords, mapBackIter, samplingRate,... - shouldBeCTF*gpuIDX, reconScaling,1,'',super_sample); + gpuIDX, reconScaling,1,'',super_sample); % We'll handle image statistics locally, but first place the global environment @@ -288,7 +298,11 @@ templateBIN = templateBIN - mean(templateBIN(:)); templateBIN = templateBIN ./rms(templateBIN(:)); -[templateMask] = gather(EMC_maskReference(gpuArray(templateBIN),pixelSize,{'fsc', true})); +[templateMask] = (EMC_maskReference(gpuArray(templateBIN),pixelSize,{'fsc', true})); +templateMask = gather(templateMask); + +% templateMask = gather(EMC_maskShape('sphere', size(templateBIN), [3,3,3].*2, 'gpu', {'shift', [0,0,0];'kernel',false})); + sizeTemp = size(template); @@ -336,6 +350,8 @@ highThr=sqrt(2).*erfcinv(ceil(peakThreshold.*0.10).*2./(prod(size(tomogram)).*nAngles(1))) + + [ OUTPUT ] = BH_multi_iterator( [targetSize; ... size(tomogram);... sizeTempBIN; ... @@ -390,6 +406,7 @@ fprintf('# of iterations %d %d %d\n', nIters); fprintf('-----\n'); +valid_ratio = prod(sizeChunk) ./ prod(validCalc); size(tomogram) % [ tomogram ] = BH_padZeros3d(tomogram, tomoPre, tomoPost, ... @@ -399,7 +416,9 @@ sizeTomo = size(tomogram); -[ validAreaMask ] = gather(BH_mask3d('rectangle',sizeChunk,validCalc./2,[0,0,0])); +[ validCalcMask ] = BH_mask3d('rectangle',sizeChunk,validCalc./2,[0,0,0]); + + [ vA ] = BH_multi_padVal( validArea, sizeChunk ); % This would need to be changed to take a mask size and not just a radius. % Currently, this would not produce the correct results for odd size area @@ -411,12 +430,7 @@ if ( tmpDecoy ) RESULTS_decoy = RESULTS_peak; end -if (rescale_mip) - sum_of_x = zeros(sizeTomo, 'single'); - sum_of_x2 = zeros(sizeTomo, 'single'); -end -% Loop over tomogram -% Set this up second + % % % % optimize fft incase a power of two is not used, this will make things run ok. @@ -426,6 +440,7 @@ % % % clear opt ans [ bhF ] = fourierTransformer(randn(sizeChunk, 'single','gpuArray')); + sum_template = mean(templateBIN(:)); sum_templateMask = mean(templateMask(:)); sum_imgMask = prod(sizeChunk);% bhF.halfDimSize * sizeChunk(2) * sizeChunk(3); @@ -480,8 +495,22 @@ tomoIDX = 1; nTomograms = prod(nIters); +try + test_local = pBH.('test_local'); +catch + test_local = false; +end +if (test_local) + scale_mip = false; +end + +test_local +scale_mip tomoStack = zeros([sizeChunk,nTomograms], 'single'); +if test_local + localStack = zeros([sizeChunk,nTomograms], 'single'); +end % tomoNonZero = zeros(nTomograms,6,'uint64'); % backgroundVol = zeros(sizeChunk,'single'); @@ -558,38 +587,63 @@ [ averageMask, flgOOM ] = BH_movingAverage_2(tomoChunk, statsRadius(1)); rmsMask = BH_movingAverage_2(tomoChunk.^2, statsRadius(1)); rmsMask = sqrt(rmsMask - averageMask.^2); - tomoChunk = (tomoChunk - averageMask) ./ rmsMask; + + % SAVE_IMG(BH_padZeros3d(real(single(... + % (rmsMask))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'),'rmsMask.mrc'); + % SAVE_IMG(templateMask,'templateMask.mrc'); + % tempPADMask = zeros(size(tomoChunk),'single','gpuArray'); + % tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... + % padBIN(1,2)+1: end - padBIN(2,2), ... + % padBIN(1,3)+1: end - padBIN(2,3)) = templateMask; + % tempPADMask = tempPADMask ./ sum(tempPADMask(:)); + % SAVE_IMG(tempPADMask,'tempPADMaskPre.mrc'); + % tempPADMask = (conj(bhF.fwdFFT(bhF.normalization_factor.^-1 .* tempPADMask )));% ./ (sum(tempPADMask > 0.01)./sum(tempPADMask(:))))); + % SAVE_IMG(bhF.invFFT(conj(tempPADMask)),'tempPADMask.mrc'); + % ms = real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor.^3 .*tomoChunk.^2.*validCalcMask),'fwd').*tempPADMask)); + % ma = real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor.^3 .*tomoChunk.*validCalcMask),'fwd').*tempPADMask)).^2; + % md = BH_padZeros3d(real(single(... + % (ms-ma))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'); + % SAVE_IMG(md,'md.mrc'); + % SAVE_IMG(sqrt(md), 'smd.mrc') + % SAVE_IMG(BH_padZeros3d(real(single(... + % ms)),... + % trimValid(1,:),trimValid(2,:),'GPU','single'),'ms.mrc'); + % SAVE_IMG(BH_padZeros3d(real(single(... + % (ma))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'),'ma.mrc'); + % error('asdf') + % correctedRMS = (ms - ma); + % correctedRMS = sqrt(BH_padZeros3d(real(single(... + % correctedRMS)),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single')); + % mean(correctedRMS(:)) + % 1/mean(correctedRMS(:)) + % SAVE_IMG(correctedRMS,'correctedRMS.mrc'); + % error('asdf') + if (test_local) + localStack(:,:,:,tomoIDX) = gather(rmsMask); + else + if ~(scale_mip) + tomoChunk = (tomoChunk - averageMask) ./ rmsMask; + end + end clear rmsMask averageMask -% averageMask = gather(averageMask); -% [ rmsMask ] = gather(BH_movingRMS_3(tomoChunk, statsRadius(1), averageMask)); - -% tomoChunk = tomoChunk - averageMask; -% tomoChunk = tomoChunk ./ rmsMask; -% if (save_average_filtered) -% tomoChunk = gpuArray(tomogram(cutX:cutX+sizeChunk(1)-1,... -% cutY:cutY+sizeChunk(2)-1,... -% cutZ:cutZ+sizeChunk(3)-1)); -% avgFiltRec = zeros(size(tomogram),'single'); -% end -% statsRadius(1) -% [ rmsMask ] = BH_movingRMS_2(tomoChunk-averageMask, statsRadius(1)); -% statsRadius(1) -% tomoChunk = tomoChunk ./ rmsMask; -% figure, imshow3D(BH_padZeros3d(gather(averageMask),'fwd',trimValid,'cpu','single')) -% figure, imshow3D(BH_padZeros3d(gather(rmsMask),'fwd',trimValid,'cpu','single')) -% return -% clear avgerageMask - - tomoChunk = gather(((-1*shouldBeCTF) .* tomoChunk )).*validAreaMask; + tomoChunk = gather(tomoChunk .*validCalcMask); + + -% tomoChunk = tomoChunk .* (-1*shouldBeCTF); % This is backwards, but I don't know why - tmp_sum = sum(tomoChunk(validAreaMask > 0.1)); - fullX = fullX + tmp_sum; + tmp_sum = sum(tomoChunk(validCalcMask > 0.1)); % REVERT + % tmp_sum = sum(tomoChunk(:)); + + fullX = fullX + gather(tmp_sum); fullX2 = fullX2 + gather(tmp_sum.^2); fullnX = fullnX + gather(prod(sizeChunk)); + tomoStack(:,:,:,tomoIDX) = tomoChunk; @@ -605,13 +659,12 @@ globalVariance = (fullX2/fullnX) - (fullX/fullnX)^2; %fprintf('After local normalization, scaling also the global variance %3.3e\n',globalVariance); -for iChunk = 1:tomoIDX-1 - tomoStack(:,:,:,iChunk) = tomoStack(:,:,:,iChunk) ./ sqrt(globalVariance); -end +% for iChunk = 1:tomoIDX-1 +% tomoStack(:,:,:,iChunk) = tomoStack(:,:,:,iChunk) ./sqrt(globalVariance); +% end - -clear tomoWedgeMask bandpassFilter statBinary validAreaMask tomoChunk +clear tomoWedgeMask validCalcMask bandpassFilter statBinary tomoChunk %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -628,17 +681,17 @@ % Center the spectrum by multiplication not swapping (this should just % be in the fourierTransformer class if it isn't already) - % swapPhase(obj, inputVol, direction) with fwd should do it - [dU,dV,dW] = BH_multi_gridCoordinates(size(tomoStack(:,:,:,1)),... - 'Cartesian','GPU', ... - {'none'},1,1,0); + % % swapPhase(obj, inputVol, direction) with fwd should do it + % [dU,dV,dW] = BH_multi_gridCoordinates(size(tomoStack(:,:,:,1)),... + % 'Cartesian','GPU', ... + % {'none'},1,1,0); - swapQuadrants = exp((-2i*pi).*(dU.*(floor(size(dU,1)/2)+1) + ... - (dV.*(floor(size(dV,2)/2)+1) + ... - (dW.*(floor(size(dW,3)/2)+1))))); - clear dU dV dW + % swapQuadrants = exp((-2i*pi).*(dU.*(floor(size(dU,1)/2)+1) + ... + % (dV.*(floor(size(dV,2)/2)+1) + ... + % (dW.*(floor(size(dW,3)/2)+1))))); + % clear dU dV dW - swapQuadrants = swapQuadrants(1:floor(size(swapQuadrants,1)/2)+1,:,:); + % swapQuadrants = swapQuadrants(1:floor(size(swapQuadrants,1)/2)+1,:,:); if (use_new_grid_search) theta_search = 1:gridSearch.number_of_out_of_plane_angles; @@ -648,7 +701,7 @@ for iAngle = theta_search - + if (use_new_grid_search) theta = gridSearch.parameter_map.theta(iAngle); numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); @@ -663,12 +716,9 @@ tempImg = gpuArray(templateBIN); %%%%% NEW switch to bin - interpolationNormFactor = sum(abs(tempImg(:)).^2); + % interpolationNormFactor = sum(abs(tempImg(:)).^2); - clear referenceStack tempFilter - % Calculate all references for each out of plane tilt only once - referenceStack = zeros([sizeTempBIN,numRefIter], 'single', 'gpuArray'); tomoIDX = 1; firstLoopOverAngle = true; @@ -683,7 +733,6 @@ templateMask_interpolator = ''; [templateMask_interpolator, ~] = interpolator(gpuArray(templateMask),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - @@ -700,21 +749,38 @@ % Truth value to initialize temp results matrix each new tomo % chunk. firstLoopOverChunk = true; - + if (use_new_grid_search) - fprintf('Working on tilt(%d/%d) tomoChunk(idx%d/%d)\t' ... + fprintf('Working on tilt(%d/%d) tomoChunk(%d/%d)\t' ... ,iAngle,gridSearch.number_of_out_of_plane_angles, tomoIDX,nTomograms); else - fprintf('working on tilt(%d/%d) tomoChunk(idx%d/%d)\t' ... + fprintf('working on tilt(%d/%d) tomoChunk(%d/%d)\t' ... ,iAngle,size(angleStep,1), tomoIDX,nTomograms); end tomoFou = gpuArray(tomoStack(:,:,:,tomoIDX)); - tomoFou = swapQuadrants.*bhF.fwdFFT(tomoFou); + if test_local + localFou = BH_padZeros3d(localStack(:,:,:,tomoIDX),trimValid(1,:),trimValid(2,:),'GPU','single'); +% localStack(:,:,:,tomoIDX)); + end + % % profile on + if (scale_mip) + tomoFou_2 = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou.^2)), 'fwd'); + end + + tomoFou = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)), 'fwd'); + % profile on + % if (scale_mip) + % tomoFou_2 = swapQuadrants.*bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou.^2)); + % end + + + % tomoFou = swapQuadrants.*bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)); + if (use_new_grid_search) @@ -725,6 +791,7 @@ for iAzimuth = phi_search + if (use_new_grid_search) phi = iAzimuth; else @@ -732,6 +799,7 @@ end for iInPlane = inPlaneSearch + psi = iInPlane; %calc references only on first chunk @@ -742,50 +810,83 @@ end [ tempRot ] = template_interpolator.interp3d(... - [phi, theta, psi - phi],... - [1,1,1],rotConvention,... + [phi, theta, psi - phi],... + [0,0,0],rotConvention,... 'forward','C1'); - + tempPAD = tempPAD .* 0; tempPAD(padBIN(1,1)+1: end - padBIN(2,1), ... padBIN(1,2)+1: end - padBIN(2,2), ... padBIN(1,3)+1: end - padBIN(2,3)) = tempRot; - - tempFou = conj(bhF.fwdFFT(tempPAD)); + tempPAD = tempPAD - mean(tempPAD(:)); + + if (scale_mip) + % I should probaly switch to using the SF3D masked reference, but that also changes the baseline implementation + % so I'll leave it for now. + + + + tempPADMask = tempPADMask .* 0; + + tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... + padBIN(1,2)+1: end - padBIN(2,2), ... + padBIN(1,3)+1: end - padBIN(2,3)) = templateMask_interpolator.interp3d(... + [phi, theta, psi - phi],... + [0,0,0],rotConvention,... + 'forward','C1'); + + tempPADMask = tempPADMask ./ sum(tempPADMask(:)); + tempMaskFou = (conj(bhF.fwdFFT(bhF.normalization_factor.^-1 .* tempPADMask )));% ./ (sum(tempPADMask > 0.01)./sum(tempPADMask(:))))); + md = BH_padZeros3d(... + real(bhF.invFFT(tomoFou_2.*tempMaskFou)) - real(bhF.invFFT(tomoFou.*tempMaskFou)).^2,... + trimValid(1,:),trimValid(2,:),'GPU','single'); + + mip_scaling = sqrt(md); + + + + end + + + tempFou = conj(bhF.fwdFFT(tempPAD)); + - ccfmap = BH_padZeros3d((real(single(... - bhF.invFFT(tomoFou.*tempFou)))),...%./(tomoNorm.*tempNorm))))),... - trimValid(1,:),trimValid(2,:),'GPU','single'); + + ccfmap = BH_padZeros3d(real(single(... + bhF.invFFT(tomoFou.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + trimValid(1,:),trimValid(2,:),'GPU','single'); % - ccfmap = ccfmap ./ std(ccfmap(:)); - - if ( tmpDecoy > 0 ) - - if (firstLoopOverAngle) - - decoy = BH_padZeros3d(BH_reScale3d(tempRot./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - padDecoy(1,:),padDecoy(2,:),'GPU','single'); - else - % Probably just make a second decoy stack to avoid - % re-interpolating. If it works, then do this. - error('This is temp broken with new interpolator'); -% decoy = BH_padZeros3d(BH_reScale3d(referenceStack(:,:,:,intraLoopAngle)./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... -% padDecoy(1,:),padDecoy(2,:),'GPU','single'); - end - + if ~(scale_mip) + ccfmap = ccfmap ./ std(ccfmap(:)); + end - - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*conj(fftn(decoy)))))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU','single'); - + if ( tmpDecoy > 0 ) + + if (firstLoopOverAngle) + + decoy = BH_padZeros3d(BH_reScale3d(tempRot./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... + padDecoy(1,:),padDecoy(2,:),'GPU','single'); + else + % Probably just make a second decoy stack to avoid + % re-interpolating. If it works, then do this. + error('This is temp broken with new interpolator'); + % decoy = BH_padZeros3d(BH_reScale3d(referenceStack(:,:,:,intraLoopAngle)./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... + % padDecoy(1,:),padDecoy(2,:),'GPU','single'); + end + - elseif ( tmpDecoy < 0 ) + + decoy = BH_padZeros3d(fftshift(real(single( ... + ifftn(tomoFou.*conj(fftn(decoy)))))),..../(decoyNorm.*tomoNorm))))), + trimValid(1,:), ... + trimValid(2,:),'GPU','single'); + + + elseif ( tmpDecoy < 0 ) % Just use the mirror image of the template, i.e. take the conj % (of the conj) so just the padded FFT of the ref. @@ -794,12 +895,93 @@ trimValid(1,:), ... trimValid(2,:),'GPU','single'); - end - clear tempRot - % If first loop over tomo, initialize the storage volumes, if - % first loop over the chunk but not over the tomo, pull storage - % chunks from storage volume. - if (firstLoopOverTomo && firstLoopOverChunk) + end + + if (scale_mip) + + % tempFou = conj(bhF.fwdFFT(tempPADMask.*bhF.normalization_factor^2)); + % tempFou = conj(bhF.fwdFFT(tempPADMask.*bhF.normalization_factor^0)); + + % mip_scaling = (1.0/mean_r_mask) .* ... + % BH_padZeros3d(real(single(... + % bhF.invFFT(tomoFou.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single').^2; + + % mip_scaling = BH_padZeros3d(real(single(... + % bhF.invFFT(tomoFou_2.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single') ... + % - ... + % mip_scaling; + + + + + % try + + % mip_scaling = sqrt(mean_r2) .*sqrt(mip_scaling); % FIXME add check on zero + % catch + + % lowval = mip_scaling < 0; + % numel(mip_scaling) + % sum(lowval,'all') + % mean(mip_scaling(lowval),'all') + % mean(mip_scaling(~lowval),'all') + % fprintf('\nmean_r_mask %3.3e mean_r2 %3.3e\n',mean_r_mask,mean_r2); + % fprintf('norm factor %f\n', bhF.normalization_factor); + % a = conj(bhF.fwdFFT(tempPAD.^2)); + % b = conj(bhF.fwdFFT(tempPADMask)); + % a(2:end) = 0; + % b(2:end) = 0; + % a = bhF.invFFT(a); + % b = bhF.invFFT(b); + % fprintf('mean_r_mask %3.3e mean_r2 %3.3e\n',1.0/b(1),a(1)); + % fprintf('mean_r_mask %3.3e mean_r2 %3.3e\n',1.0/b(5),a(5)); + + + % fprintf('iAngle %d idx %d iAzimuth %d iInPlane %d\n',iAngle,tomoIDX,iAzimuth,iInPlane); + % SAVE_IMG(tempPADMask,sprintf('tempPADMask_%d.mrc',tomoIDX)); + % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX)); + % SAVE_IMG( BH_padZeros3d(real(single(... + % bhF.invFFT(tomoFou))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('tomoFou_%d.mrc',tomoIDX)); + % SAVE_IMG( BH_padZeros3d((real(single(... + % bhF.invFFT(tomoFou_2)))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('tomoFou_2_%d.mrc',tomoIDX)); + % SAVE_IMG(BH_padZeros3d(real(single(... + % bhF.invFFT(tomoFou_2.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('mip_scaling_2_%d.mrc',tomoIDX)); + % SAVE_IMG(... + % BH_padZeros3d((real(single(... + % bhF.invFFT(tomoFou.*tempFou)))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single').^2, sprintf('mip_scaling_%d.mrc',tomoIDX)); + + % error('Faild on mip scaling caclulation'); + % end + + % mip_scaling(abs(mip_scaling) < 1e-6) = 1e-6; + + % print the min,max and mean of the mip_scaling + % fprintf('\nmip_scaling min %3.3e max %3.3e mean %3.3e\n',min(mip_scaling(:)),max(mip_scaling(:)),mean(mip_scaling(:))); + ccfmap = ccfmap ./ mip_scaling; + + % Now scale the CCF to be an SNR by using the global variance which should be mostly noise peaks. + ccfmap = ccfmap ./ std(ccfmap(:)); + + end + + if test_local + % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX)); + ccfmap = ccfmap ./ localFou; + ccfmap = ccfmap ./ std(ccfmap(:)); + % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX+1)); + % error('asdf') + end + clear tempRot + + % If first loop over tomo, initialize the storage volumes, if + % first loop over the chunk but not over the tomo, pull storage + % chunks from storage volume. + if (firstLoopOverTomo && firstLoopOverChunk) %store ccfmap as complex with phase = angle of reference magTmp = ccfmap; if ( tmpDecoy ) @@ -807,178 +989,127 @@ end angTmp = ones(size(magTmp), 'single','gpuArray'); - if (rescale_mip) - sum_of_x_tmp = ccfmap; - sum_of_x2_tmp = ccfmap.^2; - end - firstLoopOverTomo = false; firstLoopOverChunk = false; intraLoopAngle = intraLoopAngle + 1; currentGlobalAngle = currentGlobalAngle + 1; - elseif (firstLoopOverChunk) - % These double cuts are old, and don't really make sense. Make - % this more consistant with current operations when there is - % time. - magTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - if ( tmpDecoy ) - decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyTmp = gpuArray(decoyTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - - - magTmp = gpuArray(magTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - - firstLoopOverChunk = false; - - replaceTmp = ( magTmp < ccfmap ); + elseif (firstLoopOverChunk) + % These double cuts are old, and don't really make sense. Make + % this more consistant with current operations when there is + % time. + magTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + if ( tmpDecoy ) + decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + decoyTmp = gpuArray(decoyTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3))); + decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); + end - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - if (rescale_mip) - - sum_of_x_tmp = sum_of_x(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sum_of_x_tmp = gpuArray(sum_of_x_tmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - sum_of_x_tmp = sum_of_x_tmp + ccfmap; + magTmp = gpuArray(magTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3))); + angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3))); - sum_of_x2_tmp = sum_of_x2(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sum_of_x2_tmp = gpuArray(sum_of_x2_tmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - sum_of_x2_tmp = sum_of_x2_tmp + ccfmap.^2; - end - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp + firstLoopOverChunk = false; - else - % update higher values of ccfmap with new reference if applicable. + replaceTmp = ( magTmp < ccfmap ); + + magTmp(replaceTmp) = ccfmap(replaceTmp); + angTmp(replaceTmp) = currentGlobalAngle; + + + intraLoopAngle = intraLoopAngle + 1; + currentGlobalAngle = currentGlobalAngle + 1; + clear replaceTmp + else + % update higher values of ccfmap with new reference if applicable. replaceTmp = ( magTmp < ccfmap ); - magTmp(replaceTmp) = ccfmap(replaceTmp); angTmp(replaceTmp) = currentGlobalAngle; if ( tmpDecoy ) decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); end - - if (rescale_mip) - sum_of_x_tmp = sum_of_x_tmp + ccfmap; - sum_of_x2_tmp = sum_of_x2_tmp + ccfmap.^2; - end - + + intraLoopAngle = intraLoopAngle + 1; currentGlobalAngle = currentGlobalAngle + 1; clear replaceTmp + end + nComplete = nComplete + 1; end - nComplete = nComplete + 1; end - end - - % After searching all angles on this chunk, but out meaningful + % profile viewer + % return + % After searching all angles on this chunk, but out meaningful % portion for storage. - + + % FIXME this double cutting and temporary allocation is ridiculous. + magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + + + magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); + angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); + + + RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; + + clear magStoreTmp + + RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; + clear angStoreTmp - % FIXME this double cutting and temporary allocation is ridiculous. - magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - - - magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); - angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); - - - RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; - - clear magStoreTmp - - RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; - clear angStoreTmp + if ( tmpDecoy ) + decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + decoyStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(decoyTmp); + RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = decoyStoreTmp; + + end - if ( tmpDecoy ) - decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(decoyTmp); - RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = decoyStoreTmp; - - end - if ( rescale_mip ) - sum_of_x_store = sum_of_x(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sum_of_x_store(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sum_of_x_tmp); - sum_of_x(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sum_of_x_store; - clear sum_of_x_store - - sum_of_x2_store = sum_of_x2(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sum_of_x2_store(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sum_of_x2_tmp); - sum_of_x2(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sum_of_x2_store; - clear sum_of_x2_store - end - tomoTime = toc; - totalTime = totalTime + toc; timeEstimate = totalTime * (nTomograms*nAngles(1)./(nComplete-1)); - fprintf('elapsed time = %f s est remain %f s\n', tomoTime, timeEstimate); - tomoIDX = tomoIDX + 1; - firstLoopOverAngle = false; - currentGlobalAngle = currentGlobalAngle - intraLoopAngle + 1; - end - + tomoTime = toc; + totalTime = totalTime + toc; timeEstimate = totalTime * (nTomograms*nAngles(1)./(nComplete-1)); + fprintf('elapsed time = %f s est remain %f s\n', tomoTime, timeEstimate); + tomoIDX = tomoIDX + 1; + firstLoopOverAngle = false; + currentGlobalAngle = currentGlobalAngle - intraLoopAngle + 1; + end currentGlobalAngle = currentGlobalAngle + intraLoopAngle - 1; + end %save('angle_list.txt','angle_list','-ascii'); clear tomoStack @@ -1000,35 +1131,7 @@ end -if ( rescale_mip ) - sum_of_x = sum_of_x(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)) ./ currentGlobalAngle; - - sum_of_x2 = sum_of_x2(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)) ./ currentGlobalAngle; - - %SAVE_IMG(sum_of_x,'sum_of_x.mrc'); - %SAVE_IMG(sum_of_x2,'sum_of_x2.mrc'); - %SAVE_IMG(RESULTS_peak,'prescaling.mrc'); - - RESULTS_peak = RESULTS_peak - sum_of_x; - sum_of_x = sqrt(sum_of_x2 - sum_of_x.^2); - clear sum_of_x2; -% SAVE_IMG(sum_of_x,'stddev.mrc'); -% mov = mean(sum_of_x(:)); -% % sov = std(sum_of_x(:)); -% sov = 0; -% sum_of_x(sum_of_x < (mov - 1*sov)) = max(sum_of_x(:)); -% SAVE_IMG(sum_of_x,'stddev_clipped.mrc'); - - RESULTS_peak = RESULTS_peak ./ sum_of_x; - - RESULTS_peak = RESULTS_peak - mean(RESULTS_peak(:)); - RESULTS_peak = RESULTS_peak ./ rms(RESULTS_peak(:)); - clear sum_of_x; -end + gpuDevice(useGPU); clear bhF @@ -1049,7 +1152,7 @@ anglesOUT = sprintf('./%s/%s_angles.mrc',convTMPNAME,mapName); angleListOUT = sprintf('./%s/%s_angles.list',convTMPNAME,mapName); SAVE_IMG(MRCImage(mag),resultsOUT); -SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); +% SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); if ( tmpDecoy ) decoyOUT = sprintf('./%s/%s_decoy.mrc',convTMPNAME,mapName); SAVE_IMG(MRCImage((RESULTS_decoy)),decoyOUT); @@ -1288,55 +1391,42 @@ errID = fopen(sprintf('./%s/%s.errID',convTMPNAME,mapName)); - -n=1 +n=1; +nSym=1; for i = 1:length(peakMat(:,1)) - if all(peakMat(i,1:3)) + if all(peakMat(i,1:3)) - if SYMMETRY > 1 - % Generate a uniform distribution over the in-plane - % randomizations - iSym = rem( n + SYMMETRY, SYMMETRY)+1; - r = reshape(BH_defineMatrix(peakMat(i,4:6), rotConvention , 'inv') *... - symOps.symmetry_matrices{iSym},1,9); - else - r = reshape(BH_defineMatrix(peakMat(i,4:6), rotConvention , 'inv'),1,9); - end - fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... - peakMat(i,4:6),r,1); + iSym = mod(nSym,symOps.nSymMats)+1; + % Generate a uniform distribution over the in-plane + % randomizations + + r = reshape(BH_defineMatrix(peakMat(i,4:6), rotConvention , 'inv') * symOps.symmetry_matrices{iSym},1,9); + nSym = nSym + 1; + fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... + '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... + i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... + peakMat(i,4:6),r,1); - if nPeaks > 1 - - for iPeak = 2:nPeaks - if SYMMETRY > 1 - % Generate a uniform distribution over the in-plane - % randomizations - iSym = rem( n + SYMMETRY, SYMMETRY)+1; - r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), rotConvention , 'inv')*... - symOps.symmetry_matrices{iSym},1,9); - else - r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), rotConvention , 'inv'),1,9); - end - fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,[1:3]+10*(iPeak-1)), ... - peakMat(i,[4:6]+10*(iPeak-1)),r,1); - end - - - end - - fprintf(fileID,'\n'); - - - - - fprintf(fileID2,'%f %f %f\n',peakMat(i,1:3)./samplingRate); + if nPeaks > 1 + for iPeak = 2:nPeaks + + iSym = mod(nSym,symOps.nSymMats)+1; + r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), rotConvention , 'inv') * symOps.symmetry_matrices{iSym},1,9); + nSym = nSym + 1; + fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... + '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... + i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,[1:3]+10*(iPeak-1)), ... + peakMat(i,[4:6]+10*(iPeak-1)),r,1); - n = n +1; + end + end + + fprintf(fileID,'\n'); + fprintf(fileID2,'%f %f %f\n',peakMat(i,1:3)./samplingRate); + + + n = n + 1; end end @@ -1345,6 +1435,8 @@ fclose(fileID); fclose(fileID2); + + system(sprintf('point2model -number 1 -sphere 3 -scat ./%s/%s.pos ./%s/%s.mod', convTMPNAME,mapName,convTMPNAME, mapName)); fileID = fopen(sprintf('./%s/%s.path',convTMPNAME,mapName),'w'); diff --git a/coordinates/BH_defineMatrix.m b/coordinates/BH_defineMatrix.m index b164a167..6ab1926b 100755 --- a/coordinates/BH_defineMatrix.m +++ b/coordinates/BH_defineMatrix.m @@ -82,9 +82,9 @@ % 0 0 1 ]; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -if strcmpi(DIRECTION, 'forward') || strcmpi(DIRECTION, 'invVector') +if strcmpi(DIRECTION, 'forward') || strcmpi(DIRECTION, 'fwd') || strcmpi(DIRECTION, 'invVector') angles = -1.*angles; -elseif strcmpi(DIRECTION, 'inv') || strcmpi(DIRECTION, 'forwardVector') +elseif strcmpi(DIRECTION, 'inverse') || strcmpi(DIRECTION, 'inv') || strcmpi(DIRECTION, 'forwardVector') % For interpolation the vectors are applied to a grid, so the sense must % be inverted to make the final transformation active. diff --git a/ctf/BH_ctfCalc.m b/ctf/BH_ctfCalc.m index 21715c4d..ea35c490 100755 --- a/ctf/BH_ctfCalc.m +++ b/ctf/BH_ctfCalc.m @@ -3,6 +3,9 @@ % Calculate a ctf and with the given modification to information at resolution % lower than the first peak. Also return the unmodified ctf. +% FIXME: I think the need for double was for overflow in the phase +% we use single in cisTEM, so it must just be scaling issue. Or better yet, switch over to mexCTF + precision = 'single'; flgComplex = 0; calcOneD = 0; diff --git a/ctf/BH_ctf_Correct.m b/ctf/BH_ctf_Correct.m index a9235640..94afc669 100755 --- a/ctf/BH_ctf_Correct.m +++ b/ctf/BH_ctf_Correct.m @@ -36,7 +36,7 @@ for iStack = 1:length(STACK_LIST_tmp) if subTomoMeta.mapBackGeometry.(STACK_LIST_tmp{iStack}).nTomos STACK_LIST{nST} = STACK_LIST_tmp{iStack}; - nST = nST +1; + nST = nST + 1; end end clear STACK_LIST_tmp diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 2fbfd4ad..bb1afe7c 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -69,6 +69,7 @@ reconstructionParameters = 0; filterProjectionsForTomoCPRBackground=0; loadSubTomoMeta = true; +flgWhitenPS = [0,0,0.0]; if nargin > 2 if ~isempty(EMC_str2double(varargin{1})) reconstructionParameters = EMC_str2double(varargin{1}); @@ -88,8 +89,9 @@ end elseif nargin > 1 if strcmpi(varargin{1},'templateSearch') - recWithoutMat = true - loadSubTomoMeta = false + recWithoutMat = true; + loadSubTomoMeta = false; + flgWhitenPS = [0,0,1.0]; if (bh_global_turn_on_phase_plate(1)) fprintf('WARNING: the filtered tomogram should only be used for viz, not template matching.'); end @@ -105,9 +107,13 @@ try % -1, whiten before ctf, 1 whiten after - test both. - flgWhitenPS = [pBH.('whitenPS')(1),0,pBH.('whitenPS')(2)]; + usr_flgWhitenPS = pBH.('whitenPS'); + if (numel(usr_flgWhitenPS) == 3) + flgWhitenPS = usr_flgWhitenPS; + else + error('flgWhitenPS should be a 3 element vector'); + end catch - flgWhitenPS = [0,0,0]; end if (bh_global_turn_on_phase_plate(1) && flgWhitenPS(1)) diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 21158b0d..5cd56a3d 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -674,7 +674,7 @@ % Workaround for partial numbers - need something better. FIXME n_tomos_found = 0; - for jTomo = 1:size(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords,1) %subTomoMeta.mapBackGeometry.(STACK_PRFX).nTomos + for jTomo = 1:size(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords,1) if any(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords(jTomo,:)) n_tomos_found = n_tomos_found + 1; % We might have skipped the update if tomoCPR failed. diff --git a/masking/BH_weightMaskMex.m b/masking/BH_weightMaskMex.m index c0011e90..87df8d17 100644 --- a/masking/BH_weightMaskMex.m +++ b/masking/BH_weightMaskMex.m @@ -1,5 +1,5 @@ function [ SF3D ] = BH_weightMaskMex(SIZE, SAMPLING, TLT, ... - xyzSubTomo,reconGeometry) + xyzSubTomo,reconGeometry, wiener_constant) %UNTITLED Summary of this function goes here % Detailed explanation goes here @@ -48,7 +48,8 @@ gather(single(iDefocus + iddF)), ... gather(single(iDefocus - iddF)), ... idPHI,iPhaseShift,nTilts,tiltAngles, ... - exposure,fractionOfElastics.*fractionOfDose,int16(1)); + exposure,fractionOfElastics.*fractionOfDose,int16(1), ... + gather(single(wiener_constant))); % SF3D = SF3D ./ (WGT+0.01); diff --git a/metaData/BH_rawAlignmentsApply.m b/metaData/BH_rawAlignmentsApply.m index ed7eae35..877ac3bd 100755 --- a/metaData/BH_rawAlignmentsApply.m +++ b/metaData/BH_rawAlignmentsApply.m @@ -65,9 +65,9 @@ for iParticle = includeList' % assuming all classes are sequential, only discarded between cycles. - particleIDX = positionList(iParticle,4 + 26*(iPeak-1)) + particleIDX = positionList(iParticle,4 + 26*(iPeak-1)); - pIndex = find(newAlignment(:,2) == particleIDX) + pIndex = find(newAlignment(:,2) == particleIDX); newAngles = newAlignment(pIndex,[3:5] + 10*(iPeak-1)); diff --git a/metaData/EMC_parseParameterFile.py b/metaData/EMC_parseParameterFile.py new file mode 100644 index 00000000..8bcc76d8 --- /dev/null +++ b/metaData/EMC_parseParameterFile.py @@ -0,0 +1,30 @@ +import EMC_str2double as emc + +def EMC_parseParameterFile(PARAMETER_FILE): + # Open the parameter file + with open(PARAMETER_FILE, 'r') as fileID: + # Read the file into a list of strings, ignoring comments and empty lines + p = [line.strip() for line in fileID if line.strip() and not line.startswith('%')] + + # Check that all parameters are name: value pairs + stringValues = ['subTomoMeta', 'Ali_mType', 'Cls_mType', 'Cls_mType', 'Raw_mType', 'Fsc_mType', + 'Pca_distMeasure', 'Kms_mType', 'flgPrecision', 'Tmp_xcfScale', 'fastScratchDisk', + 'Tmp_eraseMaskType', 'startingDirection', 'Peak_mType', 'symmetry'] + pStruct = {} + for line in p: + try: + name, value = line.split('=', 1) + name = name.strip() + value = value.strip() + # also strip any trailing ';' or ',' + if value[-1] in [';', ',']: + value = value[:-1] + if name in stringValues: + pStruct[name] = value + else: + pStruct[name] = emc.EMC_str2double(value) + except: + err_msg = f"BH_parseParameterFile: invalid parameter line!\nReceived: {line}" + raise ValueError(err_msg) + + return pStruct \ No newline at end of file diff --git a/metaData/EMC_str2double.py b/metaData/EMC_str2double.py new file mode 100644 index 00000000..1d117b27 --- /dev/null +++ b/metaData/EMC_str2double.py @@ -0,0 +1,21 @@ +from numpy import ones + +def EMC_str2double(input_str): + try: + output_double = float(input_str) + except ValueError: + try: + output_double = eval(input_str) + if output_double is None: + raise ValueError + except ValueError: + try: + # check for ones() is in the string and use the numpy function to replace it + if 'ones(' in input_str: + input_str = input_str.replace('ones(', 'ones((') + input_str = input_str.replace(')', ', dtype=float)') + print(input_str) + except ValueError: + err_msg = f"EMC_str2double: input string is not a number!\nReceived: {input_str}" + raise ValueError(err_msg) + return output_double \ No newline at end of file diff --git a/metaData/EMC_str2double_test.py b/metaData/EMC_str2double_test.py new file mode 100644 index 00000000..85f28078 --- /dev/null +++ b/metaData/EMC_str2double_test.py @@ -0,0 +1,21 @@ +import numpy as np +import EMC_str2double as emc + +def test_EMC_str2double(): + # Test valid input + assert emc.EMC_str2double('3.14') == 3.14 + assert emc.EMC_str2double('-2.718') == -2.718 + assert emc.EMC_str2double('1e6') == 1e6 + assert emc.EMC_str2double('1.23e-4') == 1.23e-4 + assert emc.EMC_str2double('inf') == np.inf + assert emc.EMC_str2double('-inf') == -np.inf + assert np.isnan(emc.EMC_str2double('nan')) + + + try: + emc.EMC_str2double('not a number') + except ValueError as e: + assert str(e) == "EMC_str2double: input string is not a number!\nReceived: not a number" + +# Run test +test_EMC_str2double() \ No newline at end of file diff --git a/metaData/test_tm.py b/metaData/test_tm.py new file mode 100644 index 00000000..44f550e6 --- /dev/null +++ b/metaData/test_tm.py @@ -0,0 +1,17 @@ +import sys +from EMC_parseParameterFile import EMC_parseParameterFile + +if __name__ == '__main__': + # Check that the script was called with the correct number of arguments + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} PARAMETER_FILE") + sys.exit(1) + + # Get the parameter file name from the command-line argument + PARAMETER_FILE = sys.argv[1] + + # Call EMC_parseParameterFile with the parameter file name + pStruct = EMC_parseParameterFile(PARAMETER_FILE) + + # Print the resulting parameter structure + print(pStruct) \ No newline at end of file diff --git a/mexFiles/include/core_headers.cuh b/mexFiles/include/core_headers.cuh index 0df0bf63..183b9301 100644 --- a/mexFiles/include/core_headers.cuh +++ b/mexFiles/include/core_headers.cuh @@ -7,6 +7,8 @@ #include #include #include +#include + #include "cublas_v2.h" // Utilities and timing functions @@ -83,4 +85,4 @@ struct ctfParams { // Kernel defs __global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered); -__global__ void ctf(cufftReal* ctf, uint2 dims, uint2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered, float radial_weight, float total_exposure); +__global__ void ctf(cufftReal* ctf, uint2 dims, uint2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered, float radial_weight, float total_exposure, float wiener_constant = 0.f); diff --git a/mexFiles/mexCompile.m b/mexFiles/mexCompile.m index b61bcf75..7de3cf64 100644 --- a/mexFiles/mexCompile.m +++ b/mexFiles/mexCompile.m @@ -40,14 +40,13 @@ }; - -if isfolder(sprintf('%s/compiled',mexPATH)) - system(sprintf('rm -rf %s/compiled/*',mexPATH)); -elseif isfile(sprintf('%s/compiled',mexPATH)) +if isfile(sprintf('%s/compiled',mexPATH)) system(sprintf('rm -rf %s/compiled',mexPATH)); system(sprintf('mkdir -p %s/compiled',mexPATH)); end + for i =1: length(mexFILE) + mexcuda( mexcuda_opts{:}, sprintf('%s/%s.cu',mexPATH,mexFILE{i}), inc{1}, inc{2}); system(sprintf('mv %s.mexa64 %s/compiled',mexFILE{i}, mexPATH)); diff --git a/mexFiles/mexFFT.cu b/mexFiles/mexFFT.cu index 25916520..004d247c 100644 --- a/mexFiles/mexFFT.cu +++ b/mexFiles/mexFFT.cu @@ -46,6 +46,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) // Input array, could also just pass the dimensions mxGPUArray const * inputArray = mxGPUCreateFromMxArray(prhs[0]); + // mxGPUArray const * mex_EO = mxGPUCreateFromMxArray(prhs[1]); invTrim = (int *) mxGetData(prhs[1]); @@ -57,19 +58,22 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) // Assuming we are only doing either R2C or C2R mxComplexity output_data_type; - if ( input_type == mxREAL ) - { + if ( input_type == mxREAL ) { // mexPrintf("It is real bitches\n"); output_data_type = mxCOMPLEX ; pReal = (cufftReal *)(mxGPUGetDataReadOnly(inputArray)); } - else if (input_type == mxCOMPLEX) - { + else if (input_type == mxCOMPLEX) { fwd_xform = false; output_data_type = mxREAL ; pComplex = (cufftComplex *)(mxGPUGetDataReadOnly(inputArray)); } + else + { + mexErrMsgIdAndTxt("MATLAB:mexFFT:rhs", + "This inputArray is not real or complex."); + } if (nrhs > 2) { @@ -132,9 +136,11 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) + int xFormRank; int fft_dims[input_dims]; int batchSize; + if (input_dims > 2) { fft_dims[2] = (int) input_size[0]; fft_dims[1] = (int) input_size[1]; @@ -200,14 +206,33 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) // Make the arrays persistent mexMakeArrayPersistent(plhs[1]); mexMakeArrayPersistent(plhs[2]); - - cufftPlanMany(plan, xFormRank, fft_dims, - NULL, NULL, NULL, NULL, NULL, NULL, - CUFFT_R2C, 1); - cufftPlanMany(planInv, xFormRank, fft_dims, - NULL, NULL, NULL, NULL, NULL, NULL, - CUFFT_C2R, 1); + + + + if ( ! fwd_xform ) { + mexErrMsgIdAndTxt("The cufft plan must be created on a forward transform.","womp"); + } + + if (input_dims == 3) { + cufftPlan3d(plan, fft_dims[0], fft_dims[1], fft_dims[2], CUFFT_R2C); + cufftPlan3d(planInv, fft_dims[0], fft_dims[1],fft_dims[2], CUFFT_C2R); + } + else if (input_dims == 2) { + cufftPlan2d(plan, fft_dims[0], fft_dims[1], CUFFT_R2C); + cufftPlan2d(planInv, fft_dims[0], fft_dims[1], CUFFT_C2R); + } + else { + mexErrMsgIdAndTxt("The cufft plan must be created on a 1, 2, or 3 dimensional array.","womp"); + } + + + // cufftPlanMany(plan, xFormRank, fft_dims, + // NULL, NULL, NULL, NULL, NULL, NULL, + // CUFFT_R2C, 1); + // cufftPlanMany(planInv, xFormRank, fft_dims, + // NULL, NULL, NULL, NULL, NULL, NULL, + // CUFFT_C2R, 1); } @@ -229,8 +254,10 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) } } + // mxGPUCreateMxArrayOnGPU is used to wrap the output data in an MxArray that can be passed back to matlab. plhs[0] = mxGPUCreateMxArrayOnGPU(outputArray); + // mexPrintf("Afft_dimsress of plan is %d\n", *plan); // mexPrintf("Afft_dimsress of planInv is %d\n", *planInv); mxGPUDestroyGPUArray(inputArray); diff --git a/mexFiles/mexSF3D.cu b/mexFiles/mexSF3D.cu index 828d2f4f..479a22ce 100644 --- a/mexFiles/mexSF3D.cu +++ b/mexFiles/mexSF3D.cu @@ -92,7 +92,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) // TODO if these aren't typed properly in the mex call, I can't cast them appropriatley here // TODO general angles, here assume single-Y-axis tilt /* Check for proper number of arguments. TODO add checks on narg and types*/ - if ( nrhs != 14 & nrhs != 15 ) + if ( nrhs != 15 & nrhs != 16 ) { mexErrMsgIdAndTxt("MATLAB:mexSF3D:rhs", "requires 14 inputs."); @@ -113,6 +113,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) float* exposure = (float*) mxGetData(prhs[12]); float* occupancy = (float*) mxGetData(prhs[13]); int* launch = (int *) mxGetData(prhs[14]); // should be an int16 in matlab + float* wiener_constant = (float *) mxGetData(prhs[15]); float * d_output_img = NULL; float * d_ctf_img = NULL; @@ -252,7 +253,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) // Create the 2d ctf ctf<<< ctfGrid, ctfBlock ,0,cudaStreamPerThread >>>(d_ctf_img, ctf_dims, o_ctf_dims, b_ctf, fourierVoxelSize, - calc_centered, occupancy[iAng], exposure[iAng]); + calc_centered, occupancy[iAng], exposure[iAng], *wiener_constant); // Put the ctf in tex2 diff --git a/mexFiles/utils/ctf.cu b/mexFiles/utils/ctf.cu index aa2f020d..29dff9ef 100644 --- a/mexFiles/utils/ctf.cu +++ b/mexFiles/utils/ctf.cu @@ -66,7 +66,7 @@ __global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, flo // createb_ctf __global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, - bool calc_centered, float radial_weight, float total_exposure) + bool calc_centered, float radial_weight, float total_exposure, float wiener_constant) { @@ -112,19 +112,24 @@ __global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, flo radius_sq = radius_sq*radius_sq + tmp_coord*tmp_coord; - a[output_IDX] = sinf(b_ctf.cs_term*powf(radius_sq,2) - b_ctf.df_term*radius_sq*(b_ctf.defocus1 + b_ctf.defocus2 * cosf(2.0f * (phi-b_ctf.astigmatism_angle))) - b_ctf.amplitudeContrast); + tmp_coord = sinf(b_ctf.cs_term*powf(radius_sq,2) - b_ctf.df_term*radius_sq*(b_ctf.defocus1 + b_ctf.defocus2 * cosf(2.0f * (phi-b_ctf.astigmatism_angle))) - b_ctf.amplitudeContrast); // if you add the radial weighting you will need to fix this. - - if (b_ctf.doSqCTF) - { - // Is this any better (or worse) than pow? - a[output_IDX] *= a[output_IDX]; + if (wiener_constant > 0.f) { + tmp_coord *= tmp_coord; + tmp_coord /= (tmp_coord + wiener_constant); } + else { + if (b_ctf.doSqCTF) { + // Is this any better (or worse) than pow? + tmp_coord *= tmp_coord; + } + } + - a[output_IDX] *= radial_weight* expf( (-0.5f * total_exposure) / (kvScale *(expA * powf(radius_sq, expB) + expC))); + a[output_IDX] = tmp_coord * radial_weight* expf( (-0.5f * total_exposure) / (kvScale *(expA * powf(radius_sq, expB) + expC))); } diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index c1b584d4..b389b18b 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -720,7 +720,6 @@ for iCone = 1:nCones+1 - iCone fitFSC{iCone} = csape(shellsFreq(:,iCone),shellsFSC(:,iCone),'variational'); fitNUM{iCone} = csape(shellsFreq(:,iCone),shellsNUM(:,iCone)); end @@ -1026,7 +1025,6 @@ plot(osX,cRef{2}(osX),'k--'); for iCone = 3:length(cRef) - iCone plot(osX,cRef{iCone}(osX),'k--'); end end @@ -1055,71 +1053,71 @@ file_out = sprintf('%s-%d-cRefAli_%s', outputPrefix, iRef, halfSet); saveas(gcf, file_out,'pdf') - % try - fitPower = fit(shellsFreq(:,1).^2,log(shellsPOWER(:,1)),'cubicSpline'); - LR = 10; - MR = 7; - HR = min((1.05*fgold).^2, shellsFreq(end-1,1).^2); - if isempty(HR) - HR = lowCut1; - end - lowRes = find(shellsFreq(:,1).^2 > (1/LR)^2, 1,'first'); - midRes = find(shellsFreq(:,1).^2 > (1/MR)^2, 1,'first'); - endRes = find(shellsFreq(:,1).^2 > HR, 1,'first'); - plot1 = false; - plot2 = false; - if (midRes - lowRes > 2) - bFactorFIT1 = fit(shellsFreq(lowRes:midRes,1).^2 , ... - log(shellsPOWER(lowRes:midRes,1)),'poly1'); - plot1= true; - end - if (endRes-midRes > 2) - bFactorFIT2 = fit(shellsFreq(midRes:endRes,1).^2 , ... - log(shellsPOWER(midRes:endRes,1)),'poly1'); - plot2 = true; - end - - figure('Visible','off'), plot(osX.^2,fitPower(osX.^2),'k'); hold on; - if (plot1) - plot(osX.^2,bFactorFIT1(osX.^2),'b--'); - else - bFactorFIT1 = struct() - bFactorFIT1.('p1') = 0; - end - if (plot2) - plot(osX.^2,bFactorFIT2(osX.^2),'b--'); - else - bFactorFIT2 = struct() - bFactorFIT2.('p1') = 0; - end - line([(1/LR)^2,(1/LR)^2], ... - [min(log(shellsPOWER(:,1))), ... - max(log(shellsPOWER(:,1)))], ... - 'Color','k','LineStyle','--'); - line([(1/MR)^2,(1/MR)^2], ... - [min(log(shellsPOWER(:,1))), ... - max(log(shellsPOWER(:,1)))], ... - 'Color','k','LineStyle','--'); - line([HR,HR], ... - [min(log(shellsPOWER(:,1))), ... - max(log(shellsPOWER(:,1)))], ... - 'Color','k','LineStyle','--'); - % plot(osX.^2,bFactorFIT2(osX.^2),'b--'); - % outCurve(:,1).^2,outCurve(:,8),'g'); - - title({'Guinier Plot',sprintf('\nbFactor(%2.1f-%2.1f-%2.1f)\n %d,%d', ... - LR,MR,sqrt(1./HR),round(bFactorFIT1.p1*-4),... - round(bFactorFIT2.p1*-4))}); ... - xlabel('1/Ang^2'),... - ylabel('log(F)'); - ylim([0.95*min(log(shellsPOWER(:,1))),... - 1.05*max(log(shellsPOWER(:,1)))]) - - legend('uncorrected','corrected','Location','northeast',... - 'Orientation', 'vertical'); - file_out = sprintf('%s-%d-guinier_%s', outputPrefix, iRef, halfSet); - savefig(gcf,file_out); - saveas(gcf, file_out,'pdf') + % % try + % fitPower = fit(shellsFreq(:,1).^2,log(shellsPOWER(:,1)),'cubicSpline'); + % LR = 10; + % MR = 7; + % HR = min((1.05*fgold).^2, shellsFreq(end-1,1).^2); + % if isempty(HR) + % HR = lowCut1; + % end + % lowRes = find(shellsFreq(:,1).^2 > (1/LR)^2, 1,'first'); + % midRes = find(shellsFreq(:,1).^2 > (1/MR)^2, 1,'first'); + % endRes = find(shellsFreq(:,1).^2 > HR, 1,'first'); + % plot1 = false; + % plot2 = false; + % if (midRes - lowRes > 2) + % bFactorFIT1 = fit(shellsFreq(lowRes:midRes,1).^2 , ... + % log(shellsPOWER(lowRes:midRes,1)),'poly1'); + % plot1= true; + % end + % if (endRes-midRes > 2) + % bFactorFIT2 = fit(shellsFreq(midRes:endRes,1).^2 , ... + % log(shellsPOWER(midRes:endRes,1)),'poly1'); + % plot2 = true; + % end + + % figure('Visible','off'), plot(osX.^2,fitPower(osX.^2),'k'); hold on; + % if (plot1) + % plot(osX.^2,bFactorFIT1(osX.^2),'b--'); + % else + % bFactorFIT1 = struct() + % bFactorFIT1.('p1') = 0; + % end + % if (plot2) + % plot(osX.^2,bFactorFIT2(osX.^2),'b--'); + % else + % bFactorFIT2 = struct() + % bFactorFIT2.('p1') = 0; + % end + % line([(1/LR)^2,(1/LR)^2], ... + % [min(log(shellsPOWER(:,1))), ... + % max(log(shellsPOWER(:,1)))], ... + % 'Color','k','LineStyle','--'); + % line([(1/MR)^2,(1/MR)^2], ... + % [min(log(shellsPOWER(:,1))), ... + % max(log(shellsPOWER(:,1)))], ... + % 'Color','k','LineStyle','--'); + % line([HR,HR], ... + % [min(log(shellsPOWER(:,1))), ... + % max(log(shellsPOWER(:,1)))], ... + % 'Color','k','LineStyle','--'); + % % plot(osX.^2,bFactorFIT2(osX.^2),'b--'); + % % outCurve(:,1).^2,outCurve(:,8),'g'); + + % title({'Guinier Plot',sprintf('\nbFactor(%2.1f-%2.1f-%2.1f)\n %d,%d', ... + % LR,MR,sqrt(1./HR),round(bFactorFIT1.p1*-4),... + % round(bFactorFIT2.p1*-4))}); ... + % xlabel('1/Ang^2'),... + % ylabel('log(F)'); + % ylim([0.95*min(log(shellsPOWER(:,1))),... + % 1.05*max(log(shellsPOWER(:,1)))]) + + % legend('uncorrected','corrected','Location','northeast',... + % 'Orientation', 'vertical'); + % file_out = sprintf('%s-%d-guinier_%s', outputPrefix, iRef, halfSet); + % savefig(gcf,file_out); + % saveas(gcf, file_out,'pdf') % catch % fprintf('\nRan into some error in the guinier analysis.\n'); % fprintf('\nSince this is not critical, skipping and continue.\n'); diff --git a/statistics/BH_multi_cRef_Vnorm.m b/statistics/BH_multi_cRef_Vnorm.m index 50d3c6c9..896b186f 100755 --- a/statistics/BH_multi_cRef_Vnorm.m +++ b/statistics/BH_multi_cRef_Vnorm.m @@ -180,7 +180,6 @@ inc = 0.5 / (bin*pixelSize); for iCone = firstCone:nCones - iCone iFSCfit = csape(fscParams{1}(:,1),fscParams{2}(:,iCone+1),'variational'); diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index ac5d9a53..db613a26 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -194,6 +194,17 @@ shape_mask_threshold = 2.4 - 0.4; end +try + tmpVal = pBH.('whitenPS'); + if (numel(tmpVal) == 3) + wiener_constant = tmpVal(3); + else + error('flgWhitenPS should be a 3 element vector'); + end +catch + wiener_constant = 0.0; +end + try % Apply the mask with the given parameters, save and exit. shape_mask_test = pBH.('shape_mask_test'); @@ -437,17 +448,18 @@ sprintf('%s_pcaVolMask.mrc',outputPrefix)))); else -if (constrain_symmetry) - gridSearch = eulerSearch(symmetry,180,5,360,5,0.0,1,true); - [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter, ... - '3d', gridSearch.number_of_asymmetric_units); -else - [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); -end + if (constrain_symmetry) + gridSearch = eulerSearch(symmetry,180,5,360,5,0.0,1,true); + [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter, ... + '3d', gridSearch.number_of_asymmetric_units); + else + [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); + end + if ( flgPcaShapeMask ) % when combining the addition is harmless, but is a convenient way to % include when sets are left 100% separate. -% volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); + % volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, ... {'pca', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); @@ -749,7 +761,7 @@ % subset of peaks. FIXME includeParticle = positionList(iSubTomo, 8); - + iPeak=0; % make sure this exists if we are no including the particle if (includeParticle) make_sf3d = true; for iPeak = 0:nPeaks-1 @@ -767,9 +779,8 @@ radialGrid = ''; padWdg = [0,0,0;0,0,0]; [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, ... - TLT, center,reconGeometry); + TLT, center,reconGeometry, wiener_constant); -% wedgeMask = sqrt(wedgeMask - min(wedgeMask(:)) + 10^-6); end % If flgGold there is no change, otherwise temporarily resample the @@ -999,6 +1010,7 @@ U = cell(nScaleSpace,1); V = cell(nScaleSpace,1); S = cell(nScaleSpace,1); + sDiag = cell(nScaleSpace,1); coeffs = cell(nScaleSpace,1); varianceMap = cell(nScaleSpace,1); for iScale = 1:nScaleSpace @@ -1007,8 +1019,8 @@ % Calculate the decomposition [ U{iScale},S{iScale},V{iScale}, convergenceFlag ] = svds(double(dataMatrix{iScale}), ... maxEigs, 'largest', ... - 'MaxIterations',500, ... % default 300 - 'SubspaceDimension',max(krylovScalar*maxEigs,15),... % default max(3*maxEigs,15) + 'MaxIterations',1000, ... % default 300 + 'SubspaceDimension',max(krylovScalar*maxEigs,30),... % default max(3*maxEigs,15) 'Display',true); % Diagnostics default false (will this work in compiled?) U{iScale} = single(U{iScale}); S{iScale} = single(S{iScale}); @@ -1016,13 +1028,13 @@ % [U{iScale},S{iScale},V{iScale}] = svd(dataMatrix{iScale}, 0); - - numNonZero = find(( diag(S{iScale}) ~= 0 ), 1, 'last'); + sDiag{iScale} = diag(S{iScale}); + numNonZero = find(( sDiag{iScale} ~= 0 ), 1, 'last'); % For Method 1, save eigenvectors 1-4 (or user-specified max) as images eigsFound = min(maxEigs, numNonZero); - fprintf('Found %d / %d non-zero eigenvalues in set %s.\n All singular values converged is t/f ( %d ) ', ... - numNonZero, size(S{iScale}, 1),halfSet, convergenceFlag); + fprintf('Found %d / %d non-zero eigenvalues sum = %4.4f, in set %s.\n All singular values converged is t/f ( %d ) ', ... + numNonZero, size(S{iScale}, 1), sum(sDiag{iScale}), halfSet, convergenceFlag); coeffs{iScale} = S{iScale} * V{iScale}' @@ -1103,12 +1115,12 @@ end end - save(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet), 'nTOTAL', 'coeffs','idxList','peakList'); + save(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet), 'nTOTAL', 'coeffs','idxList','peakList', 'sDiag'); else if (randomSubset) save(sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet),'U', 'idxList','peakList'); else - save(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet), 'nTOTAL', 'coeffs','idxList','peakList'); + save(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet), 'nTOTAL', 'coeffs','idxList','peakList', 'sDiag'); end end diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index c3b577dc..5bfe7a7e 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -318,6 +318,7 @@ [tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); + if (flgAltRun && ~skip_to_the_end_and_run) nParts = ceil(nTiltSeries ./ cycle_denominator); diff --git a/testScripts/fourierTransformer.m b/testScripts/fourierTransformer.m index 8a1af660..0956eca9 100644 --- a/testScripts/fourierTransformer.m +++ b/testScripts/fourierTransformer.m @@ -6,6 +6,7 @@ bandpass; % for now create one internally and just chop off half. inputSize = ''; + phaseSwapOffset = ''; is2d; halfDim; halfDimSize; @@ -56,6 +57,8 @@ obj.inputSize = size(inputVol); obj.halfDim = 1; obj.halfDimSize = floor(obj.inputSize(1)/2)+1; + + obj.phaseSwapOffset = mod(obj.inputSize,2); @@ -153,6 +156,7 @@ [ ft ] = mexFFT(inputVol,obj.invTrim,obj.plan_FWD, obj.plan_INV); + % For some reason calling this is OUTRAGEOUSLY slow, add option to jsut pass it in to the mexFFT if (doNorm) ft = ft .* (obj.normalization_factor^doNorm); end @@ -195,29 +199,32 @@ function delete(obj) function [inputVol] = swapPhase(obj, inputVol, direction) + % Create the phase swap indices if needed if isempty(obj.phaseCenter) if obj.is2d [ obj.phaseCenter, dV ] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... {'none'},1,0,0,{'halfgrid'}); if (obj.inputSize(1) == obj.inputSize(2)) - obj.phaseCenter = exp(-2i.*pi.*(obj.halfDimSize-obj.OddSizeOversampled).*(obj.phaseCenter+dV)); + sx = obj.halfDimSize-1+obj.phaseSwapOffset(1); + obj.phaseCenter = exp(-2i.*pi.*sx.*(obj.phaseCenter+dV)); clear dU dV else - hX = floor(obj.inputSize(1)/2) + 1; - hY = floor(obj.inputSize(2)/2) + 1; + hX = floor(obj.inputSize(1)/2) + obj.phaseSwapOffset(1); + hY = floor(obj.inputSize(2)/2) + obj.phaseSwapOffset(2); obj.phaseCenter = exp(-2i.*pi.*(hX.*obj.phaseCenter+hY.*dV)); end else [ obj.phaseCenter, dV, dW] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... {'none'},1,0,0,{'halfgrid'}); - if (obj.inputSize(1) == obj.inputSize(2) == obj.inputSize(3)) - obj.phaseCenter = exp(-2i.*pi.*(obj.halfDimSize-obj.OddSizeOversampled).*(obj.phaseCenter+dV+dW)); + if ((obj.inputSize(1) == obj.inputSize(2)) && (obj.inputSize(2) == obj.inputSize(3))) + sx = obj.halfDimSize-1+obj.OddSizeOversampled; + obj.phaseCenter = exp(-2i.*pi.*sx.*(obj.phaseCenter+dV+dW)); clear dU dV dW else - hX = floor(obj.inputSize(1)/2);% + 1; - hY = floor(obj.inputSize(2)/2);% + 1; - hZ = floor(obj.inputSize(3)/2);% + 1; + hX = floor(obj.inputSize(1)/2) + obj.phaseSwapOffset(1); + hY = floor(obj.inputSize(2)/2) + obj.phaseSwapOffset(2); + hZ = floor(obj.inputSize(3)/2) + obj.phaseSwapOffset(3); obj.phaseCenter = exp(-2i.*pi.*(hX.*obj.phaseCenter+hY.*dV+hZ.*dW)); clear dU dV dW end diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 3fba6136..66393e9c 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -114,11 +114,21 @@ nPeaks = 1; end +try + tmpVal = pBH.('whitenPS'); + if (numel(tmpVal) == 3) + wiener_constant = tmpVal(3); + else + error('flgWhitenPS should be a 3 element vector'); + end +catch + wiener_constant = 0.0; +end try fscBfactor = pBH.('Fsc_bfactor'); catch - fscBfactor = 0; + fscBfactor = 40; end mapBackIter = subTomoMeta.currentTomoCPR; @@ -1252,16 +1262,17 @@ angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); wdgIDX = positionList(iSubTomo,9); - if (flgFinalAvg) - angles = reshape(angles,3,3)*oddRot; - end + % tmpang = BH_defineMatrix([0,0,-14],'Bah','inv'); + % angles = reshape(angles,3,3)*tmpang; + % if (flgFinalAvg) + % angles = reshape(angles,3,3)*oddRot; + % end TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); if (make_sf3d) if (use_v2_SF3D) - [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, ... - center,reconGeometry); + [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); else iSF3D = gpuArray(wedgeMask{wdgIDX}); end From 6ec28818c7a7a311701c291f2c3a734ad5e8c67e Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 18 Jan 2024 17:55:01 -0500 Subject: [PATCH 002/151] Hella Wip --- alignment/BH_alignRaw3d_v2.m | 3 +- alignment/BH_templateSearch3d_2.m | 2 +- ctf/BH_ctf_Correct.m | 14 +- ctf/BH_ctf_Correct3d.m | 138 ++--- ctf/BH_ctf_Estimate.m | 84 +-- ctf/BH_ctf_Refine2.m | 682 ++++--------------------- ctf/BH_ctf_Updatefft.m | 668 +++++++++++------------- masking/alignmentVol.m | 281 ++++++++++ metaData/BH_geometryAnalysis.m | 22 +- statistics/BH_clusterPub.m | 7 +- statistics/BH_fscGold_class.m | 2 +- statistics/BH_pcaPub.m | 219 +++++--- testScripts/fourierTransformer.m | 26 +- testScripts/mCompile.sh | 5 +- transformations/BH_average3d.m | 130 +++-- transformations/BH_multi_loadOrBuild.m | 12 +- 16 files changed, 1034 insertions(+), 1261 deletions(-) create mode 100644 masking/alignmentVol.m diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 9c70efc4..085a3a76 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -161,6 +161,7 @@ flgCenterRefCOM = 1; end +% FIXME: unused, fix experimental options option try flgSymmetrizeSubTomos = pBH.('flgSymmetrizeSubTomos'); catch @@ -284,7 +285,7 @@ % make sure the number of references match the unique groups in the classVector % and also that the class/group pairs match the class/ref pairs. nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})] +nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]'' nRefOut(1:2) = [length(unique(refGroup{1})) + sum(( refSym{1} < 0 )),... diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index f6ff478a..4af18a8d 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -22,7 +22,7 @@ tomoNumber = EMC_str2double(tomoNumber); -[ useGPU ] = BH_multi_checkGPU( gpuIDX ) +[ useGPU ] = BH_multi_checkGPU( gpuIDX ); diff --git a/ctf/BH_ctf_Correct.m b/ctf/BH_ctf_Correct.m index 94afc669..6678cd35 100755 --- a/ctf/BH_ctf_Correct.m +++ b/ctf/BH_ctf_Correct.m @@ -71,16 +71,10 @@ STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - try - % make sure there isn't a refined version first. - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',STACK_PRFX,mapBackIter+1); - TLT = load(TLTNAME) - fprintf('using refined TLT %s\n', TLTNAME); - catch - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - end + + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1); + TLT = load(TLTNAME); + fprintf('using TLT %s\n', TLTNAME); inputStack = sprintf('aliStacks/%s_ali%d.fixed',STACK_PRFX,mapBackIter+1); outputStack = sprintf('ctfStacks/%s_ali%d_ctf.fixed',STACK_PRFX,mapBackIter+1); diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index bb1afe7c..347e5523 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -76,14 +76,14 @@ recWithoutMat = true; if length(varargin) > 2 % Full recon for tomoCPR - bh_global_turn_on_phase_plate = 0 - filterProjectionsForTomoCPRBackground = 28 + bh_global_turn_on_phase_plate = 0; + filterProjectionsForTomoCPRBackground = 28; else loadSubTomoMeta = false; % Default to on for subregion picking % If user has specified phakePhasePlate, don;t use ...otherwise if isempty(bh_global_turn_on_phase_plate(1)) || bh_global_turn_on_phase_plate(1) == 0 - bh_global_turn_on_phase_plate = [1,2] + bh_global_turn_on_phase_plate = [1,2]; end end end @@ -116,9 +116,9 @@ catch end -if (bh_global_turn_on_phase_plate(1) && flgWhitenPS(1)) +if (bh_global_turn_on_phase_plate(1) && any(flgWhitenPS)) fprintf('WARNING: phakePhasePlate and whitening are conflicting preocesses. Turning off whitening.\n') - flgWhitenPS(1) = 0; + flgWhitenPS = [0,0,0]; end @@ -379,9 +379,7 @@ % Divide the tilt series up over each gpu iterList = cell(nGPUs,1); % If there is only one tilt, things break in a weird way -nGPUs -nTilts -nGPUs = min(nGPUs, nTilts) +nGPUs = min(nGPUs, nTilts); for iGPU = 1:nGPUs iterList{gpuList(iGPU)} = iGPU+(tiltStart-1):nGPUs:nTilts; iterList{gpuList(iGPU)}; @@ -402,16 +400,9 @@ % For now, since the tilt geometry is not necessarily updated (it is manual) % in the masterTM, check that newer (possible perTilt refined) data is % not present. - try - % make sure there isn't a refined version first. - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',tiltList{iTilt},mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using refined TLT %s\n', TLTNAME); - catch - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - end + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); + TLT = load(TLTNAME); + fprintf('using TLT %s\n', TLTNAME); % Get all the tomogram names that belong to a given tilt-series. @@ -489,19 +480,11 @@ iTomoList = cell(nTomos,1); - % For now, since the tilt geometry is not necessarily updated (it is manual) - % in the masterTM, check that newer (possible perTilt refined) data is - % not present. - try - % make sure there isn't a refined version first. - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',tiltList{iTilt},mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using refined TLT %s\n', TLTNAME); - catch - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - end + + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); + TLT = load(TLTNAME); + fprintf('using TLT %s\n', TLTNAME); + if (~recWithoutMat) @@ -693,12 +676,7 @@ % like there is something odd about its use with a parfor loop % FIXME, when setting up the iterator, make clean copies for each % worker that are local in scope.e - -% This is slated to be deleted, just leave pre erasure to happen in ctf estimate/update -% if ~(flgEraseBeads_aferCTF) -% scalePixelsBy = samplingRate; -% maskedStack = BH_eraseBeads(maskedStack,eraseRadius, tiltList{iTilt}, scalePixelsBy,mapBackIter,sortrows(TLT,1)); -% end + [ correctedStack ] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... avgZ,TLT,pixelSize,maskedStack,... @@ -745,6 +723,8 @@ TA = TA(:,4); else if (mapBackIter) + % FIXME: I don't think this block should work, it should only be the tilt angles! + error('THis block should not be reached.') TA = load(sprintf('%smapBack%d/%s_ali%d_ctf.tlt',CWD,mapBackIter,tiltList{iTilt},... mapBackIter)); else @@ -765,17 +745,6 @@ LOCAL = sprintf('%sfixedStacks/%s.local',CWD,tiltList{iTilt}); end - % Put a local copy if using a nondefault cache -% if ( flgCleanCache ) -% sprintf('cp %s/%s %s/%s',CWD,rawTLT,tmpCache,rawTLT) -% sprintf('cp %s/%s %s/%s',CWD,LOCAL,tmpCache,LOCAL) -% system(sprintf('cp %s/%s %s/%s',CWD,rawTLT,tmpCache,rawTLT)); -% system(sprintf('cp %s/%s %s/%s',CWD,LOCAL,tmpCache,LOCAL)); -% rawTLT = sprintf('%s/%s',tmpCache,rawTLT) -% LOCAL = sprintf('%s/%s',tmpCache,LOCAL) -% -% end - fprintf('Local file %s\n',LOCAL); @@ -1336,17 +1305,17 @@ if (useSurfaceFit) - %rZ = (surfaceFit.p00 + surfaceFit.p10.*(rX+oX)) + surfaceFit.p01.*(rY+oY); - try - rZ = feval(surfaceFit,rX,rY); - catch - d1 - d2 - rX - rY - surfaceFit - error('surface fit failed'); - end + %rZ = (surfaceFit.p00 + surfaceFit.p10.*(rX+oX)) + surfaceFit.p01.*(rY+oY); + try + rZ = feval(surfaceFit,rX,rY); + catch + d1 + d2 + rX + rY + surfaceFit + error('surface fit failed'); + end else rZ = zeros([d1,d2],'single','gpuArray'); end @@ -1373,41 +1342,38 @@ % To track sampling in case I put in overlap samplingMask = zeros([d1,d2],'single','gpuArray'); - for iDefocus = minDefocus-ctf3dDepth/1:ctf3dDepth/1:maxDefocus+ctf3dDepth/1 -% fprintf('correcting for iDefocus %3.3e\n',iDefocus); - %search tz take those xy and add to the prj and mask + defVect = [iDefocus - ddF, iDefocus + ddF, dPhi]; + + if (phakePhasePlate(1) > 0) + + if numel(phakePhasePlate) == 2 + modPower = floor(phakePhasePlate(2)); + SNR = rem(phakePhasePlate(2),1); + else + modPower = 1; + SNR = 1; + end - defVect = [iDefocus - ddF, iDefocus + ddF, dPhi]; - - if (phakePhasePlate(1) > 0) - if numel(phakePhasePlate) == 2 - modPower = floor(phakePhasePlate(2)); - SNR = rem(phakePhasePlate(2),1); - else - modPower = 1; - SNR = 1; - end - - [Hqz, ~] = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,1,SNR); + [Hqz, ~] = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,1,SNR); + + Hqz = (-1).^modPower.*(phakePhasePlate(1).*Hqz).^1; - Hqz = (-1).^modPower.*(phakePhasePlate(1).*Hqz).^1; - - modHqz = []; + modHqz = []; + else + if PIXEL_SIZE < 2.0e-10 + % use double precision - this is not enabled, but needs to be - + % requires changes to radial grid as well. + Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,-1); else - if PIXEL_SIZE < 2.0e-10 - % use double precision - this is not enabled, but needs to be - - % requires changes to radial grid as well. - Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,-1); - else - Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1); - end - end + Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1); + end + end - + if (flgWhitenPS(3)) tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz./(abs(Hqz).^2+flgWhitenPS(3)))),trimVal(1,:),trimVal(2,:),'GPU','single'); else diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 3a392414..d749d96f 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -90,13 +90,7 @@ end end -% If true, then parameters will be adjusted to make this initial estimate -% faster, since it is less critical to be exact. -try - do_ctf_refine = pBH.('skip_ctf_refine'); -catch - do_ctf_refine = true; -end + PIXEL_SIZE = pBH.('PIXEL_SIZE'); Cs = pBH.('Cs'); @@ -183,11 +177,7 @@ try deltaZTolerance = pBH.('deltaZTolerance'); catch - if (do_ctf_refine) - deltaZTolerance = 50e-9; - else - deltaZTolerance = 100e-9; - end + deltaZTolerance = 100e-9; end try @@ -203,11 +193,9 @@ try maxNumberOfTiles = pBH.('ctfMaxNumberOfTiles'); catch - if (do_ctf_refine) - maxNumberOfTiles = 4000; - else - maxNumberOfTiles = 10000; - end + + maxNumberOfTiles = 10000; + end % Starting at +/- 100nm @@ -783,16 +771,10 @@ for iTilt = 1:3 if (skipFitting) - currentDefocusEst = defEST; - + currentDefocusEst = defEST; % Add the determined defocus, and write out with mic paramters as well. TLT(:,15) = repmat(-1.*defEST,size(TLT,1),1); -% if (flgAstigmatism) && (refineCCC(c,3)~=-9999) -% TLT(:,12) = repmat(gather(refineCCC(c,2)),size(TLT,1),1); -% TLT(:,13) = repmat(gather(refineCCC(c,1)),size(TLT,1),1); -% end - [~, idx] = sortrows(abs(TLT(:,4)), -1); TLT = TLT(idx,:); @@ -809,16 +791,11 @@ end radialAvg = [rotAvgPowerSpec((paddedSize/2)+1:end,(paddedSize/2)+1,iTilt)]'; -% radialPS = [AvgPowerSpec((paddedSize/2)+1:end,(paddedSize/2)+1,iTilt)]'; defRange = [currentDefocusEst-currentDefocusWin,currentDefocusEst+currentDefocusWin]; -% % % if defRange(2) > -0.05 -% % % fprintf('\n\nCapping defocus to 50nm from wanted %f. Do you mean to search so close to focus??\n\n',abs(defRange(2))); -% % % defRange(2) = -0.05; -% % % end - defInc = [0.01]; + defInc = 0.01; defVal = (defRange(1):defInc:defRange(2))'; @@ -829,7 +806,6 @@ DF = iDF*10^-6; % TODO add a global switch for the damping -% [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); if (PIXEL_SIZE < 1*10^-10) [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0,-1); @@ -854,7 +830,7 @@ end [~,maxVal] = max(cccStorage(:,2)); - maxDef = cccStorage(maxVal,1) + maxDef = cccStorage(maxVal,1); if (iTilt == 1) % Only save for the "true" defocus at the tilt-axes @@ -884,14 +860,9 @@ end - % [ diagnosticIMG ] = make_diagnosticIMG( Hqz, pixelOUT, bandpass, bg, {rotAvgPowerSpec}); - % - % SAVE_IMG(MRCImage(diagnosticIMG), ... - % sprintf('%s/ctf/%s_diag%s',pathName,fileName,extension)); + clear STACK exposureFilter - clear STACK exposureFilter - - pdfOUT = sprintf('%s/ctf/%s_psRadial_%d.pdf',pathName,stackNameOUT,iTilt) + pdfOUT = sprintf('%s/ctf/%s_psRadial_%d.pdf',pathName,stackNameOUT,iTilt); bgSubPS = (abs(radialAvg) - bg(freqVector)').*bandpass; @@ -955,13 +926,8 @@ [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0); end - % [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff ,... - % freqVector, radialPS, radialAstig); - - - [ iCCC ] = calc_CCC( radialAstig, bgSubPS, bandpass, AvgPowerSpec, Hqz, cccScale) - + [ iCCC ] = calc_CCC( radialAstig, bgSubPS, bandpass, AvgPowerSpec, Hqz, cccScale); initAstigCCC(n,:) = [iAng,iDelDF*astigStep,iCCC]; n = n + 1; @@ -969,7 +935,6 @@ end end - nPeaks = 1; top3 = zeros(nPeaks,3,'gpuArray'); @@ -979,8 +944,6 @@ initAstigCCC = initAstigCCC(initAstigCCC(:,1)~=initAstigCCC(c,1),:); end - top3 - refineCCC = zeros(length(fineDefSearch)*length(fineAngSearch)*nPeaks,3,'gpuArray'); n=1; @@ -998,23 +961,23 @@ % values |df1| < |df2| which is against convention. if abs(df1) >= abs(df2) - if (PIXEL_SIZE < 1*10^-10) + if (PIXEL_SIZE < 1*10^-10) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng+mAng], ... - size(radialForCTF{1}), -AMPCONT,-1.0,-1); - else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng+mAng], ... - size(radialForCTF{1}), -AMPCONT,-1.0); - end + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [df1,df2,iAng+mAng], ... + size(radialForCTF{1}), -AMPCONT,-1.0,-1); + else + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [df1,df2,iAng+mAng], ... + size(radialForCTF{1}), -AMPCONT,-1.0); + end [ iCCC ] = calc_CCC( radialAstig,bgSubPS, bandpass, ... - AvgPowerSpec, Hqz,cccScale ) + AvgPowerSpec, Hqz,cccScale ); else - iCCC = -9999 + iCCC = -9999; end refineCCC(n,:) = [iAng+mAng,mDef + iDelDF,iCCC]; @@ -1106,11 +1069,8 @@ end % end flgSkip -if (do_ctf_refine) % TODO should I restart the parallel pool BH_ctf_Refine2(varargin{1},varargin{2}); - -end end % end of ctf estimate function diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index abb5c151..7fecf1f5 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -40,18 +40,14 @@ gpuIDX = BH_multi_checkGPU(-1); gDev = gpuDevice(gpuIDX); -flgAstigmatism= 1; -flgGroupProjections =1;% pBH.('flgGroupProjections'); -calcAvg = 1;%pBH.('flgCalcAvg'); -outputForCTFFIND = 1; + reScaleRealSpace = 0; -normFactor=0;%pBH.('normalizationFactor'); PRJ_STACK = {sprintf('aliStacks/%s_ali%d.fixed',STACK_PRFX,mapBackIter+1)}; -[pathName,fileName,extension] = fileparts(PRJ_STACK{1}) +[pathName,fileName,extension] = fileparts(PRJ_STACK{1}); if isempty(pathName) - pathName = '.' + pathName = '.'; end PIXEL_SIZE = pBH.('PIXEL_SIZE'); @@ -59,10 +55,9 @@ Cs = pBH.('Cs'); VOLTAGE = pBH.('VOLTAGE'); -AMPCONT = pBH.('AMPCONT') - -ctfParams = [PIXEL_SIZE*10^10,VOLTAGE./1000,Cs.*1000,AMPCONT] +AMPCONT = pBH.('AMPCONT'); +ctfParams = [PIXEL_SIZE*10^10,VOLTAGE./1000,Cs.*1000,AMPCONT]; % Sanity check @@ -78,7 +73,7 @@ WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; end -if Cs == 0 +if (Cs == 0) fprintf('You set Cs to zero, over-riding to 5 micron\n'); Cs = 5e-6; end @@ -86,10 +81,6 @@ % Assuming that the first CTF zero is always less than this value FIXED_FIRSTZERO = PIXEL_SIZE / 40*10^-10 ; -% highCutoff = PIXEL_SIZE/pBH.('defCutOff'); -highCutoff = 1/pBH.('defCutOff'); - - % Size to padTile to should be even, large, and preferably a power of 2 try paddedSize = pBH.('paddedSize'); @@ -102,15 +93,9 @@ tileSize = floor(680e-10 / PIXEL_SIZE); tileSize = tileSize + mod(tileSize,2); fprintf('Using a tile size of %d',tileSize); -overlap = floor(tileSize ./ tileOverlap) +overlap = floor(tileSize ./ tileOverlap); - -% Starting at +/- 750nm -deltaZTolerance = 750e-9 / PIXEL_SIZE; -% Use to check for proper gradient. -zShift = 0; - inc = (0.5 - FIXED_FIRSTZERO) / (paddedSize/2); freqVector = [inc+FIXED_FIRSTZERO:inc:0.5 ]; @@ -119,32 +104,31 @@ %PRJ_OUT = {fileName}; nStacks = length(tlt); - stacksFound = [] +stacksFound = []; INPUT_CELL = cell(nStacks,6); -for i = 1:nStacks - if exist(tlt{i}, 'file') && exist(PRJ_STACK{i}, 'file') - INPUT_CELL{i,1} = load(tlt{i}); - INPUT_CELL{i,2} = PRJ_STACK{i}; - [pathName,fileName,extension] = fileparts(PRJ_STACK{i}); +for iStack = 1:nStacks + if exist(tlt{iStack}, 'file') && exist(PRJ_STACK{iStack}, 'file') + INPUT_CELL{iStack,1} = load(tlt{iStack}); + INPUT_CELL{iStack,2} = PRJ_STACK{iStack}; + [pathName,fileName,extension] = fileparts(PRJ_STACK{iStack}); if isempty(pathName) pathName = '.'; end - INPUT_CELL{i,3} = pathName; - INPUT_CELL{i,4} = fileName; - INPUT_CELL{i,5} = extension; - %INPUT_CELL{i,6} = PRJ_OUT; - + INPUT_CELL{iStack,3} = pathName; + INPUT_CELL{iStack,4} = fileName; + INPUT_CELL{iStack,5} = extension; + %INPUT_CELL{iStack,6} = PRJ_OUT; + else - fprintf('ignoring %s, because the file is not found.\n', tlt{i}) + fprintf('ignoring %s, because the file is not found.\n', tlt{iStack}) end end for iStack = 1%stacksFound - iStack - + STACK = single(getVolume(MRCImage(INPUT_CELL{iStack,2}))); % The pixel size should be previously set correctly, but if it is not, then we % must maintain whatever is there in case beads are to be erased. The model @@ -153,55 +137,43 @@ [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); iPixelHeader = EMC_str2double(iPixelHeader); [d1,d2,d3] = size(STACK) - - - + + + TLT = INPUT_CELL{iStack,1}; - pathName = INPUT_CELL{iStack,3} - fileName = INPUT_CELL{iStack,4} - extension = INPUT_CELL{iStack,5} + pathName = INPUT_CELL{iStack,3}; + fileName = INPUT_CELL{iStack,4}; + extension = INPUT_CELL{iStack,5}; - SIZEOUT = [d1,d2]; - - - - [radialForCTF,phi,~,~,~,~] = ... - BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU',{'none'},1,1,0); + BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU',{'none'},1,1,0); radialForCTF = {radialForCTF./PIXEL_SIZE,1,phi} ; - clear phi - - -if (calcAvg) -% [exposureFilter] = BH_exposureFilter(paddedSize.*[1,1], TLT,'cpu',1, 1); - - - clear sumVector radialAvg - sumVector(length(freqVector)) = gpuArray(double(0)); - radialAvg(length(freqVector)) = gpuArray(double(0)); - - tic + clear phi + clear sumVector radialAvg + sumVector(length(freqVector)) = gpuArray(double(0)); + radialAvg(length(freqVector)) = gpuArray(double(0)); - psTile = zeros([paddedSize,paddedSize,d3],'single'); + tic + psTile = zeros([paddedSize,paddedSize,d3],'single'); - flgReplaceStack = 0; - for iPrj = 1:d3 - iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); - iProjection = iProjection - ... - BH_movingAverage(iProjection,[tileSize,tileSize]); - iProjection = iProjection ./ ... - BH_movingRMS(iProjection,[tileSize,tileSize]); - % Taking a cue from Alexis - maxPixelSizeWanted = 2.0e-10; - if TLT(iPrj,16) < maxPixelSizeWanted + flgReplaceStack = 0; + for iPrj = 1:d3 + iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); + iProjection = iProjection - ... + BH_movingAverage(iProjection,[tileSize,tileSize]); + iProjection = iProjection ./ ... + BH_movingRMS(iProjection,[tileSize,tileSize]); + % Taking a cue from Alexis + maxPixelSizeWanted = 2.0e-10; + if TLT(iPrj,16) < maxPixelSizeWanted %fprintf(ftmp,'Resampling pixel size\n'); % Resample to 2Ang/pix padSq = BH_multi_padVal(size(iProjection),max(size(iProjection)).*[1,1]); @@ -232,8 +204,8 @@ else pixelSize = TLT(iPrj,16); end - end - + end % iPrj 1:d3 + if ( flgReplaceStack ) STACK = newSTACK ; clear newSTACK; end @@ -248,21 +220,15 @@ coordShift = (-1).^(X+Y); clear X Y - % with > 250,000 tiles, it doesn't make sense to call resample2d for just a - % simple rescaling. - - try - ppool = EMC_parpool(nWorkers); + ppool = EMC_parpool(nWorkers); catch delete(gcp('nocreate')); ppool = EMC_parpool(nWorkers); end for iPrj = 1:d3 - - pFuture(iPrj) = parfeval(ppool,@runAvgTiles,2, TLT, paddedSize, tileSize, ... d1,d2, iPrj, overlap, ... @@ -271,26 +237,24 @@ 1, ... x1, y1, Xnew, Ynew,coordShift, ... reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); - + end - - for i = 1:d3 - fprintf('Refining defocus on prj %d/ %d\n',i,d3); + + for iWorker = 1:d3 + fprintf('Refining defocus on prj %d/ %d\n',iWorker,d3); [iPrj, ctfCorr,pixelSize] = fetchNext(pFuture); psTile(:,:,TLT(iPrj,1)) = ctfCorr; - - end - %%pixelSize = PIXEL_SIZE*10^10; + pixelSize = pixelSize*10^10; SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS.mrc',fileName),pixelSize); bpLog = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); bpLog = bpLog > 0.99; bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); - + for iPrj = 1:d3 iTile = gpuArray(psTile(:,:,iPrj)); iTile = iTile.*bp.*bp2; @@ -298,425 +262,17 @@ psTile(:,:,iPrj) = gather(iTile); end + SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName),pixelSize); delete(ppool); -else - - psTile = gpuArray(getVolume(MRCImage(sprintf('fixedStacks/ctf/%s-PS.mrc',fileName)))); - end - - if (calcAvg) delete(gcp('nocreate')) - end - - %%%%%%%%%%%%%%%%%%%%%%%%% - if ( outputForCTFFIND ) - % exit an fit the PS using CTFFIND4 - BH_runCtfFind(sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName), ... - sprintf('%s_ctf.tlt',fileName), ctfParams,TLT) - return - end - %%%%%%%%%%%%%%%%%%%%%%%%% - defInc = cell(3,1); astInc = cell(3,1); - defRange = cell(3,1); astRange = cell(3,1); - defSearch = cell(3,d3); astSearch = cell(3,1); - - % Find a close value for symmetric defocus. Do this on a rotationally averaged - % image so that the value is centered between the astigmatic extremes rather - % than potentially sitting at one end or the other. - % Search around this value to get a ballpark on astigmatism. - % Use this astigmatic value to refine symmetric results, and this to refine - % astigmatic results. - % Lather rinse and repeat 1x. - - - defInc{1} = 25*10^-9; - defRange{1} = 1000 *10^-9; - - defInc{2} = 10*10^-9; - defRange{2} = 100*10^-9; - - defInc{3} = 5 *10^-9; - defRange{3} = 50*10^-9; - - maxAstig = 200*10^-9; - astigStep =10*10^-9; - coarseAngStep = (pi/180)*10; - - astigDefSearch{1} = 0:astigStep:maxAstig; -% astigAngSearch{1} = -pi/2:coarseAngStep:pi/2; - astigAngSearch{1} = 0:coarseAngStep:pi; - - astigDefSearch{2} = -5*astigStep:astigStep/2:astigStep*5; - astigAngSearch{2} = -2*coarseAngStep:coarseAngStep/5:coarseAngStep*2; - - astigDefSearch{3} = -3*astigStep:astigStep/4:astigStep*3; - astigAngSearch{3} = -coarseAngStep/2:coarseAngStep/20:coarseAngStep/2; - - astigCCC = cell(3,d3); - for iSearch = 1:3 - for iPrj = 1:d3 - astigCCC{iSearch}{iPrj} = zeros(length(astigDefSearch{iSearch})* ... - length(astigAngSearch{iSearch}),3, 'gpuArray'); - end - end - - rotationalAvg = 1; - if (rotationalAvg) - rotBgSubPS = zeros(size(psTile),'single','gpuArray'); - end - - bgSubPS = zeros(size(psTile),'single','gpuArray'); - minRes = calcMinResolution(TLT, radialForCTF, Cs,WAVELENGTH,AMPCONT); - fprintf('\nMin Resolution fit is %3.3f Angstrom.\n',minRes); - nBgPix = floor(paddedSize.*PIXEL_SIZE.*10^10*sqrt(2)/minRes); - nBgPix = nBgPix + mod(nBgPix,2) - if (normFactor) - nRMSpix = floor(nBgPix/normFactor) + mod(nBgPix/normFactor,2); - end - cccResults = zeros(d3,1); -% - - [rot1, rot2, ~, r1,r2, ~] = BH_multi_gridCoordinates(paddedSize.*[1,1], ... - 'Cartesian','GPU', ... - {'none'},0,1,0); - - if ( flgGroupProjections ) - % Create a temporary copy of the tiles scaled by the sin of the tilt angle - % which results in stronger averaging at high tilts - scaledTile = zeros(size(psTile),'single'); - - % Since they will be averaged, first center and scale the total per - % prj intensities - for iPrj = 1:d3 - psTile(:,:,iPrj) = psTile(:,:,iPrj) - mean(mean(psTile(:,:,iPrj))); - psTile(:,:,iPrj) = psTile(:,:,iPrj) ./ rms(rms( psTile(:,:,iPrj))); - end - - for iPrj = 1:d3 - scaledTile(:,:,TLT(iPrj,1)) = scaledTile(:,:,TLT(iPrj,1)) .* ... - abs(sind(TLT(iPrj,4)))+0.05; - end - - end - - for iPrj = 1:d3 - - if (flgGroupProjections) - - % Take the full value at the projection of interest, sin(tiltangle) for the - if iPrj == 1 - gTMP = psTile(:,:,1) + scaledTile(:,:,2) + scaledTile(:,:,3); - elseif iPrj == d3 - gTMP = psTile(:,:,d3) + scaledTile(:,:,d3-1) + scaledTile(:,:,d3-2) - else - gTMP = psTile(:,:,iPrj)+ scaledTile(:,:,iPrj-1) + scaledTile(:,:,iPrj+1); - end - else - - gTMP = psTile(:,:,iPrj); - end - -% % Take the full value at the projection of interest, sin(tiltangle) for the -% if iPrj == 1 -% gTMP=(psTile(:,:,1) + ... -% psTile(:,:,2).*0.55 +... -% psTile(:,:,3).*0.25)./1.9; -% elseif (iPrj > 1 && iPrj < 7) || (iPrj > d3-6 && iPrj < d3) -% gTMP = (psTile(:,:,iPrj-1).*0.4+... -% psTile(:,:,iPrj)+... -% psTile(:,:,iPrj+1).*0.4)./1.8; -% elseif (iPrj >= 7 && iPrj < 18) -% gTMP = (psTile(:,:,iPrj) + ... -% psTile(:,:,iPrj+1).*0.3)./1.3; -% elseif (iPrj > d3-17 && iPrj <= d3-6) -% gTMP = (psTile(:,:,iPrj-1).*0.3+... -% psTile(:,:,iPrj))./1.3; -% elseif iPrj == d3 -% gTMP = (psTile(:,:,d3-2).*0.25+... -% psTile(:,:,d3-1).*0.55+... -% psTile(:,:,d3))./1.8; -% else -% gTMP = psTile(:,:,iPrj); -% end -% else - -% gTMP = psTile(:,:,iPrj); -% end - - - gTMP = gTMP - BH_movingAverage(gTMP,[nBgPix,nBgPix]); - if (normFactor) - bgSubPS(:,:,iPrj) = gTMP ./ BH_movingRMS(gTMP,floor([nRMSpix,nRMSpix])); - else - bgSubPS(:,:,iPrj) = gTMP; - end - clear gTMP - - if (rotationalAvg) - - rotTMP = bgSubPS(:,:,iPrj); - for i = 0.5:0.5:360 - R = BH_defineMatrix([i,0,0],'Bah','forward'); - ROT1 = R(1).*rot1 + R(4).*rot2; - ROT2 = R(2).*rot1 + R(5).*rot2; - rotTMP = rotTMP + interpn(r1,r2,bgSubPS(:,:,iPrj),ROT1,ROT2,'linear',0); - end - rotBgSubPS(:,:,iPrj) = rotTMP./720; -% rotBgSubPS(:,:,iPrj) = rotBgSubPS(:,:,iPrj) ./ ... -% BH_movingRMS(rotBgSubPS(:,:,iPrj), ... -% [nBgPix,nBgPix]); - clear rotTMP - end - - - - end - - - - cccStorage = cell(3,1); - for iRefine = 1:3 - - iRefine - if iRefine == 1 - % Initialize the best defocus from the global estimate for the first iter. - maxDef = zeros(d3,1)+TLT(1,15); - if (flgAstigmatism) - maxAst = zeros(d3,1,'gpuArray')+TLT(1,12); - maxAng = zeros(d3,1,'gpuArray')+TLT(1,13); - else - % Only using rotational average, so don't consider previosuly estimated - % astigmatism. - maxAst = zeros(d3,1,'gpuArray'); - maxAng = zeros(d3,1,'gpuArray'); - end - end - - for iPrj = 1:d3 - defSearch{iRefine,iPrj} = maxDef(iPrj)-defRange{iRefine}:defInc{iRefine}:maxDef(iPrj)+defRange{iRefine}; - end - - cccStorage{iRefine} = zeros(length(defSearch{iRefine,1}),d3); - - - for iDF = 1:length(defSearch{iRefine,1}) - if iRefine == 1 - % On first pass, search range is the same for all projections, so limit - % calcs. - df1 = defSearch{iRefine,iPrj}(iDF) - 0;%maxAst(iPrj); - df2 = defSearch{iRefine,iPrj}(iDF) + 0;%maxAst(iPrj); - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,0],size(radialForCTF{1}), ... - AMPCONT,-1.0); - end - - for iPrj = 1:d3 - - if iRefine > 1 - - df1 = defSearch{iRefine,iPrj}(iDF) - maxAst(iPrj); - df2 = defSearch{iRefine,iPrj}(iDF) + maxAst(iPrj); - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,maxAng(iPrj)],size(radialForCTF{1}), ... - AMPCONT,-1.0); - end - - % If not calc astigmatism use rotationally averaged always. - if (iRefine == 1 || ~(flgAstigmatism)) && rotationalAvg - [ iCCC ] = calc_CCC(radialForCTF{1},highCutoff, rotBgSubPS(:,:,iPrj), Hqz,0); - else - [ iCCC ] = calc_CCC(radialForCTF{1},highCutoff, bgSubPS(:,:,iPrj), Hqz,0); - end - cccStorage{iRefine}(iDF,iPrj) = gather(iCCC); - end - - - end - % get the max scores for this iteration per projection - maxDef = zeros(d3,1); - for iPrj = 1:d3 - [~,c]=max(cccStorage{iRefine}(:,iPrj)); - maxDef(iPrj) = defSearch{iRefine,iPrj}(c); - end - - - if (flgAstigmatism) - % If not leave maxAstig and maxAngle set to their initial values from the - % TLT geometry. - for iPrj = 1:d3 - n=1; - iPrj - for iAng = astigAngSearch{iRefine} - for iDelDF = astigDefSearch{iRefine} - iDelDfFull = iDelDF + maxAst(iPrj); - iAngFull = iAng + maxAng(iPrj); - - df1 = maxDef(iPrj) - iDelDfFull; - df2 = maxDef(iPrj) + iDelDfFull; - - % For values very close to zero, the search range may include - % values |df1| < |df2| which is against convention. - if abs(df1) >= abs(df2) - - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAngFull],size(radialForCTF{1}), ... - AMPCONT,-1.0); - - [ iCCC ] = calc_CCC(radialForCTF{1},highCutoff, bgSubPS(:,:,iPrj), Hqz,1); - - %fprintf('%d / %d coarse astigmatism search\n',n,size(initAstigCCC,1)); - else - - iCCC = -9999 - - end - - astigCCC{iRefine}{iPrj}(n,:) = [iAngFull,iDelDfFull,iCCC]; - n = n + 1; - end - end - end - - for iPrj = 1:d3 - [~,c]=max(astigCCC{iRefine}{iPrj}(:,3)); - maxAst(iPrj) = astigCCC{iRefine}{iPrj}(c,2); - maxAng(iPrj) = astigCCC{iRefine}{iPrj}(c,1); - end - - maxAst - maxAng - end - end - - - - + % exit an fit the PS using CTFFIND4 + BH_runCtfFind(sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName), ... + sprintf('%s_ctf.tlt',fileName), ctfParams,TLT) - -avgCCC=0; -for iPrj = 1:d3 - avgCCC = avgCCC+maxDef(iPrj); -end -avgCCC = avgCCC ./ d3; - - - - for iPrj = 1:d3 - - defAstig = [maxDef(iPrj) - maxAst(iPrj), maxDef(iPrj) + maxAst(iPrj),maxAng(iPrj) ]; - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,defAstig,paddedSize,AMPCONT,-1.0); - - [ ~, bandpass ] = calc_CCC(radialForCTF{1},highCutoff, bgSubPS(:,:,iPrj), Hqz, 1); - - bgSubPS(:,:,iPrj) = bgSubPS(:,:,iPrj) .* bandpass; - if (rotationalAvg) - rotBgSubPS(:,:,iPrj) = rotBgSubPS(:,:,iPrj) .* bandpass; - end end - - - -SAVE_IMG(MRCImage(gather(bgSubPS)),sprintf('fixedStacks/ctf/%s_bgOUT.mrc',fileName)); -clear bgSubPS -if (rotationalAvg) - SAVE_IMG(MRCImage(gather(rotBgSubPS)),sprintf('fixedStacks/ctf/%s_bgOUT_rotAvg.mrc',fileName)); -end -clear rotBgSubPS - -ang = TLT(:,4); -r1 = maxDef(TLT(:,1)); -r2 = fit(ang, r1, 'smoothingSpline'); -r3 = fit(ang, r2(ang), 'smoothingSpline'); - -ast1 = fit(ang,gather(maxAst(TLT(:,1))), 'smoothingSpline'); -ang1 = fit(ang,gather(maxAng(TLT(:,1))), 'smoothingSpline'); - - -figure('Visible','off'), ... -plot(ang,r3(ang),'bo',ang,zeros(1,d3)+TLT(1,15),'b--', ... - ang,zeros(1,d3)+TLT(1,15)+defRange{1},'k--',... - ang,zeros(1,d3)+TLT(1,15)-defRange{1},'k--'); - title(sprintf('CTF refine\nmeanDef %03.3f μm\nmeanDefPerTilt %03.3f μm ', TLT(1,15)*10^6,avgCCC*10^6)); - xlabel('Projection'); ylabel('defocus'); - - saveas(gcf,sprintf('fixedStacks/ctf/%s_refine.pdf',fileName), 'pdf'); - - if (flgAstigmatism) - figure('Visible','off'), ... - plot(ang,ast1(TLT(:,4)).*10^9,'bo',ang,(180/pi).*ang1(TLT(:,4)),'go'); - title('CTF refine astigmatism'); - xlabel('Projection'); ylabel('astig(nm) angle(deg)'); - - saveas(gcf,sprintf('fixedStacks/ctf/%s_astig.pdf',fileName), 'pdf'); - end - - TLT(:,15) = r3(TLT(:,4)); - if (flgAstigmatism) - TLT(:,12) = ast1(TLT(:,4)); - TLT(:,13) = ang1(TLT(:,4)); - end - fileID = fopen(sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',STACK_PRFX,mapBackIter+1), 'w'); - fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\n'], TLT'); - fclose(fileID); - -end -clear - -for i = 1:gpuDeviceCount - gpuDevice(i); -end -end - - - - - -function [ iCCC, bandpass ] = calc_CCC(radialForCTF,highCutoff, bgSubPS, Hqz, flgAst) - - - ctfSQ = abs(Hqz); - - rV = Hqz(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); - freqVector = radialForCTF(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); - firstZero = find(rV > 0, 1,'first'); - secondZero = find(rV(firstZero+1:end) < 0, 1,'first') + firstZero; - - [~,firstMax]=min(abs(rV(firstZero:secondZero-1)- ... - rV(firstZero+1:secondZero))) ; - firstMax = firstMax + firstZero; - -% if (flgAst) -% bandpass = ( radialForCTF > freqVector(firstZero) & ... -% radialForCTF < highCutoff & ctfSQ < 3*rms(ctfSQ(:))); -% else -% % % bandpass = ( radialForCTF > freqVector(firstMax) & ... -% % % radialForCTF < highCutoff ); -% end - - lowRes = radialForCTF > freqVector(firstZero) & radialForCTF < freqVector(firstMax); - useRes = radialForCTF > freqVector(firstZero) & radialForCTF < highCutoff; - bandpass = single( useRes ); - bandpass(lowRes) = ctfSQ(lowRes).^4; - - - ctfSQ = ctfSQ .* bandpass; - - bgSubPS = bgSubPS .* bandpass; - - - - iCCC = sum(sum((bgSubPS(useRes) .* ... - ctfSQ(useRes)))) ./ ... - ( numel(ctfSQ(useRes)).*... - std2(ctfSQ(useRes)).*... - std2(bgSubPS(useRes)) ); - - end @@ -759,60 +315,52 @@ iDeltaZ = (i - tiltOrigin)*pixelSize*-1.*tand(TLT(iPrj,4)); if any(ismember(i-tileSize/2+1:i+tileSize/2,iEvalMask)) %evalMask(i,paddedSize/2+1) - doSplineInterp=1; - mag = (1+iDeltaZ./DFo).^0.5; + mag = (1+iDeltaZ./DFo).^0.5; - estSize = 2048; - ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); - ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),iDeltaZ+DFo,estSize,TLT(iPrj,19),-1,1); - ctf1 = ctf1(1:estSize/2); - ctf2 = ctf2(1:estSize/2); - firstZero = find(ctf1 > 0, 1, 'first'); - secondZero= find(ctf1(firstZero:end) < 0 , 1, 'first') + firstZero - 1; - - %fprintf(ftmp,'firstZero %d %2.2f\n',firstZero,estSize/firstZero*pixelSize); - % This range will depend on the size of the field of view. For now, setting manually for Florian's HIV - % data, but will derive a formula to make sure the search is appropriate. Here we expect at most ~ 300 nm - % deltaZ, the strongest difference is at the lowest defocus which is ~ 1500 nm, which gives an estimated mag - % ~ 1.095 - defRange = mag-.1:.001:mag+.1; - nDef = length(defRange); - scoreDef = zeros(nDef,1,'gpuArray'); - for iDef = 1:nDef - ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); - % Larger scalings will have zeros rather than extroplation, so% - % % don't let this influence the score. - % lastZero = find(abs(ci) > 0 , 1, 'last'); - %fprintf(ftmp,'%d %d %d %d',size(ci),size(ctf1)); - scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - - end - [~,maxCoord] = max(scoreDef); - mag = defRange(maxCoord); - - defRange = mag-.01:.0001:mag+.01; - nDef = length(defRange); - scoreDef = zeros(nDef,1,'gpuArray'); - for iDef = 1:nDef - ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); - scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - end - [~,maxCoord] = max(scoreDef); - - mag = defRange(maxCoord); - - -% testNoRefine = 0; - if (testNoRefine) - mag = 1 - end + estSize = 2048; + ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); + ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),iDeltaZ+DFo,estSize,TLT(iPrj,19),-1,1); + ctf1 = ctf1(1:estSize/2); + ctf2 = ctf2(1:estSize/2); + firstZero = find(ctf1 > 0, 1, 'first'); + % secondZero= find(ctf1(firstZero:end) < 0 , 1, 'first') + firstZero - 1; + + %fprintf(ftmp,'firstZero %d %2.2f\n',firstZero,estSize/firstZero*pixelSize); + % This range will depend on the size of the field of view. For now, setting manually for Florian's HIV + % data, but will derive a formula to make sure the search is appropriate. Here we expect at most ~ 300 nm + % deltaZ, the strongest difference is at the lowest defocus which is ~ 1500 nm, which gives an estimated mag + % ~ 1.095 + defRange = mag-.1:.001:mag+.1; + nDef = length(defRange); + scoreDef = zeros(nDef,1,'gpuArray'); + for iDef = 1:nDef + ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); + % Larger scalings will have zeros rather than extroplation, so% + % % don't let this influence the score. + % lastZero = find(abs(ci) > 0 , 1, 'last'); + %fprintf(ftmp,'%d %d %d %d',size(ci),size(ctf1)); + scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); + + end + [~,maxCoord] = max(scoreDef); + mag = defRange(maxCoord); + + defRange = mag-.01:.0001:mag+.01; + nDef = length(defRange); + scoreDef = zeros(nDef,1,'gpuArray'); + for iDef = 1:nDef + ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); + scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); + end + [~,maxCoord] = max(scoreDef); + mag = defRange(maxCoord); + if (testNoRefine) + mag = 1; + end scaledStrip = iProjection(i-tileSize/2+1:i+tileSize/2,:); - - - for j = 1+tileSize/2:overlap:d2-tileSize/2 @@ -833,31 +381,23 @@ oX = ceil((oX+1)./2); oY = ceil((oY+1)./2); - - iPadVal = BH_multi_padVal(size(iTile),[scaledSize,scaledSize]); + iPadVal = BH_multi_padVal(size(iTile),[scaledSize,scaledSize]); - oupSize = [floor(scaledSize./2),ceil(scaledSize./2); ... - floor(scaledSize./2),ceil(scaledSize./2)]; - - % Get rid of th fftshift - tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... - oY-oupSize(2,1):oY+oupSize(2,2)-1) = ... - tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... - oY-oupSize(2,1):oY+oupSize(2,2)-1) + ... - fftshift(abs(fftn(BH_padZeros3d(iTile,iPadVal(1,:),iPadVal(2,:), ... - 'GPU','singleTaper', mean(iTile(:)))))); - % Using singleTaper here produces - % a grid like artifact. Test - % switch for EMC functions - - - - + oupSize = [floor(scaledSize./2),ceil(scaledSize./2); ... + floor(scaledSize./2),ceil(scaledSize./2)]; - end - end - end + % Get rid of th fftshift + tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... + oY-oupSize(2,1):oY+oupSize(2,2)-1) = ... + tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... + oY-oupSize(2,1):oY+oupSize(2,2)-1) + ... + fftshift(abs(fftn(BH_padZeros3d(iTile,iPadVal(1,:),iPadVal(2,:), ... + 'GPU','singleTaper', mean(iTile(:)))))); + + end % loop over j + end % if over eval mask + end % over tiles psTile = gather(BH_padZeros3d(tmpTile, [-1,-1].* ... diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 5cd56a3d..b13b6ad3 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -14,9 +14,9 @@ eucentric_minTilt = 15; end try - flgShiftEucentric = pBH.('eucentric_fit') + flgShiftEucentric = pBH.('eucentric_fit'); catch - flgShiftEucentric = 0 + flgShiftEucentric = 0; end try % Should be negative, but to test. @@ -95,141 +95,130 @@ end for iTilt = 1:length(ITER_LIST{iGPU}) - - STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - - if (mapBackIter) - mapBackPrfx = sprintf('mapBack%d/%s_ali%d_ctf',mapBackIter,STACK_PRFX,mapBackIter) - else - mbEST=''; - end - - -if strcmpi(applyFullorUpdate, 'full') - % Combine old and new transformations and apply as well as erasing beads, - % e.g. go from raw stack to preCTF. - flgSkipErase = 0; - flgApplyFullXform = 1; - PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} - PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); - outputDirectory = 'aliStacks' -elseif strcmpi(applyFullorUpdate, 'fullScale') - % Combine old and new transformations and apply as well as erasing beads, - % e.g. go from raw stack to preCTF. - - % Also remove shifts due to the sample being non-eucentric. This is a - % test, and if it helps, it would be even better to just apply this shift - % to all the subtomos pre-emptivel. - flgSkipErase = 0; - flgApplyFullXform = 1; - PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} - PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); - - outputDirectory = 'aliStacks'; -elseif strcmpi(applyFullorUpdate, 'refine') - - % Don't combine, just use tlt with new defocus values and erase beads - flgSkipErase = 0; - flgApplyFullXform = 1; - PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} - PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); - - - outputDirectory = 'aliStacks'; -elseif strcmpi(applyFullorUpdate,'update') - % Combine old and new transformations, but only apply the new xform and - % don't erase beads. e.g. in inital binned mapback just update the tilt - % files and resample the ctfCorrected stack. - flgSkipErase = 1; - flgApplyFullXform = 0; - PRJ_STACK ={sprintf('ctfStacks/%s_ali%d_ctf.fixed',STACK_PRFX,mapBackIter+flgInitResample)} - PRJ_OUT = {sprintf('%s_ali%d_ctf',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d_ctf',STACK_PRFX,mapBackIter); - - outputDirectory = 'ctfStacks'; -else - - error('applyFullorUpdate should be [full], [fullScale] or [update]') -end + STACK_PRFX = ITER_LIST{iGPU}{iTilt}; -try - tlt = {sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt',STACK_PRFX,mapBackIter+flgInitResample)}; - testLoad = load(tlt{1}); -catch - tlt = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+flgInitResample)}; - testLoad = load(tlt{1}); + if (mapBackIter) + mapBackPrfx = sprintf('mapBack%d/%s_ali%d_ctf',mapBackIter,STACK_PRFX,mapBackIter) + else + mbEST=''; + end -end -tlt_OUT = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1)}; - -eraseStack = sprintf('rm cache/%s_*.fixed',STACK_PRFX); -eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); -% Converte bead diameter to pixels and add a little to be safe. -PIXEL_SIZE = pBH.('PIXEL_SIZE'); -SuperResolution = pBH.('SuperResolution'); - -% Don't apply any fourier cropping of super-res data if only updating, -% as it would already be done. -if strcmpi(applyFullorUpdate,'update') - SuperResolution = 0; -end -if (SuperResolution) - % Transform the raw images at full sampling then crop the fft to physical - % nyquist - PIXEL_SIZE = 2.* PIXEL_SIZE; -end + if strcmpi(applyFullorUpdate, 'full') + % Combine old and new transformations and apply as well as erasing beads, + % e.g. go from raw stack to preCTF. + flgSkipErase = 0; + flgApplyFullXform = 1; + PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} + PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); + outputDirectory = 'aliStacks' + elseif strcmpi(applyFullorUpdate, 'fullScale') + % Combine old and new transformations and apply as well as erasing beads, + % e.g. go from raw stack to preCTF. + + % Also remove shifts due to the sample being non-eucentric. This is a + % test, and if it helps, it would be even better to just apply this shift + % to all the subtomos pre-emptivel. + flgSkipErase = 0; + flgApplyFullXform = 1; + PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} + PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); + + outputDirectory = 'aliStacks'; + elseif strcmpi(applyFullorUpdate, 'refine') + + % Don't combine, just use tlt with new defocus values and erase beads + flgSkipErase = 0; + flgApplyFullXform = 1; + PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} + PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); + + + outputDirectory = 'aliStacks'; + elseif strcmpi(applyFullorUpdate,'update') + % Combine old and new transformations, but only apply the new xform and + % don't erase beads. e.g. in inital binned mapback just update the tilt + % files and resample the ctfCorrected stack. + flgSkipErase = 1; + flgApplyFullXform = 0; + PRJ_STACK ={sprintf('ctfStacks/%s_ali%d_ctf.fixed',STACK_PRFX,mapBackIter+flgInitResample)} + PRJ_OUT = {sprintf('%s_ali%d_ctf',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d_ctf',STACK_PRFX,mapBackIter); + + outputDirectory = 'ctfStacks'; + else + error('applyFullorUpdate should be [full], [fullScale] or [update]') + end -eraseSigma = 3;%pBH.('beadSigma'); - -eraseRadius = ceil(1.2.*(pBH.('beadDiameter')./PIXEL_SIZE.*0.5)); -flgImodErase = 0 - - % FIXME, this should be stored from previous mask calc and accessed there. - % For now just take based on tomogram (which will be larger than the true specimen thickness) - %THICKNESS = recGeomForThickness.(sprintf('%s_1',STACK_PRFX)); - %THICKNESS = min(10,abs(THICKNESS(1,3)-THICKNESS(2,3)).*PIXEL_SIZE.*10^9); - THICKNESS = 100; -% Assuming all extreme pixels have already been removed from the stack. -%PRJ_STACK = {sprintf('%s_local04_18.mrc',mjIDX)};%,sprintf('%s_local14_18.mrc',mjIDX),sprintf('%s_local24_18.mrc',mjIDX),sprintf('%s_local34_18.mrc',mjIDX)}; -nStacks = length(tlt); -INPUT_CELL = cell(nStacks,7); -TLT_Trans = cell(nStacks,1); - -% killed the loop, clean up later -if exist(tlt{1}, 'file') && exist(PRJ_STACK{1}, 'file') - INPUT_CELL{1,1} = load(tlt{1}); - INPUT_CELL{1,2} = PRJ_STACK{1}; - [pathName,fileName,extension] = fileparts(PRJ_STACK{1}); - if isempty(pathName) - pathName = '.'; - end - [ctfPath,~,~] = fileparts(tlt{1}); - INPUT_CELL{1,3} = pathName; - INPUT_CELL{1,4} = fileName; - INPUT_CELL{1,5} = extension; - INPUT_CELL{1,6} = PRJ_OUT{1}; - INPUT_CELL{1,7} = ctfPath; -else - if ~exist(tlt{1}, 'file') - fprintf('\nignoring %s, because the file is not found.\n', tlt{1}); - end - if ~exist(PRJ_STACK{1}, 'file') - fprintf('\nignoring %s, because the file is not found.\n',PRJ_STACK{1}); + tlt = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+flgInitResample)}; + tlt_OUT = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1)}; + + eraseStack = sprintf('rm cache/%s_*.fixed',STACK_PRFX); + eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); + % Converte bead diameter to pixels and add a little to be safe. + PIXEL_SIZE = pBH.('PIXEL_SIZE'); + SuperResolution = pBH.('SuperResolution'); + + % Don't apply any fourier cropping of super-res data if only updating, + % as it would already be done. + if strcmpi(applyFullorUpdate,'update') + SuperResolution = 0; end -end + if (SuperResolution) + % Transform the raw images at full sampling then crop the fft to physical + % nyquist + PIXEL_SIZE = 2.* PIXEL_SIZE; + end - -% killed the loop, clean up later -iStack=1; + eraseSigma = 3;%pBH.('beadSigma'); + + eraseRadius = ceil(1.2.*(pBH.('beadDiameter')./PIXEL_SIZE.*0.5)); + flgImodErase = 0 + + % FIXME, this should be stored from previous mask calc and accessed there. + % For now just take based on tomogram (which will be larger than the true specimen thickness) + %THICKNESS = recGeomForThickness.(sprintf('%s_1',STACK_PRFX)); + %THICKNESS = min(10,abs(THICKNESS(1,3)-THICKNESS(2,3)).*PIXEL_SIZE.*10^9); + THICKNESS = 100; + % Assuming all extreme pixels have already been removed from the stack. + %PRJ_STACK = {sprintf('%s_local04_18.mrc',mjIDX)};%,sprintf('%s_local14_18.mrc',mjIDX),sprintf('%s_local24_18.mrc',mjIDX),sprintf('%s_local34_18.mrc',mjIDX)}; + nStacks = length(tlt); + INPUT_CELL = cell(nStacks,7); + TLT_Trans = cell(nStacks,1); + + % killed the loop, clean up later + if exist(tlt{1}, 'file') && exist(PRJ_STACK{1}, 'file') + INPUT_CELL{1,1} = load(tlt{1}); + INPUT_CELL{1,2} = PRJ_STACK{1}; + [pathName,fileName,extension] = fileparts(PRJ_STACK{1}); + if isempty(pathName) + pathName = '.'; + end + [ctfPath,~,~] = fileparts(tlt{1}); + INPUT_CELL{1,3} = pathName; + INPUT_CELL{1,4} = fileName; + INPUT_CELL{1,5} = extension; + INPUT_CELL{1,6} = PRJ_OUT{1}; + INPUT_CELL{1,7} = ctfPath; + else + if ~exist(tlt{1}, 'file') + fprintf('\nignoring %s, because the file is not found.\n', tlt{1}); + end + if ~exist(PRJ_STACK{1}, 'file') + fprintf('\nignoring %s, because the file is not found.\n',PRJ_STACK{1}); + end + end + + % FIXME: removed the loop, clean up later cells etc later + iStack=1; iMrcObj = MRCImage(INPUT_CELL{iStack,2},0); @@ -237,8 +226,7 @@ % must maintain whatever is there in case beads are to be erased. The model % used for this process depends on the pixel size in the header when it was % created in IMod alignment. -% [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); -% iPixelHeader = EMC_str2double(iPixelHeader); + iHeader = getHeader(iMrcObj); iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* (1+abs(SuperResolution)), ... iHeader.cellDimensionY/iHeader.nY .* (1+abs(SuperResolution)), ... @@ -250,31 +238,23 @@ d1 = iHeader.nX; d2 = iHeader.nY; d3 = size(INPUT_CELL{iStack,1},1);%iHeader.nZ; - osX = 1-mod(d1,2); osY = 1-mod(d2,2); + osX = 1-mod(d1,2); osY = 1-mod(d2,2); + if (SuperResolution) + gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,-0.235,'GPU','nyquistHigh'); + else + gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); + end -if (SuperResolution) - gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,-0.235,'GPU','nyquistHigh'); -else - gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); -end - -TLT = INPUT_CELL{iStack,1}; -pathName = INPUT_CELL{iStack,3} -fileName = INPUT_CELL{iStack,4} -extension = INPUT_CELL{iStack,5} - -% Copy with column for defocus = input to CTF correct -% saved as _ctf.tlt - - -nPrjs = size(TLT,1); - + TLT = INPUT_CELL{iStack,1}; + pathName = INPUT_CELL{iStack,3} + fileName = INPUT_CELL{iStack,4} + extension = INPUT_CELL{iStack,5} -% Optionally address magnification changes. -% system(sprintf('mkdir -p %s/recon',INPUT_CELL{i,3})); -system('mkdir -p aliStacks'); + % Optionally address magnification changes. + % system(sprintf('mkdir -p %s/recon',INPUT_CELL{i,3})); + system('mkdir -p aliStacks'); if (mapBackIter) fprintf('Combining tranformations\n\n'); @@ -282,7 +262,7 @@ try mbEST = load(sprintf('%s.tltxf',mapBackPrfx)); catch - fprintf('WARNING: did not load %s.tltxf, cannot update alignments',mapBackPrfx) + error('WARNING: did not load %s.tltxf, cannot update alignments',mapBackPrfx) system(sprintf('cp fixedStacks/ctf/%s_ali1_ctf.tlt fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,STACK_PRFX,mapBackIter+1)); continue; end @@ -310,66 +290,44 @@ fprintf('\n\nYou suspect a eucentric drift\n\n') eucShift = fit(mbTLT(toFit),1 .* mbEST(toFit,5) ./ sind(mbTLT(toFit)),'poly1'); - fprintf('\n\nFound a possible eucentric shift of %3.3f\nThe slope (%3.3f) should be close to zero.\n\n',eucShift.p2,eucShift.p1); - -% mbEST(:,5) = mbEST(:,5) - eucShift.*sind(mbTLT); -% mbEST(:,5) + fprintf('\n\nFound a possible eucentric shift of %3.3f\nThe slope (%3.3f) should be close to zero.\n\n',eucShift.p2,eucShift.p1); eucShiftsResults{iGPU}{iTilt} = eucShift.p2; - end outputStackName = sprintf('%s/%s%s',outputDirectory,INPUT_CELL{iStack,6},INPUT_CELL{iStack,5}); oldStackName = sprintf('%s/%s%s',outputDirectory,PRJ_OLD,INPUT_CELL{iStack,5}); - -try - erase_beads_after_ctf = pBH.('erase_beads_after_ctf'); -catch - erase_beads_after_ctf = false; -end - -if (erase_beads_after_ctf) - flgEraseBeads = 0; -else - if exist(sprintf('fixedStacks/%s.erase',fileName),'file') - flgEraseBeads = 1; - % create and later run a script to erase gold beads using imods - % ccderaser and the present fiducial model. - - else - flgEraseBeads = 0; + try + erase_beads_after_ctf = pBH.('erase_beads_after_ctf'); + catch + erase_beads_after_ctf = false; end -end - - - - - - - - SIZEOUT = [d1,d2]; - - + if (erase_beads_after_ctf) + flgEraseBeads = 0; + else + if exist(sprintf('fixedStacks/%s.erase',fileName),'file') + flgEraseBeads = 1; + % create and later run a script to erase gold beads using imods + % ccderaser and the present fiducial model. + else + flgEraseBeads = 0; + end + end tlt_tmp = cell(d3,1); out_tmp = cell(d3,1); - - origOrder = TLT(:,1); TLT = sortrows(TLT,1); - for i = 1:d3 tlt_tmp{i} = TLT(i,:); end - - if (flgSkipUpdate) - continue + continue; end if (SuperResolution) @@ -380,216 +338,202 @@ end sizeCropped(3) = d3; - STACK = zeros(sizeCropped,'single'); - samplingMaskStack = zeros(sizeCropped,'single'); + STACK = zeros(sizeCropped,'single'); + samplingMaskStack = zeros(sizeCropped,'single'); for i = 1:d3 + if (SuperResolution) + % The transform shifts need to be scaled by 2 since the stored values + % are relative to full sampling, while the tomoCPR are relative to + % physical pixel size. + updateScale = 2; + else + updateScale = 1; + end + if (mapBackIter) + % Stored in row order as output by imod, st transpose is needed. Inversion + % of the xform is handled in resample2d. + origXF = reshape(tlt_tmp{i}(7:10),2,2)'; + newXF = reshape(mbEST(i,1:4),2,2)'; - if (SuperResolution) - % The transform shifts need to be scaled by 2 since the stored values - % are relative to full sampling, while the tomoCPR are relative to - % physical pixel size. - updateScale = 2; - else - updateScale = 1; - end - - if (mapBackIter) - - % Stored in row order as output by imod, st transpose is needed. Inversion - % of the xform is handled in resample2d. - origXF = reshape(tlt_tmp{i}(7:10),2,2)'; - newXF = reshape(mbEST(i,1:4),2,2)'; + dXYZ = [(newXF*tlt_tmp{i}(2:3)')' + mbEST(i,5:6).*updateScale , 0]; + if ~isvector(dXYZ) + % In case some implicit expansion were to happen for whatever reason. + error('dXYZ is a matrix and should be a vector'); + end + tlt_tmp{i}(2:3) = dXYZ(1:2); + - dXYZ = [(newXF*tlt_tmp{i}(2:3)')' + mbEST(i,5:6).*updateScale , 0]; - if ~isvector(dXYZ) - % In case some implicit expansion were to happen for whatever reason. - error('dXYZ is a matrix and should be a vector'); + combinedXF = reshape((newXF*origXF)',1,4); + tlt_tmp{i}(7:10) = combinedXF; + else + combinedXF = tlt_tmp{i}(7:10); + dXYZ = [tlt_tmp{i}(2:3),0]; end - tlt_tmp{i}(2:3) = dXYZ(1:2); - - - - combinedXF = reshape((newXF*origXF)',1,4); - tlt_tmp{i}(7:10) = combinedXF; - else - combinedXF = tlt_tmp{i}(7:10); - dXYZ = [tlt_tmp{i}(2:3),0]; - end % Now that we are always cropping prior to transforming, reduce the % scale. Probable should just instruct to fourier crop prior to tilt % alignment.flgSkipUpdate dXYZ = dXYZ ./ updateScale; - - - - - - + % Pad the projection prior to xforming in Fourier space. + if (SuperResolution) + + iProjection = single(getVolume(iMrcObj,[],[],tlt_tmp{i}(23),'keep')); + iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); + + % Information beyond the physical nyquist should be removed to limit + % aliasing of noise prior tto interpolation. + iProjection = BH_padZeros3d(iProjection,[0,0],[0,0],'GPU','singleTaper',mean(iProjection(:))); + trimVal = BH_multi_padVal(1.*size(iProjection),sizeCropped(1:2)); - % Pad the projection prior to xforming in Fourier space. - if (SuperResolution) - - iProjection = single(getVolume(iMrcObj,[],[],tlt_tmp{i}(23),'keep')); - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); - - % Information beyond the physical nyquist should be removed to limit - % aliasing of noise prior tto interpolation. - iProjection = BH_padZeros3d(iProjection,[0,0],[0,0],'GPU','singleTaper',mean(iProjection(:))); - trimVal = BH_multi_padVal(1.*size(iProjection),sizeCropped(1:2)); - - largeOutliersMean= mean(iProjection(:)); - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single','gpuArray'); - - iProjection = real(ifftn(ifftshift(... - BH_padZeros3d(fftshift(... - fftn(iProjection)), ... - trimVal(1,:),trimVal(2,:),... - 'GPU','single')))); - - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),[0,0,0],[0,0],'Bah','GPU','forward',1/2,sizeCropped(1:2)); - sizeODD = size(iProjection)-[osX,osY]; - else - sizeODD = [d1,d2]-[osX,osY]; - - - - % If it is even sized, shift up one pixel so that the origin is in the middle - % of the odd output here we can just read it in this way, unlike super res. - - iProjection = ... - single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep')); - - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); + largeOutliersMean= mean(iProjection(:)); + largeOutliersSTD = std(iProjection(:)); + largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... + iProjection > largeOutliersMean + 6*largeOutliersSTD); + iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single','gpuArray'); + + iProjection = real(ifftn(ifftshift(... + BH_padZeros3d(fftshift(... + fftn(iProjection)), ... + trimVal(1,:),trimVal(2,:),... + 'GPU','single')))); + + + iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),[0,0,0],[0,0],'Bah','GPU','forward',1/2,sizeCropped(1:2)); + sizeODD = size(iProjection)-[osX,osY]; + else + sizeODD = [d1,d2]-[osX,osY]; + + + + % If it is even sized, shift up one pixel so that the origin is in the middle + % of the odd output here we can just read it in this way, unlike super res. + + iProjection = ... + single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep')); + + iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); - largeOutliersMean= mean(iProjection(:)); - - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); + largeOutliersMean= mean(iProjection(:)); + + largeOutliersSTD = std(iProjection(:)); + largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... + iProjection > largeOutliersMean + 6*largeOutliersSTD); + iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); - + - end + end - % Because the rotation/scaling and translation are done separately, - % we must use a square transform; otherwise, a rotation angle dependent - % anisotropic distortion (like mag distortion) is introduced. - sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp.*0.25).*max(sizeODD)); - - - padVal = BH_multi_padVal(sizeODD,sizeSQ); - trimVal = BH_multi_padVal(sizeSQ,sizeCropped(1:2)); - - iProjection = iProjection - mean(iProjection(:)); - iProjection = iProjection ./ std(iProjection(:)); + % Because the rotation/scaling and translation are done separately, + % we must use a square transform; otherwise, a rotation angle dependent + % anisotropic distortion (like mag distortion) is introduced. + sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp.*0.25).*max(sizeODD)); + + + padVal = BH_multi_padVal(sizeODD,sizeSQ); + trimVal = BH_multi_padVal(sizeSQ,sizeCropped(1:2)); + + iProjection = iProjection - mean(iProjection(:)); + iProjection = iProjection ./ std(iProjection(:)); - if ( SuperResolution ) - iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... - padVal(1,:),padVal(2,:),'GPU','singleTaper'); - else - iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - 'GPU','singleTaper'); - end + if ( SuperResolution ) + iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... + padVal(1,:),padVal(2,:),'GPU','singleTaper'); + else + iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... + 'GPU','singleTaper'); + end - if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); - end + if (i == 1 && bh_global_do_2d_fourier_interp) + bhF = fourierTransformer(iProjection,'OddSizeOversampled'); + end - if (flgApplyFullXform) - % Do the phase shift after rotating - need to invert the scaling since - % we are in reciprocal space - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(combinedXF); - % Assuming stretch and skew are not fit, leave defined for possible - % later consideration. - + if (flgApplyFullXform) + % Do the phase shift after rotating - need to invert the scaling since + % we are in reciprocal space + [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... + BH_decomposeIMODxf(combinedXF); + % Assuming stretch and skew are not fit, leave defined for possible + % later consideration. + - if (bh_global_do_2d_fourier_interp) - if (i == 1) - fprintf('resampling at 2x padding with fourier interp\n'); - end -% combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - combinedInverted = combinedInverted([1,2,4,5]); - - - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); - else - if (i == 1) - fprintf('resampling at 1x padding with linear interp\n'); - end - % Real space, do not invert mag - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); + if (bh_global_do_2d_fourier_interp) + if (i == 1) + fprintf('resampling at 2x padding with fourier interp\n'); + end + % combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); + combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); - end - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - - else - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(mbEST(i,1:4)); - % Assuming stretch and skew are not fit, leave defined for possible - % later consideration. - % NOTE mag is ignored when the rotation matrix has 4 elements (IMOD) - - if (bh_global_do_2d_fourier_interp) - if (i == 1) - fprintf('resampling at 2x padding with fourier interp\n'); - end -% mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - mbEstInverted = mbEstInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); + iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); else - if (i == 1) - fprintf('resampling at 1x padding with linear interp\n'); - end - mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - mbEstInverted = mbEstInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); + if (i == 1) + fprintf('resampling at 1x padding with linear interp\n'); + end + % Real space, do not invert mag + combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); + combinedInverted = combinedInverted([1,2,4,5]); + iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); end - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),mbEST(i,1:4),dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - end - - - STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... - trimVal(1,:),trimVal(2,:),... - 'GPU','single'))); - - iSamplingMask(isnan(iSamplingMask(:))) = 0; - samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); - iSamplingMask = []; -% -% end - - end + iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); + + else + [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... + BH_decomposeIMODxf(mbEST(i,1:4)); + % Assuming stretch and skew are not fit, leave defined for possible + % later consideration. + % NOTE mag is ignored when the rotation matrix has 4 elements (IMOD) + + if (bh_global_do_2d_fourier_interp) + if (i == 1) + fprintf('resampling at 2x padding with fourier interp\n'); + end + % mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); + mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); + mbEstInverted = mbEstInverted([1,2,4,5]); + iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); + else + if (i == 1) + fprintf('resampling at 1x padding with linear interp\n'); + end + mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); + mbEstInverted = mbEstInverted([1,2,4,5]); + iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); + end + iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),mbEST(i,1:4),dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); + end - - for i= 1:d3 - TLT(i,:) = tlt_tmp{i}; -% STACK(:,:,TLT(i,1)) = out_tmp{i}; - end + + STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... + trimVal(1,:),trimVal(2,:),... + 'GPU','single'))); + + iSamplingMask(isnan(iSamplingMask(:))) = 0; + samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); + iSamplingMask = []; + % + % end + + end + + for i= 1:d3 + TLT(i,:) = tlt_tmp{i}; + % STACK(:,:,TLT(i,1)) = out_tmp{i}; + end out_tmp = []; diff --git a/masking/alignmentVol.m b/masking/alignmentVol.m new file mode 100644 index 00000000..e76477d2 --- /dev/null +++ b/masking/alignmentVol.m @@ -0,0 +1,281 @@ +classdef alignmentVol < handle + properties (Access = 'private') + data; + input_size; + pad_val; + end + properties (Access = 'public') + end + + methods + + function obj = alignmentVol(data, pad_val) + pause(3) + fprintf('alignmentVol constructor\n') + pause(3) + obj.data = data; + obj.input_size = size(obj.data); + obj.pad_val = pad_val; + end + + function [] = isAllocated(obj) + if isempty(obj.data) + error('data is not allocated') + else + fprintf('data is allocated\n') + end + end + + function [] = zero_data(obj) + obj.data = obj.data .* 0; + end + end + + + + end + +% function [ IMAGE ] = BH_padZerosSimple3d(IMAGE) + + %PADLOW, PADTOP, ... + %METHOD, PRECISION, varargin ) +%Pad an image volume with zeros. +% +% +% Input variables: +% +% IMAGE = 3d image to be padded +% +% PADLOW = size of padding pre +% +% PADTOP = size of padding post +% +% PRECISION = 'single' or 'double' +% +% METHOD = case sensitive 'GPU' otherwise cpu +% +% Output variables:BH_bandpass3d.m +% +% PADDED_IMG = the padded image. +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% Goals & limitations: +% +% When doing both pre and post padding with the build in padarray function, +% two steps are required, this slows things down. Here the whole matrix is +% allocated once, and much faster than zeros. +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% +% TODO: +% +% - Test for expected function, and return value (template matching depends) +% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% pause(3) +% IMAGE = IMAGE .* 0; +% IMAGE(1:7,1:7,1:7) = 1; +% pause(3) +% if isnumeric(PADLOW) +% padLOW = PADLOW ; +% padTOP = PADTOP; +% elseif strcmpi(PADLOW,'fwd') +% padLOW = PADTOP(1,:); +% padTOP = PADTOP(2,:); +% elseif strcmpi(PADLOW,'inv') +% padLOW = PADTOP(3,:); +% padTOP = PADTOP(4,:); +% else +% error('padLOW should be numeric, fwd or inv'); +% end + + +% twoD = 0; +% doRand = false; +% if numel(padLOW) == 2 +% twoD = 1; +% padLOW= [padLOW, 0]; +% padTOP= [padTOP, 0]; +% end + +% if nargin > 5 +% if isnumeric(varargin{1}) +% extrapVal = varargin{1}; +% else +% extrapVal = std(IMAGE(:)); +% extrapMean = mean(IMAGE(:)); +% doRand = 1; +% end +% else +% extrapVal = 0; +% end + +% fourierOverSample = 0; +% if nargin > 5 && length(varargin) > 1 +% extrapVal = 0; +% % It is assumed that a bandpass is already applied to that the value at +% % Nyquist is ~0, only applies for 2d right now. +% fourierOverSample = 1; +% end + +% trimLOW = abs(padLOW .* (padLOW < 0)); +% trimTOP = abs(padTOP .* (padTOP < 0)); +% padLOW = padLOW .* (padLOW >= 0); +% padTOP = padTOP .* (padTOP >= 0); + +% % If any pad values are negative, first trim the image. + +% try +% IMAGE = IMAGE(1+trimLOW(1):end-trimTOP(1),... +% 1+trimLOW(2):end-trimTOP(2),... +% 1+trimLOW(3):end-trimTOP(3)); +% catch +% fprintf('%f %f %f\n,%f %f %f\n',PADLOW,PADTOP); +% fprintf('%f %f %f\n, %f %f %f\n',trimLOW,trimTOP); +% end + +% if ismatrix(IMAGE) +% imgSize = [size(IMAGE),1]; +% else +% imgSize = size(IMAGE); +% end + +% padSize = imgSize + padLOW + padTOP; + + + + +% % Optionally taper the edges, rolling over 6 pixels with the first zero +% % outside the original image, in the padding +% % 1.0000 0.9505 0.8117 0.6113 0.3887 0.1883 0.0495 + +% taper=false; +% if strcmpi(PRECISION, 'singleTaper') +% taper = 0.5+0.5.*cos((((1:7)-1).*pi)./(length((1:7)))); +% PRECISION = 'single'; +% elseif strcmpi(PRECISION, 'doubleTaper') +% taper= 0.5+0.5.*cos((((1:7)-1).*pi)./(length((1:7)))); +% PRECISION = 'double'; +% end + +% if strcmp(METHOD, 'GPU') +% if strcmpi(PRECISION, 'single') +% if (doRand) +% PADDED_IMG = randn(padSize,'single','gpuArray').*extrapVal+extrapMean; +% else +% PADDED_IMG = zeros(padSize,'single','gpuArray'); +% end +% elseif strcmpi(PRECISION, 'double') +% if (doRand) +% PADDED_IMG = randn(padSize,'double','gpuArray').*extrapVal+extrapMean; +% else +% PADDED_IMG = zeros(padSize,'double','gpuArray'); +% end +% else +% error('PRECISION must be single or double, not %s', PRECISION) +% end +% else +% if strcmpi(PRECISION, 'single') +% if (doRand) +% PADDED_IMG = randn(padSize,'single').*extrapVal+extrapMean; +% else +% PADDED_IMG = zeros(padSize,'single'); +% end +% elseif strcmpi(PRECISION, 'double') +% if (doRand) +% PADDED_IMG = randn(padSize,'double').*extrapVal+extrapMean; +% else +% PADDED_IMG = zeros(padSize,'double'); +% end +% else +% error('PRECISION must be single or double, not %s', PRECISION) +% end +% end + +% if ( gather(extrapVal) ) +% PADDED_IMG = PADDED_IMG + extrapVal; +% end + +% if (twoD) +% if (taper) +% [d1,d2,d3] = size(IMAGE); + +% IMAGE(:,1:7) = IMAGE(:,1:7) .* repmat(flip(taper),d1,1,d3) + ... +% repmat(flip(extrapVal.*(1-taper)),d1,1,d3); +% IMAGE(1:7,:) = IMAGE(1:7,:) .* repmat(flip(taper)',1,d2,d3) + ... +% repmat(flip(extrapVal.*(1-taper))',1,d2,d3); + + +% IMAGE(:,end-6:end) = IMAGE(:,end-6:end) .* repmat(taper,d1,1,d3) + ... +% repmat(extrapVal.*(1-taper),d1,1,d3); +% IMAGE(end-6:end,:) = IMAGE(end-6:end,:) .* repmat(taper',1,d2,d3) + ... +% repmat(extrapVal.*(1-taper'),1,d2,d3); + +% end + +% if ( fourierOverSample ) +% sX1 = ceil((size(IMAGE,1)+1)./2); +% sX2 = size(IMAGE,1)-sX1-1; +% sY1 = ceil((size(IMAGE,2)+1)./2); +% sY2 = size(IMAGE,2)-sY1-1; + +% PADDED_IMG(1:sX1,1:sY1) = IMAGE(1:sX1,1:sY1); +% PADDED_IMG(end-sX2:end,1:sY1) = IMAGE(end-sX2:end,1:sY1); +% PADDED_IMG(1:sX1,end-sY2:end) = IMAGE(1:sX1,end-sY2:end); +% PADDED_IMG(end-sX2:end,end-sY2:end) = IMAGE(end-sX2:end,end-sY2:end); +% else +% PADDED_IMG(padLOW(1)+1: end - padTOP(1), ... +% padLOW(2)+1: end - padTOP(2)) = IMAGE; +% end +% else +% if (taper) +% [d1,d2,d3] = size(IMAGE); + +% IMAGE(:,1:7,:) = IMAGE(:,1:7,:) .* repmat(flip(taper),d1,1,d3) + ... +% repmat(flip(extrapVal.*(1-taper)),d1,1,d3); +% IMAGE(1:7,:,:) = IMAGE(1:7,:,:) .* repmat(flip(taper)',1,d2,d3) + ... +% repmat(flip(extrapVal.*(1-taper')),1,d2,d3); +% IMAGE(:,:,1:7) = IMAGE(:,:,1:7) .* repmat(permute(flip(taper),[3,1,2]),d1,d2,1) + ... +% repmat(permute(flip(extrapVal.*(1-taper)),[3,1,2]),d1,d2,1); + +% IMAGE(:,end-6:end,:) = IMAGE(:,end-6:end,:) .* repmat(taper,d1,1,d3) + ... +% repmat(extrapVal.*(1-taper),d1,1,d3); +% IMAGE(end-6:end,:,:) = IMAGE(end-6:end,:,:) .* repmat(taper',1,d2,d3) + ... +% repmat(extrapVal.*(1-taper'),1,d2,d3); +% IMAGE(:,:,end-6:end) = IMAGE(:,:,end-6:end) .* repmat(permute(taper,[3,1,2]),d1,d2,1) + ... +% repmat(permute(extrapVal.*(1-taper),[3,1,2]),d1,d2,1); +% end + +% if ( fourierOverSample ) +% sX1 = ceil((size(IMAGE,1)+1)./2); +% sX2 = size(IMAGE,1)-sX1-1; +% sY1 = ceil((size(IMAGE,2)+1)./2); +% sY2 = size(IMAGE,2)-sY1-1; +% sZ1 = ceil((size(IMAGE,3)+1)./2); +% sZ2 = size(IMAGE,3)-sZ1-1; + +% PADDED_IMG(1:sX1,1:sY1,1:sZ1) = IMAGE(1:sX1,1:sY1,1:sZ1); +% PADDED_IMG(end-sX2:end,1:sY1,1:sZ1) = IMAGE(end-sX2:end,1:sY1,1:sZ1); +% PADDED_IMG(1:sX1,end-sY2:end,1:sZ1) = IMAGE(1:sX1,end-sY2:end,1:sZ1); +% PADDED_IMG(end-sX2:end,end-sY2:end,1:sZ1) = IMAGE(end-sX2:end,end-sY2:end,1:sZ1); + +% PADDED_IMG(1:sX1,1:sY1,end-sZ2:end) = IMAGE(1:sX1,1:sY1,end-sZ2:end); +% PADDED_IMG(end-sX2:end,1:sY1,end-sZ2:end) = IMAGE(end-sX2:end,1:sY1,end-sZ2:end); +% PADDED_IMG(1:sX1,end-sY2:end,end-sZ2:end) = IMAGE(1:sX1,end-sY2:end,end-sZ2:end); +% PADDED_IMG(end-sX2:end,end-sY2:end,end-sZ2:end) = IMAGE(end-sX2:end,end-sY2:end,end-sZ2:end); +% else + +% PADDED_IMG(padLOW(1)+1: end - padTOP(1), ... +% padLOW(2)+1: end - padTOP(2), ... +% padLOW(3)+1: end - padTOP(3)) = IMAGE; +% end + +% end + +% clear IMAGE +% end of the padZeros3d functions + + + diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index a6510987..473f7e34 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -260,20 +260,14 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - try - newTLT = sprintf('fixedStacks/ctf/%s_ali%d_ctf_refine.tlt', ... - STACK_PRFX,mapBackIter+1); - - geometry.(tomoList{iTomo}) = load(newTLT); - fprintf('Updating refined TLT %s\n', newTLT); - catch - newTLT = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', ... - STACK_PRFX,mapBackIter+1); - - - geometry.(tomoList{iTomo}) = load(newTLT); - fprintf('Updating TLT %s\n', newTLT); - end + + newTLT = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', ... + STACK_PRFX,mapBackIter+1); + + + geometry.(tomoList{iTomo}) = load(newTLT); + fprintf('Updating TLT %s\n', newTLT); + clear newTLT STACK_PRFX end diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index 181ef53b..cc988fec 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -49,6 +49,8 @@ pBH = BH_parseParameterFile(PARAMETER_FILE); +test_multi_ref_diffmap=true; + flgClassify = pBH.('flgClassify'); %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of @@ -143,7 +145,7 @@ geom_name='' if (flgMultiRefAlignment ) - geom_name='ClusterClsGeom'; + geom_name='ClusterClsGeom'; else geom_name='Avg_geometry'; end @@ -215,7 +217,6 @@ peakList = []; end - numParticles = oldPca.nTOTAL; clear oldPca; catch error('trouble loading the previous pcs mat file.') @@ -447,7 +448,7 @@ geometry.(tomoList{iTomo}) = positionList; fprintf('Size iTomo %d %d\n',size(positionList)); else - positionList(lIndPart, 26) = class(lIndClass); + positionList(lIndPart, 26) = class(lIndClass); % this is where it is breaking geometry.(tomoList{iTomo}) = positionList; end diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index b389b18b..896e3ae7 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -923,7 +923,7 @@ masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRef)) = ... {maskType, sizeMask, ... maskRadius, maskCenter, ... - 1, shape_mask_lowpass, shape_mask_threshold}; + flgFscShapeMask, shape_mask_lowpass, shape_mask_threshold}; masterTM.('currentResForDefocusError') = osX(oneBitCut).^-1; end diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index db613a26..3c1cc67e 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -83,6 +83,15 @@ % error('PARAMETER_FILE, CYCLE, PREVIOUS_PCA') end + +% FIXME: +% To test seeding the classification with existing classes, rather than always reverting to the global average, +% use the mechanism in place to handle multiple references at different length scales derived from the global average, +% to instead be used for multiple distinct classes. If the results are promising, then expand so each ref may also be +% looked at over its own scale space. +test_multi_ref_diffmap = true; +test_scale_space_bug_fix = false; + startTime = clock; CYCLE = EMC_str2double(CYCLE); @@ -144,6 +153,13 @@ maxEigs = pBH.('Pca_maxEigs'); pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; refPixelSize = pBH.('PIXEL_SIZE').*10^10.*refSamplingRate; + +% FIMXE: Probably remove this incomplete idea +if (refSamplingRate ~= samplingRate) + error('refSamplingRate ~= samplingRate') +end + +% FIXME: SuperResolution should be deprecated if pBH.('SuperResolution') pixelSize = pixelSize * 2; refPixelSize = refPixelSize * 2; @@ -243,15 +259,22 @@ flgMultiRefAlignment = 0; end +if (test_multi_ref_diffmap && ~flgMultiRefAlignment) + test_multi_ref_diffmap = false; + fprintf("WARNING: test_multi_ref_diffmap is incompatible with ~flgMultiRefAlignment, disabling\n"); +end + geom_name='' if (flgMultiRefAlignment ) - geom_name='ClusterClsGeom'; + geom_name='ClusterClsGeom'; + else geom_name='Avg_geometry'; end geometry = subTomoMeta.(cycleNumber).(geom_name); + try flgCutOutVolumes = pBH.('flgCutOutVolumes'); catch @@ -298,8 +321,11 @@ BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) -refName = 0; -averageMotif = cell(2,1); +if (test_multi_ref_diffmap) + refName = pBH.('Raw_className'); +else + refName = 0; +end % If flgClassify is negative combine the data for clustering, but don't set % any of the alignment changes to be persistant so that extracted class @@ -313,6 +339,7 @@ oddRot = reshape(aliParams(1,:),3,3)'; % refine the translation per particle. catch + error('This block sshould not be reached.'); fprintf('\nReverting from %s to Raw in loading fitFSC\n','REF'); aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)) oddRot = reshape(aliParams(1,:),3,3)'; @@ -321,44 +348,67 @@ clear iRefPrev end - + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%% Added for test_multi_ref_diffmap %% +refVector = cell(2,1); +refGroup = cell(2,1); +classVector{1} = pBH.('Raw_classes_odd')(1,:); +classVector{2} = pBH.('Raw_classes_eve')(1,:); +refVectorFull{1}= [pBH.('Raw_classes_odd');classVector{1} ] +refVectorFull{2}= [pBH.('Raw_classes_eve');classVector{2} ] for iGold = 1:2 - -% if (flgGold) - if iGold == 1; - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end -% else -% halfSet = 'STD'; -% end + % Sort low to high, because order is rearranged as such unstack + refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; + % class id corresponding to membership in ???_refName + refVector{iGold} = refVectorFull{iGold}(1,:) + % reference id, so multiple classes can be merged into one + refGroup{iGold} = refVectorFull{iGold}(3,:) +end - try +% make sure the number of references match the unique groups in the classVector +% and also that the class/group pairs match the class/ref pairs. +nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; +nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]; - imgNAME = sprintf('class_%d_Locations_Raw_%s', refName, halfSet); - +averageMotif = cell(2,1); - [ averageMotif{iGold} ] = BH_unStackMontage4d(1, ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - preSizeMask); +if (test_multi_ref_diffmap) + fprintf('nScaleSpace = %d\n',nScaleSpace); + fprintf('nReferences = %d\n',nReferences); + nScaleSpace = nReferences(1); + pause(3); +else + nReferences = [1,1]; +end - catch - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet); +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +for iGold = 1:2 + + if iGold == 1; + halfSet = 'ODD'; + else + halfSet = 'EVE'; + end + + + + + imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet); - [ averageMotif{iGold} ] = BH_unStackMontage4d(1, ... + [ averageMotif{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... masterTM.(cycleNumber).(imgNAME){1}, ... masterTM.(cycleNumber).(imgNAME){2},... preSizeMask); + + + if ~(test_multi_ref_diffmap) + averageMotif{iGold} = averageMotif{iGold}{1}; end - - averageMotif{iGold} = averageMotif{iGold}{1}; - masterTM.(cycleNumber).(imgNAME){1} if (flgLoadMask) && (iGold == 1) fprintf('\n\nLoading external mask\n'); externalMask = getVolume(MRCImage(sprintf('%s-pcaMask',masterTM.(cycleNumber).(imgNAME){1}))); @@ -367,13 +417,19 @@ % IF combining for analysis, resample prior to any possible binning. if ~(flgGold) - averageMotif{1} = averageMotif{2} + ... - BH_resample3d(gather(averageMotif{1}), ... - oddRot, ... - aliParams(2,1:3), ... - {'Bah',1,'spline'}, 'cpu', ... - 'forward'); - averageMotif{2} = []; + if (nReferences(1) ~= nReferences(2)) + error('When combining half sets, the number of references must match') + end + size(averageMotif) + for iRef = 1:nReferences(1) + averageMotif{1}{iRef} = averageMotif{2}{iRef} + ... + BH_resample3d(gather(averageMotif{1}{iRef}), ... + oddRot, ... + aliParams(2,1:3), ... + {'Bah',1,'spline'}, 'cpu', ... + 'forward'); + averageMotif{2}{iRef} = []; + end end %%% incomplete, the idea is to generate an antialiased scaled volume for PCA @@ -457,6 +513,10 @@ end if ( flgPcaShapeMask ) + % For testing we won't handle this block + if (test_multi_ref_diffmap) + error('test_multi_ref_diffmap is incompatible with flgPcaShapeMask') + end % when combining the addition is harmless, but is a convenient way to % include when sets are left 100% separate. % volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); @@ -466,6 +526,10 @@ end if (flgLoadMask) + % For testing we won't handle this block + if (test_multi_ref_diffmap) + error('test_multi_ref_diffmap is incompatible with flgLoadMask') + end volumeMask = volumeMask .* externalMask; end @@ -479,6 +543,9 @@ stHALF = sprintf('h%d',iGold); stSCALE = sprintf('s%d',iScale); if (flgVarianceMap) + if (test_multi_ref_diffmap) + error('test_multi_ref_diffmap is incompatible with flgVarianceMap') + end volTMP = gather(volumeMask.*prevVarianceMaps.(stHALF).(stSCALE)); else volTMP = gather(volumeMask); @@ -490,9 +557,6 @@ masks.('binary').(stHALF).(stSCALE)(:); masks.('binaryApply').(stHALF).(stSCALE) = (volTMP >= 0.01); -% % % volBinaryMask = (volMask >= 0.5); -% % % volBinaryApply = (volMask >= 0.01); -% % % volBinaryMask = (volBinaryMask(:)); nPixels(iGold,iScale) = gather(sum(masks.('binary').(stHALF).(stSCALE))); clear volTMP stHALF stSCALE end @@ -507,69 +571,52 @@ threeSigma = 1/3 .* (pcaScaleSpace ./ pixelSize) for iScale = 1:nScaleSpace -% if (test_updated_bandpass) -% % Filter with low res info constant to 100 Ang, but only a tight band -% % around the desired resolution. -% lowResInfo = BH_bandpass3d(sizeMask,1e-6,400,100,'GPU',pixelSize) + ... -% BH_bandpass3d(sizeMask,1e-15,pcaScaleSpace(iScale),pcaScaleSpace(iScale),'GPU',pixelSize); -% masks.('scaleMask').(sprintf('s%d',iScale)) = gather(lowResInfo ./ max(lowResInfo(:))); -% else -% -% -% masks.('scaleMask').(sprintf('s%d',iScale)) = ... -% gather(BH_bandpass3d( sizeMask, 10^-6, 400, ... -% pcaScaleSpace(iScale).*0.9, 'GPU', pixelSize )); -% end - -% masks.('scaleMask').(sprintf('s%d',iScale)) = ... -% gather(BH_bandpass3d( sizeMask, 0.1, 400, ... -% 2.25.*pixelSize, 'GPU', pixelSize )) .* ... -% fftn(BH_multi_gaussian3d( ... -% sizeMask, -1.*stdDev(iScale))); - -% masks.('scaleMask').(sprintf('s%d',iScale)) = ... -% fftn(ifftshift(BH_multi_gaussian3d(sizeMask, 1.*stdDev(iScale)))); - - kernelSize = ceil(threeSigma(iScale)) + 3; - kernelSize = kernelSize + (1-mod(kernelSize,2)); + kernelSize = ceil(threeSigma(iScale)) + 3; + kernelSize = kernelSize + (1-mod(kernelSize,2)); masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); masks.('scaleMask').(sprintf('s%d',iScale)) + end + avgMotif_FT = cell(1+flgGold,nScaleSpace); avgFiltered = cell(1+flgGold,nScaleSpace); % Here always read in both, combine if flgGold = 0 for iGold = 1:1+flgGold for iScale = 1:nScaleSpace - % Should I set these as double? Prob - iGold -% % % avgMotif_FT{iGold, iScale} = fftn(averageMotif{iGold}.*... -% % % masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScale))) .* ... -% % % masks.('scaleMask').(sprintf('s%d',iScale)) ; - -% % % % % masks.('scaleMask') -% % % % % masks.('binaryApply').(sprintf('h%d',iGold)) -% % % % % masks.('volMask').(sprintf('h%d',iGold)) + if (test_multi_ref_diffmap) + tmp_avg = averageMotif{iGold}{iScale}; + else + tmp_avg = averageMotif{iGold}; + end - averageMotif{iGold} = averageMotif{iGold} - mean(averageMotif{iGold}(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); - averageMotif{iGold} = averageMotif{iGold} ./ rms(averageMotif{iGold}(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); - averageMotif{iGold} = averageMotif{iGold} .* masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)); - averageMotif{iGold} = EMC_convn(single(gpuArray(averageMotif{iGold})) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); + tmp_avg = tmp_avg - mean(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); + tmp_avg = tmp_avg ./ rms(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); + tmp_avg = tmp_avg .* masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)); + % FIXME: ideally we would do both, but for testing I am stealing scaleSpace for iRef + if ~(test_multi_ref_diffmap) + tmp_avg = EMC_convn(single(gpuArray(tmp_avg)) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); + end avgMotif_FT{iGold, iScale} = ... - BH_bandLimitCenterNormalize(averageMotif{iGold},... + BH_bandLimitCenterNormalize(tmp_avg,... BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)),... [0,0,0;0,0,0],'single'); - -% % % avgMotif_FT{iGold, iScale} = (real(ifftn( avgMotif_FT{iGold, iScale}))); - avgFiltered{iGold, iScale} = real(ifftn(avgMotif_FT{iGold, iScale})); + % This reproduces the orginal behavior, which wrote over averageMotif. This is a bug, but who knows, it may be beneficial, so lets for now make it optional. + if (test_scale_space_bug_fix) + if (test_multi_ref_diffmap) + averageMotif{iGold}{iScale} = tmp_avg; + else + averageMotif{iGold} = tmp_avg; + end + end + avgFiltered{iGold, iScale} = real(ifftn(avgMotif_FT{iGold, iScale})); avgFiltered{iGold, iScale} = avgFiltered{iGold, iScale} - mean(avgFiltered{iGold, iScale}(masks.('binary').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); avgFiltered{iGold, iScale} = gather(avgFiltered{iGold, iScale} ./rms(avgFiltered{iGold, iScale}(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))) .* ... masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale))); -% % % cpuVols.('avgMotif_FT').(sprintf('g%d_%d',iGold,iScale)) = gather(avgMotif_FT{iGold, iScale}); end end @@ -579,6 +626,8 @@ SAVE_IMG(MRCImage(montOUT), sprintf('test_filt.mrc'),pixelSize); clear montOUT + + % If randomSubset is string with a previous matfile use this, without any % decomposition. @@ -700,7 +749,7 @@ tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); - sprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName) + fprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); @@ -775,7 +824,6 @@ if (use_v2_SF3D && make_sf3d) make_sf3d = false; - fprintf('calculating SF3D %d \n',iSubTomo); radialGrid = ''; padWdg = [0,0,0;0,0,0]; [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, ... @@ -906,7 +954,7 @@ if (keepTomo) idxList(1, nExtracted) = particleIDX; peakList(1,nExtracted) = iPeak+1; % This probably is not necessary - it should be 1:nPEaks,1:nPeaks,1:nPeaks... - nExtracted = nExtracted +1; + nExtracted = nExtracted + 1; nTemp = nTemp + 1; % pull data of the gpu every 1000 particls (adjust this to max mem) @@ -921,12 +969,14 @@ end else nIgnored = nIgnored + 1; + fprintf('Ignoring subtomo %d from %s\n',iSubTomo, tomoList{iTomo}); masterTM.(cycleNumber).(geom_name).(tomoList{iTomo})(iSubTomo, 26+iPeak*26) = -9999; end else nIgnored = nIgnored + 1; + fprintf('Ignoring subtomo %d from %s\n',iSubTomo, tomoList{iTomo}); masterTM.(cycleNumber).(geom_name).(tomoList{iTomo})(iSubTomo, 26+iPeak*26) = -9999; end % end of ignore new particles @@ -996,6 +1046,9 @@ oldPca = load(previousPCA); U = oldPca.U; clear oldPca; + sDiag = cell(nScaleSpace,1); + coeffs = cell(nScaleSpace,1); + for iScale = 1:nScaleSpace % Sanity checks on the dimensionality numEigs = size(U{iScale}, 2); diff --git a/testScripts/fourierTransformer.m b/testScripts/fourierTransformer.m index 0956eca9..23ec9cfc 100644 --- a/testScripts/fourierTransformer.m +++ b/testScripts/fourierTransformer.m @@ -10,8 +10,8 @@ is2d; halfDim; halfDimSize; - padValIn; - padValOut; + padValIn = ''; + padValOut = ''; % end % @@ -27,6 +27,9 @@ phaseCenter = ''; indexCenterFWD = ''; indexCenterINV = '' + paddedVolumeNeedsToBeInitialized = true; + paddedVolumeIsNonZero = true; + shouldPad = false; OddSizeOversampled = 0; @@ -40,10 +43,23 @@ if nargin > 1 - if (strcmpi(varargin{1},'OddSizeOversampled')) - obj.OddSizeOversampled = 1; + if (ischar(varargin{1})) + if (strcmpi(varargin{1},'OddSizeOversampled')) + obj.OddSizeOversampled = 1; + else + error('Did not recognize the extra argument when intializing the fourierTransformer'); + end else - error('Did not recognize the extra argument when intializing the fourierTransformer'); + if (isnumeric(varargin{1})) + if (numel(varargin{1}) == 6) + padValIn = varargin{1}; + shouldPad = true; + else + error('Did not recognize the extra argument as a padding value when intializing the fourierTransformer'); + end + else + error('Did not recognize the extra argument as str or number when intializing the fourierTransformer'); + end end end % Must be single and on gpu diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 119d7ef0..b4cc727c 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=7 bugs=0 -nightly=14 +nightly=15 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" @@ -77,7 +77,8 @@ else mexCompile="" fi -${MATLAB_FOR_COMPILING} -nosplash -nodisplay -nojvm -r " ${mexCompile} mcc -m ${mFile} -a fitInMap.py -a ../alignment/emC_autoAlign -a ../alignment/emC_findBeads -a ../metaData/BH_checkInstall -R -nodisplay -o "$(basename ${mFile} .m)_${binaryOutName}" ; exit" & +# NOTE: warnings are disabled to ensure that failed builds are caught. Ideally they would be addressed and removed. +${MATLAB_FOR_COMPILING} -nosplash -nodisplay -nojvm -r " ${mexCompile} mcc -w disable -w off -m ${mFile} -a fitInMap.py -a ../alignment/emC_autoAlign -a ../alignment/emC_findBeads -a ../metaData/BH_checkInstall -R -nodisplay -o "$(basename ${mFile} .m)_${binaryOutName}" ; exit" & wait diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 66393e9c..2da6c88f 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -82,6 +82,9 @@ end +% FIXME: hacking in a test +test_fuzz=false; +test_multi_ref_diffmap= true; % Explicit reference to location of variables in main memory, or on the GPU. cpu = struct(); @@ -293,13 +296,8 @@ switch STAGEofALIGNMENT case 'RawAlignment' - - if (flgClassify) - fieldPrefix = 'Raw' - - else - fieldPrefix = 'REF' - end + fieldPrefix = 'REF'; + classVector{1} = pBH.(sprintf('%s_classes_odd','Raw')); @@ -308,11 +306,13 @@ className = pBH.(sprintf('%s_className','Raw')); samplingRate = pBH.('Ali_samplingRate'); - if (flgMultiRefAlignment && ~flgClassify) - className = pBH.(sprintf('Raw_className')) - saveClassSum = pBH.(sprintf('Raw_className')) + + if (flgMultiRefAlignment && (test_multi_ref_diffmap || ~flgClassify)) + + className = pBH.(sprintf('Raw_className')); + saveClassSum = pBH.(sprintf('Raw_className')); elseif (flgMultiRefAlignment && flgClassify) - fprintf('\n\nMutliRef and Classify enabled.\n'); + fprintf('\n\nMutliRef and Classify enabled without test_multi_ref_diffmap.\n'); fprintf('Only creating the global class average for PCA\n\n.'); className = 0; saveClassSum = 0; @@ -331,40 +331,19 @@ samplingRate = pBH.(sprintf('Cls_samplingRate')); classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); className = pBH.(sprintf('%s_className',fieldPrefix)); - - case 'NoAlignment' - % Special case for the first cycle - fieldPrefix = 'Raw'; - - classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); - classVector{2} = pBH.(sprintf('%s_classes_eve',fieldPrefix)); - - className = pBH.(sprintf('%s_className',fieldPrefix)); - samplingRate = pBH.('Ali_samplingRate'); - if (flgClassify) - % samplingRate = pBH.('Pca_samplingRate'); - else - %samplingRate = pBH.('Raw_samplingRate'); - fieldPrefix = 'REF'; - end case 'FinalAlignment' % Special case for the final cycle. % Assuming RawAlignment already run for this cycle and FSC is calculated % Goal is to re-extract odd-half, applying the xform found in fscGold - fieldPrefix = 'Raw' + fieldPrefix = 'REF' - classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); - classVector{2} = pBH.(sprintf('%s_classes_eve',fieldPrefix)); + classVector{1} = pBH.(sprintf('%s_classes_odd','Raw')); + classVector{2} = pBH.(sprintf('%s_classes_eve','Raw')); - className = pBH.(sprintf('%s_className',fieldPrefix)); + className = pBH.(sprintf('%s_className','Raw')); samplingRate = pBH.('Ali_samplingRate'); - if (flgClassify) - %samplingRate = pBH.('Pca_samplingRate'); - else - %samplingRate = pBH.('Raw_samplingRate'); - fieldPrefix = 'REF' - end + flgFinalAvg = 1; % Update at some point to handle multiple classes, for now just test on the @@ -591,8 +570,13 @@ geometry = BH_mergeClassGeometry(geometry{1}, geometry{2}); else - cN{1} = sprintf('%s_%d_%d_nClass_%d_STD',outputPrefix,classCoeffs{1}(1,1), ... + if (test_fuzz) + cN{1} = sprintf('%s_%d_%d_nClass_%d_STD','cycle002_full_2',classCoeffs{1}(1,1), ... + classCoeffs{1}(1,end), className) + else + cN{1} = sprintf('%s_%d_%d_nClass_%d_STD',outputPrefix,classCoeffs{1}(1,1), ... classCoeffs{1}(1,end), className) + end geometry = subTomoMeta.(cycleRead).ClusterResults.(cN{1}); end @@ -631,6 +615,13 @@ error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); end +class_idx=''; +class_weights=''; +if (test_fuzz) + % FIXME: testing fuzzy classification, this is hardcoded + class_idx = masterTM.(cycleRead).ClusterResults.cycle002_full_2_0_64_nClass_36_STD_idxList; + class_weights = masterTM.(cycleRead).ClusterResults.cycle002_full_2_0_64_nClass_36_STD_p; +end if isfield(masterTM,('tomoCPR_run_in_cycle')) @@ -1472,9 +1463,33 @@ % Flag the particle as ignored positionList(iSubTomo, 26:26:nPeaks*26) = -9999; else -% % % iParticle = iParticle - mean(double(iParticle(:))); -% % % iParticle = iParticle .* rms(double(iParticle(:))); + if (test_fuzz) + % Find the right row in the weight array + iProb_row = find(class_idx == positionList(iSubTomo,4)); + iProb = class_weights(iProb_row,:); + % iProb_weight = 4; + % iProb = iProb.^iProb_weight ./ sum(iProb.^iProb_weight); + iParticle = gather(iParticle .* peakWgt(iPeak)); + iWedgeMask = gather(iWedgeMask .* peakWgt(iPeak)); + + % Results are weird so override and see if it replicates the "normal" behavior by only taking the class with the minimum + % distance + [tmin,tidx] = min(iProb); + iProb = iProb.*0; + iProb(tidx) = 1; + + for iWeight = 1:length(iProb) + avgVolume_tmp{iWeight, positionList(iSubTomo,7)} = ... + avgVolume_tmp{iWeight, positionList(iSubTomo,7)} + (iParticle .* iProb(iWeight)); + + avgWedge_tmp{ iWeight, positionList(iSubTomo,7)} = ... + avgWedge_tmp{ iWeight, positionList(iSubTomo,7)} + (iWedgeMask .* iProb(iWeight)); + nExtracted_tmp(iWeight, positionList(iSubTomo,7)) = ... + nExtracted_tmp(iWeight, positionList(iSubTomo,7)) + iProb(iWeight); + end + + else if positionList(iSubTomo,7) == 1 %%%%|| (flgGold == 0) iTempParticleODD = iTempParticleODD + iParticle.*peakWgt(iPeak); @@ -1508,6 +1523,8 @@ tomoCount = tomoCount + 1; end + end % end test fuzz + end else @@ -1540,14 +1557,17 @@ gather(iTempWedgeEVE); end else - avgVolume_tmp{ iClassPos, 1} = avgVolume_tmp{ iClassPos, 1} + ... - gather(iTempParticleODD); - avgVolume_tmp{ iClassPos, 2} = avgVolume_tmp{ iClassPos, 2} + ... - gather(iTempParticleEVE); - avgWedge_tmp{ iClassPos, 1} = avgWedge_tmp{ iClassPos, 1} + ... - gather(iTempWedgeODD); - avgWedge_tmp{ iClassPos, 2} = avgWedge_tmp{ iClassPos, 2} + ... - gather(iTempWedgeEVE); + % for the fuzz each particle goes into every class, so we need to pull earlier + if ~(test_fuzz) + avgVolume_tmp{ iClassPos, 1} = avgVolume_tmp{ iClassPos, 1} + ... + gather(iTempParticleODD); + avgVolume_tmp{ iClassPos, 2} = avgVolume_tmp{ iClassPos, 2} + ... + gather(iTempParticleEVE); + avgWedge_tmp{ iClassPos, 1} = avgWedge_tmp{ iClassPos, 1} + ... + gather(iTempWedgeODD); + avgWedge_tmp{ iClassPos, 2} = avgWedge_tmp{ iClassPos, 2} + ... + gather(iTempWedgeEVE); + end end end % end of loop over classes end% Update geometry to include information on ignored particles. @@ -1810,10 +1830,11 @@ % multi-reference alignment. if strcmpi(STAGEofALIGNMENT, 'Cluster') masterTM.(cycleNumber).(ClusterGeomNAME) = geometry; -elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') && flgMultiRefAlignment +elseif ( strcmpi(STAGEofALIGNMENT, 'RawAlignment') && flgMultiRefAlignment ) if (flgClassify) masterTM.(cycleNumber).('ClusterClsGeom') = geometry; else + error('This branch is broken, ClusterRefGeom'); masterTM.(cycleNumber).('ClusterRefGeom') = geometry; end else @@ -1879,16 +1900,16 @@ [montOUT, imgLocations] = BH_montage4d(classSum(iGold), ''); imout = sprintf('%s_class%d_%s_%s_NoWgt.mrc',outputPrefix, ... - saveClassSum, 'Raw', halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'Raw', halfSet); + saveClassSum, 'REF', halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'REF', halfSet); SAVE_IMG(montOUT, imout,pixelSize); masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; [montOUT, imgLocations] = BH_montage4d(classWgtSum(iGold), ''); imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... - saveClassSum, 'Raw', halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'Raw', halfSet); + saveClassSum, 'REF', halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'REF', halfSet); SAVE_IMG(montOUT, imout,pixelSize); masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; @@ -1995,7 +2016,7 @@ for iRef = 1:nClassesReWgt fprintf('Stage of alignment %s\niRef %d\n',STAGEofALIGNMENT,iRef); if strcmpi(STAGEofALIGNMENT, 'RawAlignment') || strcmpi(STAGEofALIGNMENT, 'NoAlignment') - savePrefix = 'Raw' + savePrefix = 'REF' else savePrefix = fieldPrefix end @@ -2029,6 +2050,7 @@ aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); catch + error('This block should not be reached.'); fprintf('\nReverting from %s to Raw in loading fitFSC\n',savePrefix); fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d','Raw',iRefPrev)); aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)); diff --git a/transformations/BH_multi_loadOrBuild.m b/transformations/BH_multi_loadOrBuild.m index 42ecfa39..1ff4dfc4 100755 --- a/transformations/BH_multi_loadOrBuild.m +++ b/transformations/BH_multi_loadOrBuild.m @@ -174,17 +174,17 @@ failedLoads = 0; while failedLoads < 6 try - fprintf('pwd is %s\n', pwd); - fprintf(... - 'attempting to load %s\n', recon); + % fprintf('pwd is %s\n', pwd); + % fprintf(... + % 'attempting to load %s\n', recon); m = MRCImage(sprintf('%s', recon),0); - fprintf('Loaded the MRCImage\n'); + % fprintf('Loaded the MRCImage\n'); if ( flgLoad ) IMG_OUT = single(getVolume(m)); - fprintf('Loaded the volume\n'); + % fprintf('Loaded the volume\n'); else IMG_OUT = m; - fprintf('Did not load the full Volume\n'); + % fprintf('Did not load the full Volume\n'); end failedLoads = 6; catch From 81fbf3f3c557259e2c9e1e35ae239a91efbe9486 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 08:04:13 -0500 Subject: [PATCH 003/151] Removes previously deprecated stage of alignment NoAlignment --- .../runTutorial.sh | 96 ------------------- statistics/BH_fscGold_class.m | 4 +- transformations/BH_average3d.m | 61 ++++++------ 3 files changed, 28 insertions(+), 133 deletions(-) delete mode 100755 docs/exampleParametersAndRunScript/runTutorial.sh diff --git a/docs/exampleParametersAndRunScript/runTutorial.sh b/docs/exampleParametersAndRunScript/runTutorial.sh deleted file mode 100755 index 7e66e71e..00000000 --- a/docs/exampleParametersAndRunScript/runTutorial.sh +++ /dev/null @@ -1,96 +0,0 @@ -skipThis=1 -runThis=1 - -# -# - - - -if [[ ${skipThis} -eq 0 ]] ; then - exit - -fi # - emClarity init param0.m ; [[ $? -ne 0 ]] && exit - - emClarity ctf update param0.m ; [[ $? -ne 0 ]] && exit - - emClarity ctf 3d param0.m ; [[ $? -ne 0 ]] && exit - - - - for i in 0 1 2 ; do - if [[ $i -eq 0 ]] ; then - ST='NoAlignment' - else - ST='RawAlignment' - fi - - if [[ $i -ge 0 ]] ; then emClarity avg param${i}.m ${i} $ST ; fi; [[ $? -ne 0 ]] && exit - - if [[ $i -ge 0 ]] ; then emClarity alignRaw param${i}.m ${i}; fi; [[ $? -ne 0 ]] && exit - - done - - - - emClarity removeDuplicates param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity tomoCPR param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity ctf update param$((${i}+1)).m ; [[ $? -ne 0 ]] && exit - - emClarity ctf 3d param$((${i}+1)).m; [[ $? -ne 0 ]] && exit - - - - for i in 3 4 5 ; do - - if [[ $i -ge 3 ]] ; then emClarity avg param${i}.m ${i} RawAlignment ; fi; [[ $? -ne 0 ]] && exit - - if [[ $i -ge 3 ]] ; then emClarity alignRaw param${i}.m ${i} ; fi; [[ $? -ne 0 ]] && exit - - done - - - - emClarity removeDuplicates param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity tomoCPR param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity ctf update param$((${i}+1)).m ; [[ $? -ne 0 ]] && exit - - emClarity ctf 3d param$((${i}+1)).m; [[ $? -ne 0 ]] && exit - - - - - for i in 6 7 8 ; do - if [[ $i -ge 6 ]] ; then emClarity avg param${i}.m ${i} RawAlignment ; fi; [[ $? -ne 0 ]] && exit - - if [[ $i -ge 6 ]] ; then emClarity alignRaw param${i}.m ${i} ; fi; [[ $? -ne 0 ]] && exit - - done - - emClarity removeDuplicates param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity tomoCPR param${i}.m ${i} ; [[ $? -ne 0 ]] && exit - - emClarity ctf update param$((${i}+1)).m ; [[ $? -ne 0 ]] && exit - - emClarity ctf 3d param$((${i}+1)).m ; [[ $? -ne 0 ]] && exit - - - - for i in 9 10 11 12 ; do - - if [[ $i -ge 9 ]] ; then emClarity avg param${i}.m ${i} RawAlignment ; fi; [[ $? -ne 0 ]] && exit - - if [[ $i -ge 9 ]] ; then emClarity alignRaw param${i}.m ${i} ; fi; [[ $? -ne 0 ]] && exit - - done - - - emClarity avg param13.m 13 RawAlignment; [[ $? -ne 0 ]] && exit - emClarity avg param13.m 13 FinalAlignment; [[ $? -ne 0 ]] && exit - # fi # end of skipThis -#fi # end of runTHis diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 896e3ae7..dc30ed05 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -159,14 +159,13 @@ elseif size(STAGEofALIGNMENT{1}) ~= size(STAGEofALIGNMENT{2}) error('Size of img1 and img2 are inconsistent.') else - %flgAlignImages = 0; IMG1 = STAGEofALIGNMENT{1}; IMG2 = STAGEofALIGNMENT{2}; flgJustFSC=1; nReferences=2; refVector{1} =1; refVector{2}= 1; - STAGEofALIGNMENT = 'NoAlignment'; + STAGEofALIGNMENT = 'RawAlignment'; fieldPrefix = 'REF' end else @@ -174,7 +173,6 @@ switch STAGEofALIGNMENT case 'RawAlignment' savePrefix = 'Raw'; -% % % % if (flgClassify || flgMultiRefAlignment) if (flgClassify) fieldPrefix = 'Raw'; diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 2da6c88f..3a7dc58d 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -93,10 +93,6 @@ startTime = clock; CYCLE = EMC_str2double(CYCLE); -% Until people forget this existed -if strcmpi(STAGEofALIGNMENT,'NoAlignment') - STAGEofALIGNMENT = 'RawAlignment'; -end if strcmpi(STAGEofALIGNMENT, 'RawAlignment') % Ensure we don't have any duplicates: TODO: add an override flag @@ -296,8 +292,13 @@ switch STAGEofALIGNMENT case 'RawAlignment' - fieldPrefix = 'REF'; - + + if (flgClassify) + fieldPrefix = 'Raw' + + else + fieldPrefix = 'REF' + end classVector{1} = pBH.(sprintf('%s_classes_odd','Raw')); @@ -306,13 +307,11 @@ className = pBH.(sprintf('%s_className','Raw')); samplingRate = pBH.('Ali_samplingRate'); - if (flgMultiRefAlignment && (test_multi_ref_diffmap || ~flgClassify)) - - className = pBH.(sprintf('Raw_className')); - saveClassSum = pBH.(sprintf('Raw_className')); + className = pBH.(sprintf('Raw_className')) + saveClassSum = pBH.(sprintf('Raw_className')) elseif (flgMultiRefAlignment && flgClassify) - fprintf('\n\nMutliRef and Classify enabled without test_multi_ref_diffmap.\n'); + fprintf('\n\nMutliRef and Classify enabled.\n'); fprintf('Only creating the global class average for PCA\n\n.'); className = 0; saveClassSum = 0; @@ -336,14 +335,19 @@ % Special case for the final cycle. % Assuming RawAlignment already run for this cycle and FSC is calculated % Goal is to re-extract odd-half, applying the xform found in fscGold - fieldPrefix = 'REF' + fieldPrefix = 'Raw' - classVector{1} = pBH.(sprintf('%s_classes_odd','Raw')); - classVector{2} = pBH.(sprintf('%s_classes_eve','Raw')); + classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); + classVector{2} = pBH.(sprintf('%s_classes_eve',fieldPrefix)); - className = pBH.(sprintf('%s_className','Raw')); + className = pBH.(sprintf('%s_className',fieldPrefix)); samplingRate = pBH.('Ali_samplingRate'); - + if (flgClassify) + %samplingRate = pBH.('Pca_samplingRate'); + else + %samplingRate = pBH.('Raw_samplingRate'); + fieldPrefix = 'REF' + end flgFinalAvg = 1; % Update at some point to handle multiple classes, for now just test on the @@ -531,14 +535,6 @@ switch STAGEofALIGNMENT case 'ClassAlignment' geometry = subTomoMeta.(cycleRead).ClassAlignment; -%%% case 'NoAlignment' -%%% geometry = subTomoMeta.(cycleRead).geometry; - % Extract an average from each tomogram for visual inspection, filter to - % 0.75xtemplateSearch_lowpass -%%% eachTomo = true; -% %% lpTomo = pBH.('Tmp_bandpassFilter'); -%%% doNotTrim = true; - case 'RawAlignment' if ( CYCLE ) geometry = subTomoMeta.(cycleRead).RawAlign; @@ -1830,11 +1826,10 @@ % multi-reference alignment. if strcmpi(STAGEofALIGNMENT, 'Cluster') masterTM.(cycleNumber).(ClusterGeomNAME) = geometry; -elseif ( strcmpi(STAGEofALIGNMENT, 'RawAlignment') && flgMultiRefAlignment ) +elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') && flgMultiRefAlignment if (flgClassify) masterTM.(cycleNumber).('ClusterClsGeom') = geometry; else - error('This branch is broken, ClusterRefGeom'); masterTM.(cycleNumber).('ClusterRefGeom') = geometry; end else @@ -1900,16 +1895,16 @@ [montOUT, imgLocations] = BH_montage4d(classSum(iGold), ''); imout = sprintf('%s_class%d_%s_%s_NoWgt.mrc',outputPrefix, ... - saveClassSum, 'REF', halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'REF', halfSet); + saveClassSum, 'Raw', halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'Raw', halfSet); SAVE_IMG(montOUT, imout,pixelSize); masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; [montOUT, imgLocations] = BH_montage4d(classWgtSum(iGold), ''); imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... - saveClassSum, 'REF', halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'REF', halfSet); + saveClassSum, 'Raw', halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'Raw', halfSet); SAVE_IMG(montOUT, imout,pixelSize); masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; @@ -1948,7 +1943,6 @@ % The final condition is setting up RefAlignment which with classification % would have been averaging with 'Cluster' == Stageof if strcmpi(STAGEofALIGNMENT, 'RawAlignment') || ... - strcmpi(STAGEofALIGNMENT, 'NoAlignment') || ... strcmpi(STAGEofALIGNMENT, 'RefAlignment') BH_fscGold_class(PARAMETER_FILE, num2str(CYCLE), STAGEofALIGNMENT); elseif strcmpi(STAGEofALIGNMENT, 'Alignment') @@ -2015,8 +2009,8 @@ for iRef = 1:nClassesReWgt fprintf('Stage of alignment %s\niRef %d\n',STAGEofALIGNMENT,iRef); - if strcmpi(STAGEofALIGNMENT, 'RawAlignment') || strcmpi(STAGEofALIGNMENT, 'NoAlignment') - savePrefix = 'REF' + if strcmpi(STAGEofALIGNMENT, 'RawAlignment') + savePrefix = 'Raw' else savePrefix = fieldPrefix end @@ -2050,7 +2044,6 @@ aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); catch - error('This block should not be reached.'); fprintf('\nReverting from %s to Raw in loading fitFSC\n',savePrefix); fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d','Raw',iRefPrev)); aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)); From 1d8e27d4e6ccfcbcc3e8b58a013133a36696346c Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 08:17:27 -0500 Subject: [PATCH 004/151] Removes StageOfAlignment ClassAlignment, as well as old functions to align classes to each other and related functionality that is years long broken --- alignment/BH_alignClassRotAvg3d.m | 670 ----------- alignment/BH_alignRaw3d.m | 1676 ---------------------------- alignment/BH_alignReferences3d.m | 759 ------------- metaData/BH_classAlignmentsApply.m | 89 -- metaData/BH_geometryAnalysis.m | 33 +- metaData/BH_skipClassAlignment.m | 59 +- statistics/BH_fscGold_class.m | 5 - testScripts/emClarity.m | 1 - transformations/BH_average3d.m | 14 +- 9 files changed, 10 insertions(+), 3296 deletions(-) delete mode 100755 alignment/BH_alignClassRotAvg3d.m delete mode 100755 alignment/BH_alignRaw3d.m delete mode 100755 alignment/BH_alignReferences3d.m delete mode 100755 metaData/BH_classAlignmentsApply.m diff --git a/alignment/BH_alignClassRotAvg3d.m b/alignment/BH_alignClassRotAvg3d.m deleted file mode 100755 index 7be7a7d0..00000000 --- a/alignment/BH_alignClassRotAvg3d.m +++ /dev/null @@ -1,670 +0,0 @@ -function [ ] = BH_alignClassRotAvg3d(PARAMETER_FILE, CYCLE) - - -%Extract and align class averages and references from 4D montages derived. -% -% Input variables: -% -% IMAGE = 4d volume, or a string specifing a volume to read in. -% -% CLASSES = Align subset of class averages. [1, 2, 5, 6] -% -% CLASS_NAME = a number (e.g. 64) that refers to the class/montage to use. -% -% REFERENCES a list same as CLASSES with the class id, and symmetry to apply. -% -% REF_NAME = a number (e.g. 8) that refers to the class/montage to draw the -% references from. -% -% REAL_MASK = {maskType, maskSize, maskRadius, maskCenter} -% -% BANDPASS = [HIGH_THRESH, HIGH_CUT, LOW_CUT, PIXEL_SIZE] applied to the -% particle prior to interpolation. -% -% ANGLE_SEARCH = [a1 a2 a3 a4 ] the angular search is a grid searched -% designed to exhaustively sample a unit sphere over the range -% you specify. Out of plane +/- a1 in steps of size a2, in -% plane +/- a3 in steps of a4. These together define a set of -% "latitudes" if you will, and the in plane sampling at each -% point sampled on that latitude, while the longitudinal -% sampling is calculated to be evenly sampled at the same rate -% as the latitude. The smaller the out of plane step size, the -% more longitudinal sampling points, and also the larger the -% absolute out of plane angle, the greater number of steps it -% takes to make a full revolution. -% -% PEAK = [x y z] = *RADIUS* of peak search -% PEAK_MASS = [x y z] = *RADIUS* for center of mass search around max peak -% Set to a zero to ignore either option. -% -% REFERENCES = a cell with list of images to use as references. -% -% GEOMETRY = A structure with tomogram names as the field names, and geometry -% information in a 26 column array. -% Additionally, a field called 'source_path' has a value with the -% absolute path to the location of the tomograms. -% -% The input is a string 'Geometry_templatematching.mat' for -% example, and it is expected that the structure is saved as the -% variable named geometry. -% -% -% OUTPUT_PREFIX = String to prepend to output volumes. -% -% -% Output variables: -% -% None = files are written to disk in the current directory. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% Goals & Limitations: -% -% Align class averages to a reference. Here it is implicitly assumed that the -% references and the volumes they will be aligned against are the same -% dimension. I am not going to remove all of the information and steps related -% to binning, as these will be needed when writing the function to handle -% alignment of raw subTomos, but later I will clean this up. -% -% Assumed to run on GPU. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% TODO - -% - Update geometry to record % sampling -% - deal with assumption that 256 256 256 is sufficient for volumes. -% - Add error check for mask size/ v radius -% - Change angular searches to allow for a translational only search. -% - Work through angular sampling to be sure it is doing what you think it -% is. -% - Read sampling from metadata to deal with bandpass. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -if (nargin ~= 2) - error('args = PARAMETER_FILE, CYCLE') -end - -PRECISION = 'single'; - -startTime = clock; -CYCLE = EMC_str2double(CYCLE); - - -cycleNumber = sprintf('cycle%0.3u', CYCLE); - -pBH = BH_parseParameterFile(PARAMETER_FILE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - -maxGoldStandard = subTomoMeta.('maxGoldStandard'); - -bFactor = pBH.('Fsc_bfactor'); - -try - flgRotAvgRef = pBH.('flgRotAvgRef'); -catch - flgRotAvgRef = 0; -end -samplingRate = pBH.('Ali_samplingRate'); -pixelSize = pBH.('PIXEL_SIZE').*samplingRate.*10^10; -if pBH.('SuperResolution') - pixelSize = pixelSize * 2; -end -try - scaleCalcSize = pBH.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end - -angleSearch = pBH.('Cls_angleSearch'); -refName = pBH.('Cls_className'); %pBH.('Ref_className'); -className = pBH.('Cls_className'); -peakSearch = floor(pBH.('particleRadius')./pixelSize) -peakCOM = [1,1,1].*peakCOM; -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); - - - -flgAngleShift{1}= pBH.('ref_AngleShift_odd'); -flgTransShift{1}= pBH.('ref_TransShift_odd'); -flgRefRef{1} = pBH.('ref_Ref_odd'); -refVectorFull{1}= pBH.('Ref_references_odd'); -classVector{1} = pBH.('Cls_classes_odd')(1,:); -features{1} = pBH.('Pca_coeffs_odd'); -vol_geometry{1} = subTomoMeta.(cycleNumber).('ClusterResults').( ... - sprintf('%s_%d_%d_nClass_%d_ODD', ... - outputPrefix,features{1}(1,1), ... - features{1}(1,end), className)); - -flgAngleShift{2}= pBH.('ref_AngleShift_eve'); -flgTransShift{2}= pBH.('ref_TransShift_eve'); -flgRefRef{2} = pBH.('ref_Ref_eve'); -refVectorFull{2}= pBH.('Ref_references_eve'); -classVector{2} = pBH.('Cls_classes_eve')(1,:); -features{2} = pBH.('Pca_coeffs_eve'); -vol_geometry{2} = subTomoMeta.(cycleNumber).('ClusterResults').( ... - sprintf('%s_%d_%d_nClass_%d_EVE', ... - outputPrefix,features{2}(1,1), ... - features{2}(1,end), className)); -% Merge alignments that could come from using different feature vectors in the -% classification, back into one metadata. -vol_geometry = BH_mergeClassGeometry(vol_geometry{1},vol_geometry{2}); - - - - - - -refVector = cell(2,1); -refGroup = cell(2,1); -refSym = cell(2,1); - -for iGold = 1:2 - % Sort low to high, because order is rearranged as such unstack - refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; - % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) - % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) - % axial symmetry to apply, negative value indicates creating a mirrored ref - % accros the corresponding axis - refSym{iGold} = refVectorFull{iGold}(2,:) -end - - -pathList= subTomoMeta.mapPath; -extList = subTomoMeta.mapExt; -masterTM = subTomoMeta; clear subTomoMeta - -% make sure the number of references match the unique groups in the classVector -% and also that the class/group pairs match the class/ref pairs. -nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})] - -uniqueSym = cell(2,1); -for iGold = 1:2 - [~,uniqueGroup,~] = unique(refGroup{iGold}); - uniqueSym{iGold} = refSym{iGold}(uniqueGroup) -end - - - - - - -[ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Cls', samplingRate); -[ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) -padREF = [0,0,0;0,0,0]; -if any(peakSearch > maskRadius) - fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') - peakSearch( (peakSearch > maskRadius) ) = ... - maskRadius( (peakSearch > maskRadius) ); -end - - -% Read in the references. -refIMG = cell(2,1); -imgCounts = cell(2,1); - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet); - weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', refName, halfSet); - - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - [ refIMG{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - - [ refWDG{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... - sizeCalc); - - - sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}; - sizeREF = sizeREF(2:2:6)' - - -end - -[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, maxGoldStandard); - - -% % % [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc, padREF ] = ... -% % % BH_multi_validArea( maskRadius, sizeREF ) -% optimize the fft for the given size. Padding to the next power of 2 is usually -% slower given the dimensionality of the volume data. -fftPlanner = rand(sizeCalc); -fftw('planner', 'exhaustive'); -fftn(fftPlanner); -clear fftPlanner - -% Make a mask, and apply to the average motif && save a masked, binned copy of -% the average for inspection. - - -[ volMask ] = gpuArray(BH_mask3d(maskType, sizeMask, maskRadius, maskCenter)); -volBinary = (volMask > 0.01); - -[ peakMask] = gpuArray(BH_mask3d(maskType, sizeMask, peakSearch, maskCenter)); -peakBinary = (peakMask > 0.01); - -bandpassFilt = cell(nReferences(1),1); - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'cpu', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); -for iRef = 1:nReferences(1) - - - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); - - % The class averages have roughly the same SNR as the references so apply any - % bFactor to them as well. - [ ~, bandpassFilt{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, bFactor,1 ); - bandpassFilt{iRef} = gpuArray(bandpassFilt{iRef}); -end - -bestAnglesResults = cell(2,1); - - -try - EMC_parpool(2) -catch - delete(gcp('nocreate')) - EMC_parpool(2) -end - -parfor iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - - ref_FT = zeros([sizeWindow,nReferences(iGold)], PRECISION, 'gpuArray'); - refRotAvg_FT = zeros([sizeWindow,nReferences(iGold)], PRECISION, 'gpuArray'); - - - refRotAvg = refIMG{iGold}; - refTrans = cell(length(refIMG{iGold}).*2); - nRefOut = 1; - for iRef = 1:nReferences(iGold) - - - if (flgRotAvgRef) - refRotAvg{iRef} = gather( ... - BH_axialSymmetry(gpuArray(refIMG{iGold}{iRef}),120, 0, ... - 'GPU', [0,0,0])); - - % For later stages where the angles are very small, still scan only - % azimuthal and out of plane, but don't rotationally average the in-plane - % angles. - else - refRotAvg{iRef} = refIMG{iGold}{iRef}; - - - end - - padTransTrim = padWindow + padREF; - refTransTrim = refIMG{iGold}{iRef}(padTransTrim(1,1)+1 : end - padTransTrim(2,1), ... - padTransTrim(1,2)+1 : end - padTransTrim(2,2), ... - padTransTrim(1,3)+1 : end - padTransTrim(2,3) ); - - refTrans{nRefOut} = real(ifftn( BH_bandLimitCenterNormalize(refTransTrim.*volMask, ... - bandpassFilt{iRef}, volBinary,padCalc,'double'))); - - refTrans{nRefOut} = gather(refTrans{nRefOut}(padCalc(1,1) + 1: end - padCalc(2,1),... - padCalc(1,2) + 1: end - padCalc(2,2),... - padCalc(1,3) + 1: end - padCalc(2,3)) .*volMask); - - refTrans{nRefOut+1} = real(ifftn( BH_bandLimitCenterNormalize(refTransTrim.*peakMask, ... - bandpassFilt{iRef}, peakBinary,padCalc,'double'))); - - refTrans{nRefOut+1} = gather(refTrans{nRefOut+1}(padCalc(1,1) + 1: end - padCalc(2,1),... - padCalc(1,2) + 1: end - padCalc(2,2),... - padCalc(1,3) + 1: end - padCalc(2,3)) .*peakMask); - - nRefOut = nRefOut +2; - - -iGold -iRef -size(refIMG{iGold}{iRef}) -size(ref_FT) - %winRefDiff = padREF - padWindow; - ref_FT(:,:,:,iRef) = refIMG{iGold}{iRef}( ... - padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - refRotAvg_FT(:,:,:,iRef) = refRotAvg{iRef}( ... - padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - - - - - end - refMontage = BH_montage4d(refTrans,''); - SAVE_IMG(MRCImage(refMontage),sprintf('%s_class_refFiltered_%s.mrc',cycleNumber,halfSet)); - % Read in the class averages. -% refMontage{iGold} = BH_montage4d(refTrans,''); - imgClassNAME = sprintf('class_%d_Locations_%s_%s_NoWgt', className, 'Cls', halfSet); - wdgClassNAME = sprintf('class_%d_Locations_%s_%s_Wgt', className, 'Cls', halfSet); - - [ classIMG ] = BH_unStackMontage4d(classVector{iGold}, ... - masterTM.(cycleNumber).(imgClassNAME){1}, ... - masterTM.(cycleNumber).(imgClassNAME){2},sizeWindow); - - [ classWDG ] = BH_unStackMontage4d(classVector{iGold}, ... - masterTM.(cycleNumber).(imgClassNAME){1}, ... - masterTM.(cycleNumber).(imgClassNAME){2},sizeWindow); - - - nClassesPossible = length(classVector{iGold}) - nClasses = length(classIMG) - - %%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't - %%%%%%%%%%%%%%%%%%%%% search at all in that dimension. - [ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch) - - fprintf('%d ',inPlaneSearch); - fprintf('\n'); - - % Store the cross correlation score, peak location, and wedge weight - bestAnglesTotal = zeros(nClassesPossible,10); - nCount = 1; - - - for iClass = classVector{iGold} - - tic; - - - % Load the class into gpu, center and normalize - iClassImg = gpuArray(classIMG{iClass}( ... - padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) )); - - - iClassWdg = ifftshift(gpuArray(classWDG{iClass}( ... - padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ))); - - - - % [ iClassImg ] = BH_padZeros3d(iClassImg, padPre, padPost, 'GPU', 'single'); - %[ iClassImg ] = BH_bandLimitCenterNormalize(iClassImg, bandpassFilt, volMask); - % iClassImg = real(ifftn(iClassImg)); - % First loop over all out of plane, no in plane, with rotationally averaged - % reference. - cccStorage1 = []; - cccStorage2 = []; - cccStorage3 = []; - cccStorage4 = []; - % Out of plane search - if any(angleStep(2:4)) - flgSearchDepth = 3; - elseif any(angleStep(5)) - % in plane - flgSearchDepth = 2; - peakListTop10(angleStep(5).*nReferences(iGold),6) = gpuArray(0); - - nPeak = 1; - for iRef = 1 - for iPsi = inPlaneSearch - peakListTop10(nPeak,1) = iRef; - nPeak = nPeak + 1; - end - end - else - error('specify at least an in plane search') - end - - - if (flgSearchDepth == 3) - % Note the rotationally averaged ref is passed as main ref - % - [ cccStorage1 ] = BH_multi_angularSearch( angleStep, 0, 0, ... - iClassImg, iClassWdg, ... - refRotAvg_FT, NaN, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padWindow,... - peakMask, peakCOM, iClass, ... - uniqueSym{iGold}); - - - cccStorage1(1:10,:) - - - % Second loop over top 10 peaks now using the non-rotationally averaged - % reference and including out of plane angles. - - % peakList is # rows = top peaks - % reference, phi, theta - % put zero peak at top of list - zeroPeak = sortrows(cccStorage1,[3, 4, 5]); - zeroPeak = [zeroPeak(1,:) ; cccStorage1 ]; - % if zero peak was already there, remove it so no duplicate - zeroPeak = unique(zeroPeak, 'stable', 'rows'); - peakListTop10 = [zeroPeak(1:10,1),zeroPeak(1:10,3:4),zeroPeak(1:10,8:10)] - end - - [ cccStorage2 ] = BH_multi_angularSearch( angleStep, peakListTop10, ... - inPlaneSearch, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG{iGold}, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM,iClass, ... - uniqueSym{iGold}); - - - cccStorage2 = unique(cccStorage2((cccStorage2(:,6) ~= 0),:), 'stable','rows'); - if size(cccStorage2, 1) > 9 - cccStorage2(1:10,:) - else - cccStorage2 - end - % This is to save time assuming that we can get a good estimate of the - % particles shift by taking the average of the higher ranking alignments. In - % testing this was always within ~ half a pixel. If CCC scores are strangely - % low, suspect this as a break point. - - - % Get the top three peaks with unique phi, and theta - % Return [ref,phi,theta,psi,phistep,thetastep,psistep] - % Stable prevents any sorting - - if (flgSearchDepth== 3) - [~,ia,~] = unique(cccStorage2(:,3:4),'stable' ,'rows'); - peakListTop3 = zeros(3,10); - else - [~,ia,~] = unique(cccStorage2(:,3:5),'stable' ,'rows'); - peakListTop3 = zeros(3,10); - end - - if numel(ia) >= 10 - TOP = 10; - else - TOP = numel(ia); - end - - for top3 = 1:TOP - outOfPlaneAngle = cccStorage2(ia(top3),4); - angleIndex = find(angleStep(:,1)==outOfPlaneAngle,1,'first'); - if (flgSearchDepth == 3 ) - % Search around top 3 +/- 0.5 the original out of plane angular increment - peakListTop3(top3,:) = [cccStorage2(ia(top3),1), ... - cccStorage2(ia(top3),3:5),... - angleStep(angleIndex,3)./4,... - angleStep(angleIndex,4)./2,... - angleStep(angleIndex,5)./2, cccStorage2(ia(top3),8:10)]; - else - % Search around top 3 +/- 0.5 the original out of plane angular increment - peakListTop3(top3,:) = [cccStorage2(ia(top3),1), ... - cccStorage2(ia(top3),3:5),... - 0,... - 0,... - angleStep(1,5)./2, cccStorage2(ia(top3),8:10)]; - end - end - peakListTop3 - [ cccStorage3 ] = BH_multi_angularSearch( angleStep, peakListTop3, ... - 0, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG{iGold}, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask,peakCOM,iClass, ... - uniqueSym{iGold}); - - - cccStorage3 = unique(cccStorage3((cccStorage3(:,6) ~= 0),:), 'stable','rows'); - cccStorage3(1:10,:) - - if (flgSearchDepth == 3 ) - % Use previous increments/2 - - % Search around the top peak +/- 0.25 the orginal angular increment - topPeak = [cccStorage3(1,1), ... - cccStorage3(1,3:5), ... - cccStorage3(1,11:13)./2, ... - cccStorage3(1,8:10)] - else - topPeak = [cccStorage3(1,1), ... - cccStorage3(1,3:5), ... - 0, ... - 0,... - angleStep(1,5)./3, cccStorage3(1,8:10)] - end - - - [ cccStorage4 ] = BH_multi_angularSearch( angleStep, topPeak, ... - 0, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG{iGold}, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padWindow,... - peakMask, peakCOM,iClass, ... - uniqueSym{iGold}); - - cccStorage4(1,:) - - - bestAnglesTotal(iClass,:) = gather(cccStorage4(1,1:10)); - - timeClass = toc; - fprintf('finished working on %d/%d classes...%fs\n',nCount,nClassesPossible,timeClass) - nCount = nCount + 1; - - % Save alignment incase of crash, long runs can be resumed. - -%%% save('bestAnglesTotalClass.mat', 'bestAnglesTotal'); - - - - - end % loop over (classes) - - bestAnglesResults{iGold} = gather(bestAnglesTotal); - % Save a text copy of results. - resultsOut = fopen(sprintf('%s_bestAngles_%s.txt', cycleNumber, halfSet),'w'); - fprintf(resultsOut,'%d %d %3.3f %3.3f %3.3f %1.6f %d %4.4f %4.4f %4.4f\n',bestAnglesTotal'); - fclose(resultsOut); -end - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - imgClassNAME = sprintf('class_%d_Locations_%s_%s', className, 'Cls', halfSet); - [ classIMG ] = BH_unStackMontage4d(classVector{iGold}, ... - masterTM.(cycleNumber).(imgClassNAME){1}, ... - masterTM.(cycleNumber).(imgClassNAME){2},sizeWindow); - - [ vol_geometry ] = BH_classAlignmentsApply( vol_geometry, bestAnglesResults{iGold},... - samplingRate,classVector{iGold},iGold,halfSet); - - - - % SAVE_IMG(MRCImage(refMontage{iGold}),sprintf('%s_class_refFiltered_%s.mrc',cycleNumber,halfSet)); - % Max a montage of the applied corrections for viewing. - % Insert blank images so subsets are still corresponding. - classInterp = cell(className,1); - emptyImg = zeros(sizeWindow, 'single'); - for iClass = 1:className - if ismember(iClass, classVector{iGold}) - iClass - - iClassImg = single(gpuArray(classIMG{iClass})); - sizeMask - size(iClassImg) - %figure, imshow3D(gather(iClassImg)) ; pause(2) ; close(gcf) - % Note 'forward' affects the shifts but not the angles, a value of 10 means the - % class was 10 away from the ref - idx = find(bestAnglesResults{iGold}(:,2) == iClass); - iClassImg = BH_resample3d(iClassImg, bestAnglesResults{iGold}(idx, 3:5), ... - bestAnglesResults{iGold}(idx, 8:10), ... - 'Bah', 'GPU', 'inv'); - else - - iClassImg = emptyImg; - end - - classInterp{iClass} = gather(single(iClassImg(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3)) )); - - - end - - - try - [montOUT] = BH_montage4d(classInterp, ''); - imout = sprintf('%s_class%d_%s_aligned.mrc',outputPrefix, className,halfSet); - SAVE_IMG(MRCImage(gather(montOUT)), imout); - catch - fprintf('error in montaging the aligned classes line 603.\n') - end -end - -masterTM.(cycleNumber).('ClassAlignment') = vol_geometry; -subTomoMeta = masterTM; -save(pBH.('subTomoMeta'), 'subTomoMeta'); - -fprintf('Total execution time : %f seconds\n', etime(clock, startTime)); -gpuDevice(1); - -try - delete(gcp('nocreate')) -catch -end -end % end of average3d function - - diff --git a/alignment/BH_alignRaw3d.m b/alignment/BH_alignRaw3d.m deleted file mode 100755 index 87d2e20a..00000000 --- a/alignment/BH_alignRaw3d.m +++ /dev/null @@ -1,1676 +0,0 @@ - function [ ] = BH_alignRaw3d(PARAMETER_FILE, CYCLE, varargin) - -%Extract and align class averages and references from 4D montages derived. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% Goals & Limitations: -% -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% TODO -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -global bh_global_print_shifts_in_particle_basis; -if isempty(bh_global_print_shifts_in_particle_basis) - bh_global_print_shifts_in_particle_basis = true; -end - -global bh_global_zero_lag_score; -if isempty(bh_global_zero_lag_score) - bh_global_zero_lag_score = false -end - -if (nargin ~= 2 && nargin ~= 3) - error('args = PARAMETER_FILE, CYCLE, [1,abs(ccc),2,weighted,3,abs(weighted)]') -else - parentFunc = mfilename; - resumeVars = struct(); -end - -if nargin == 3 - - flgWeightCCC = EMC_str2double(varargin{1}); -else - % default to linear ccc (which is actually weighted by the SNR though) - flgWeightCCC = 0; -end - -% Explicit reference to location of variables in main memory, or on the GPU. As -% in pcaPub, looking ahead to re-write in c++ for cuda, no cells allowed. -cpuVar = struct(); -GPUVar = struct(); - -startTime = clock; -CYCLE = EMC_str2double(CYCLE); -cycle_numerator = ''; -cycle_denominator =''; - flgStartThird = 0; - flgReverseOrder = 0; -if numel(CYCLE) == 3 - cycle_numerator = CYCLE(2); - cycle_denominator = CYCLE(3); - CYCLE = CYCLE(1); - flgStartThird = true; -elseif CYCLE < 0 - % Simple option to process in reverse order so that the load can be run on two - % physically distinct systems at once. - flgReverseOrder = 1; - flgStartThird = 0; - CYCLE = abs(CYCLE); - - -end - - - -pBH = BH_parseParameterFile(PARAMETER_FILE); -cycleNumber = sprintf('cycle%0.3u', CYCLE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); -mapBackIter = subTomoMeta.currentTomoCPR; -reconScaling = 1; -try - nPeaks = pBH.('nPeaks'); -catch - nPeaks = 1; -end - -try - flgCutOutVolumes=pBH.('flgCutOutVolumes') -catch - flgCutOutVolumes=0 -end - -% TODO decide on a "reasonable" padding based on expected shifts. -try - CUTPADDING = subTomoMeta.('CUTPADDING') -catch - CUTPADDING=20 -end - -maxGoldStandard = subTomoMeta.('maxGoldStandard'); - - -nGPUs = pBH.('nGPUs') - - -flgClassify= pBH.('flgClassify'); -try - flgMultiRefAlignment=pBH.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end -try - flgCenterRefCOM = pBH.('flgCenterRefCOM'); -catch - flgCenterRefCOM = 1; -end - -try - flgSymmetrizeSubTomos = pBH.('flgSymmetrizeSubTomos'); -catch - flgSymmetrizeSubTomos = 0; -end -flgRaw_shapeMask = 0;%= pBH.('experimentalOpts')(3) -samplingRate = pBH.('Ali_samplingRate'); - -pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; -if pBH.('SuperResolution') - pixelSize = pixelSize * 2; -end - -flgPrecision = 'single'; %pBH.('flgPrecision'); -angleSearch = pBH.('Raw_angleSearch'); -peakSearch = (pBH.('particleRadius')./pixelSize); -peakCOM = [1,1,1].*3; -className = pBH.('Raw_className'); - -try - loadTomo = pBH.('loadTomo') -catch - loadTomo = 0; -end -try - eraseMaskType = pBH.('Peak_mType'); - eraseMaskRadius = pBH.('Peak_mRadius')./pixelSize; - fprintf('Further restricting peak search to radius %f %f %f\n',... - eraseMaskRadius); - eraseMask = 1; -catch - eraseMask = 0; - fprintf('Using particle radius for peak search\n'); -end - -rotConvention = 'Bah'; -% Check and override the rotational convention to get helical averaging. -% Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = pBH.('doHelical'); -catch - doHelical = 0; -end -if ( doHelical ) - rotConvention = 'Helical' -end - -rotConvention - -bFactor = pBH.('Fsc_bfactor'); -if length(bFactor) > 1 - fprintf('multiple bFactors specified, using the first for alignment.\n'); - bFactor = bFactor(1); -end - -try - scaleCalcSize = pBH.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end -% % % % if (flgClassify || flgMultiRefAlignment) -if (flgClassify) - refName = pBH.('Ref_className'); -else - refName = pBH.('Raw_className'); -end - -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); - - - -classVector{1} = pBH.('Raw_classes_odd')(1,:); -classSymmetry{1}= pBH.('Raw_classes_odd')(2,:); - - -classVector{2} = pBH.('Raw_classes_eve')(1,:); -classSymmetry{2}= pBH.('Raw_classes_eve')(2,:); - - -% % % % if (flgClassify || flgMultiRefAlignment) -if (flgClassify) - geometry = subTomoMeta.(cycleNumber).ClassAlignment; - refVectorFull{1}= [pBH.('Ref_references_odd');1] - refVectorFull{2}= [pBH.('Ref_references_eve');1] -elseif (flgMultiRefAlignment) - geometry = subTomoMeta.(cycleNumber).ClusterRefGeom; - refVectorFull{1}= [pBH.('Raw_classes_odd');classVector{1} ] - refVectorFull{2}= [pBH.('Raw_classes_eve');classVector{2} ] -else - geometry = subTomoMeta.(cycleNumber).Avg_geometry; - refVectorFull{1} = [pBH.('Raw_classes_odd');1]; - refVectorFull{2} = [pBH.('Raw_classes_eve');1]; -end - - -% % % pathList= subTomoMeta.mapPath; -% % % extList = subTomoMeta.mapExt; -masterTM = subTomoMeta; clear subTomoMeta - - - - -refVector = cell(2,1); -refGroup = cell(2,1); -refSym = cell(2,1); - -for iGold = 1:2 - % Sort low to high, because order is rearranged as such unstack - refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; - % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) - % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) - % axial symmetry to apply, negative value indicates creating a mirrored ref - % accros the corresponding axis - refSym{iGold} = refVectorFull{iGold}(2,:) -end - -% make sure the number of references match the unique groups in the classVector -% and also that the class/group pairs match the class/ref pairs. -nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})] - - -nRefOut(1:2) = [length(unique(refGroup{1})) + sum(( refSym{1} < 0 )),... - length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; - - -%%%%%%%%%%%%%%%%%%%%%%% - -% Get the number of tomograms to process. -tomoList = fieldnames(geometry); -nTomograms = length(tomoList); -tiltList = masterTM.tiltGeometry; -ctfGroupList = masterTM.('ctfGroupSize'); - - - -% mask defines area for angular search, peakRADIUS restricts translational - - -[ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Ali', pixelSize) - -[ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) - - -try - flgLimitToOneProcess = pBH.('flgLimitToOneProcess'); -catch - flgLimitToOneProcess = 0; -end - -if ( loadTomo ) - limitToOne = loadTomo; - if (flgLimitToOneProcess) - limitToOne = min(limitToOne, flgLimitToOneProcess); - end -elseif (flgLimitToOneProcess) - limitToOne = flgLimitToOneProcess; -else - limitToOne = pBH.('nCpuCores'); -end - -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); -if ( flgReverseOrder ) - % Flip the order for reverse processing on a second machine. This will also disable saving of - % of the metadata so there aren't conflicts. - for iParProc = 1:nParProcesses - iterList{iParProc} = flip(iterList{iParProc}); - end - -elseif ( flgStartThird ) - - % Shift to start at one third through to process on a third machine. This will also disable saving of - % of the metadata so there aren't conflicts. - for iParProc = 1:nParProcesses - % Note the use of floor is more like ceiling here (rounds away from - % zero) - nParts = ceil(length(iterList{iParProc}) ./ cycle_denominator); - fIDX = 1+(cycle_numerator - 1)*nParts; - lIDX = min(cycle_numerator*nParts,length(iterList{iParProc})); - iterList{iParProc} = iterList{iParProc}(fIDX:lIDX); - end - -end - -if any(peakSearch > maskRadius) - fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') - peakSearch( (peakSearch > maskRadius) ) = ... - maskRadius( (peakSearch > maskRadius) ); -end - -if ( any(classSymmetry{1}~=1) || any(classSymmetry{2}~=1) ) && flgSymmetrizeSubTomos - flgSymmetry = true -else - flgSymmetry = false -end - -% Read in the references. -% Read in the references. -refIMG = cell(2,1); -refWGT = cell(2,1); -refWgtROT = cell(2,1); -imgCounts = cell(2,1); -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) - - - weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', refName, halfSet); - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - - [ refTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - - [ wdgTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... - sizeCalc); - - sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}(2:2:6)'; - - if (flgCenterRefCOM) -% % % % % % % [ comMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); - [ comMask ] = EMC_maskShape(maskType, sizeMask, maskRadius, 'gpu', {'shift', maskCenter}); - end - - % get boxSize - n = 1 ; tIMG = cell(numel(refVector{iGold})); tWDG = cell(numel(refVector{iGold}));tWDG_r = tWDG; - for iP = 1:numel(refTMP) - if ~isempty(refTMP{iP}) - tIMG{n} = refTMP{iP}; refTMP{iP} = []; - if (flgCenterRefCOM) - % Not sure if this is always the best approach, but it may be - % useful in some cases. -% % % % % % % [~,iCOM] = BH_mask3d(gpuArray(tIMG{n}).*comMask,pixelSize,'','',1); - - [~, ~, ~,iCOM] = EMC_maskReference(gpuArray(tIMG{n}).*comMask, pixelSize, {'fsc',true; 'com', true}); - fprintf('centering ref %d on COM %3.3f %3.3f %3.3f \n',n,iCOM); - - tIMG{n} = BH_resample3d(tIMG{n},[0,0,0],gather(iCOM), ... - {'Bah',1,'spline'},'cpu','inv'); - - end - tWDG{n} = wdgTMP{iP}; wdgTMP{iP} = []; - tWDG{n} = tWDG{n} - min(tWDG{n}(:)) + 1e-6; - tWDG{n} = tWDG{n} ./ max(tWDG{n}(:)); - n = n + 1; - end - end - - wdgPAD = BH_multi_padVal(size(tWDG{1}), sizeCalc) - for iWdg = 1:n-1 - tWDG_r{iWdg} = BH_padZeros3d(tWDG{iWdg},wdgPAD(1,:),wdgPAD(2,:),... - 'cpu',flgPrecision); - tWDG{iWdg} = ifftshift(tWDG_r{iWdg}); - end - - refIMG{iGold} = tIMG ; clear tIMG refTMP - refWGT{iGold} = tWDG; clear tWDG wdgTMP - refWgtROT{iGold} = tWDG_r; clear tWDG_r - - clear comMask - -end - -[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, maxGoldStandard ); - - - -% optimize the fft for the given size. Padding to the next power of 2 is usually -% slower given the dimensionalityl of the volume data. -fftPlanner = rand(sizeCalc); -fftw('planner', 'exhaustive'); -fftn(fftPlanner); -clear fftPlanner - -% Make a mask, and apply to the average motif && save a masked, -% binned copy of the average for inspection. -% % % -% % % [ volMask ] = gather(BH_mask3d(maskType, sizeMask, maskRadius, maskCenter)); -% % % volBinary = (volMask >= 0.01); - - -% In principle the window and mask could be different sizes, however, I -% think I am currently forcing them to be the same. - - - % make rotationally invariant -% % % % % % % [ peakMask] = gather(BH_mask3d('sphere', sizeWindow, [1,1,1].*max(peakSearch), maskCenter)); - [ peakMask ] = gather(EMC_maskShape('sphere', sizeWindow, [1,1,1].*floor(max(peakSearch)), 'gpu', {'shift', maskCenter})); - - if (eraseMask) - % Mask could be smaller than the normal taper would allow, so instead - % of thresholding a normal mask, take this alt route. - eraseMask = ones(ceil(2.*eraseMaskRadius),'single'); - padEraseMask = BH_multi_padVal(size(eraseMask),sizeCalc); - eraseMask = BH_padZeros3d(eraseMask,padEraseMask(1,:),padEraseMask(2,:),'cpu','single'); - eraseMask = single(find(eraseMask < 1)); - else - eraseMask = []; - end - - - if ( flgRaw_shapeMask ) -% % % % % % % [ volMask ] = BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter); - [ volMask ] = EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter}); - - % Currently not set up for mult-ref alignment - iRef = 1; - % Use the geometric mean so that excluded areas mask out -% % % % % % % [ volMask ] = gather(sqrt(volMask .* ... -% % % % % % % BH_mask3d(refIMG{1}{iRef}+refIMG{2}{iRef},pixelSize,'',''))); - - [ volMask ] = gather(sqrt(volMask .* ... - EMC_maskReference(refIMG{1}{iRef}+refIMG{2}{iRef}, pixelSize, {'fsc', true}))); - - else -% % % % % % % [ volMask ] = gather(BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter)); - [ volMask ] = gather(EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter})); - - end - -% % % [ peakMask] = gather(BH_mask3d(maskType, sizeMask, peakSearch, maskCenter)); -% % % peakBinary = (peakMask >= 0.01); - -% [ refInterp] = gather(BH_mask3d(maskType, sizeREF, peakSearch, maskCenter)); -% refInterp = (refInterp >= 0.01); - - bandpassFilt = cell(nReferences(1),1); - bandpassFiltREF = bandpassFilt; - wCCC = cell(nReferences(1),1); - for iWccc = 1:length(nReferences(1)); - wCCC{iWccc} = 0; - end - if (flgClassify || flgMultiRefAlignment) - for iRef = 1:nReferences(1) - if (flgClassify) - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); - else - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Raw%d',iRef)); % % % % - end - - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - % returns a cpu array - if (flgWeightCCC) - [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1); - else - [ bandpassFilt{iRef}, ~] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1); - end - - - bandpassFiltREF{iRef} = 1; - - end - else - - - - for iRef = 1 - fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - % returns a cpu array - if (flgWeightCCC) - [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1 ); - else - [ bandpassFilt{iRef},~ ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1 ); - end - - bandpassFiltREF{iRef} = 1; - - - end - - end - -% if (flgWeightCCC) -% for i = 1:length(wCCC{1}) -% i -% length(wCCC{1}{i}) -% end -% end - - % This is just used to limit the interpolation search so use the most - % permissive bandpass, while the appropriate bandpass (given a multi-ref - % alignment) will still be applied. - mostPermissive = zeros(1,nReferences(1)); - for iRef = 1:nReferences(1) - mostPermissive(iRef) = sum(bandpassFilt{iRef}(:)); - end - [~,mPidx] = max(mostPermissive); - - wdgBinary = single(find(fftshift(bandpassFilt{mPidx} > 10^-2))); - - -ref_FT1 = cell(2,1); -ref_FT2 = cell(2,1); - - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - nOut = 1; - refOUT = cell(2.*nReferences(iGold),2); - - for iRef = 1:nReferences(iGold) - - refTMP_2 = refIMG{iGold}{iRef}; refIMG{iGold}{iRef} = []; - refTMP = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); - - - % if not using a weighted average (adapted SPW filter), apply an - % approximation the cRef from Rosenthal/Henderson. This is currently always set to one - % and is just doing the masking and normalization. It should be okay to just apply the mask - % and rely on the normalization during the CCC calc. TODO - ref_FT1{iGold}{iRef} = gather(conj(BH_bandLimitCenterNormalize(... - refTMP.*volMask, bandpassFiltREF{iRef}, (volMask>0.01), padCalc, flgPrecision))); - - - - - ref_FT2{iGold}{iRef} = gather(refTMP_2); - % Trim for output reference - refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); - - % Overwrite a copy of the filtered, bandpassed ref for output - refOUT{nOut} = real(ifftn(conj(ref_FT1{iGold}{iRef}))); - refOUT{nOut} = gather(refOUT{nOut}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); - - - - refOUT{nOut} = refOUT{nOut}.*volMask; - - refOUT{nOut+1} = real(ifftn(BH_bandLimitCenterNormalize(... - refTMP_2.*peakMask, '', (peakMask > 0.01), padCalc, 'single'))); - refOUT{nOut+1} = gather(refOUT{nOut+1}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) .* peakMask ); - nOut = nOut + 2; - - refOUT{nOut} = refOUT{nOut} - mean(refOUT{nOut}(:)); - refOUT{nOut} = refOUT{nOut} ./ rms(refOUT{nOut}(:)); - - refOUT{nOut+1} = refOUT{nOut+1} - mean(refOUT{nOut+1}(:)); - refOUT{nOut+1} = refOUT{nOut+1} ./ rms(refOUT{nOut+1}(:)); - end - - - % Save a montage of the masked reference & shape masks if requested. - - maskedOUTFILE = sprintf('%s_maskedRef-mont_%s.mrc',outputPrefix,halfSet); - [ maskedReferences, ~ ] = BH_montage4d(refOUT, ''); - SAVE_IMG(MRCImage(single(maskedReferences)), maskedOUTFILE); - - -end - -clear refIMG refWDG refOUT iRef - -%%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't -%%%%%%%%%%%%%%%%%%%%% search at all in that dimension. - -[ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch) - -[masterTM] = BH_recordAngularSampling( masterTM, cycleNumber, angleStep, inPlaneSearch); - -% set truth value for refinement during out of plane search - -if any(angleStep(:,1)) - flgRefine = true; - fprintf('flgRefine set to %s','True'); -else - flgRefine = false; - fprintf('flgRefine set to %s','False'); -end - -angleStep(:,1) -any(angleStep(:,1)) -nCount = 1; - -firstLoop = true; -nIgnored = 0; - -bestAnglesResults = cell(nParProcesses,1); -geometryResults = cell(nParProcesses,1); - - - -try - EMC_parpool(nParProcesses+1) -catch - delete(gcp('nocreate')) - EMC_parpool(nParProcesses+1) -end - -size(ref_FT2) - -system('mkdir -p alignResume'); - -system(sprintf('mkdir -p alignResume/%s',outputPrefix)); -softenWeight = 1/sqrt(samplingRate); -for iParProc = 1:nParProcesses - - % Caclulating weights takes up a lot of memory, so do all that are necessary - % prior to the main loop -- CHANGE THE CHECK TO JUST READ THE HEADER NOT LOAD - % THE WEIGHT INTO GPU MEMORY - iParProc - iterList{iParProc} - for iTomo = iterList{iParProc} - - BH_multi_loadOrCalcWeight(masterTM,ctfGroupList,tomoList{iTomo},samplingRate ,... - sizeCalc,geometry,flgPrecision,1); - - - end -end - -% Clear all of the GPUs prior to entering the main processing loop -for iGPU = 1:nGPUs - g = gpuDevice(iGPU); - fprintf('\n\nClear gpu %d mem prior to main loop, %3.3e available\n\n',iGPU,g.AvailableMemory); - clear g -end - -parVect = 1:nParProcesses; -parfor iParProc = parVect -%for iParProc = 1:nParProcesses -%profile on - bestAngles_tmp = struct(); - geometry_tmp = geometry; - -% % % % Get the gpuIDX assigned to this process -% % % iGPUidx = gpuDevice(); -% % % iGPUidx = iGPUidx.Index; - gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; - iGPUidx = gpuIDXList(iParProc); - gpuDevice(iGPUidx); - fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); - for iTomo = iterList{iParProc} - - - - nCtfGroups = ctfGroupList.(tomoList{iTomo})(1); - % Check for interupted alignment. - previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); - if exist(previousAlignment,'file') - % Sometimes when multiple nodes are used, an extra line is added. - % TODO fix this workaround - system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s_clean; mv %s_clean %s',... - previousAlignment,previousAlignment,previousAlignment,previousAlignment)); - bestAngles_tmp.(tomoList{iTomo}) = load(previousAlignment); - fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); - else - % There is some memory leak somewhere that I haven't been able to figure - % out. I am clearing all vars but output in the children functions ... this - % isn't ideal, but for now is an acceptable stop gap. - %D = gpuDevice(gpuList(iGPU)); - - % shake up the random number generator for phi and theta - rng('shuffle'); - - bandpassFilt_tmp = cell(nReferences(1),1); - bandpassFiltREF_tmp = cell(nReferences(1),1); - for iRef = 1:nReferences(1) - if flgMultiRefAlignment <= 2 - bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); - else - bandpassFilt_tmp{iRef} = (bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = (bandpassFiltREF{iRef}); - end - end - - - - ref_FT1_tmp = cell(2,1); - ref_FT2_tmp = cell(2,1); - ref_WGT_tmp = cell(2,1); - ref_WGT_rot = cell(2,1); - - - volMask_tmp = gpuArray(volMask); - volBinary_tmp = single(find( volMask_tmp > 0.01 )); - peakMask_tmp = gpuArray(peakMask); - peakBinary_tmp = single(find( peakMask_tmp > 0.01 )); - wdgBinary_tmp = gpuArray(wdgBinary); - eraseMask_tmp = gpuArray(eraseMask); - - wCCC_tmp = cell(length(wCCC)); - - - - for iRef = 1:nReferences(1) - for iWccc = 1:length(wCCC{iRef}) - if (flgWeightCCC) - wCCC_tmp{iRef}{iWccc} = gpuArray(wCCC{iRef}{iWccc}); - else - % The check in xcf_rotational looks for a cell - wCCC_tmp{iRef} = 0; - end - end - end - - - - for iGold = 1:2 - for iRef = 1:nReferences(iGold) - if flgMultiRefAlignment <= 2 - ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = gpuArray(refWgtROT{iGold}{iRef}); - else - % Temp workaround, six big ribo refs crashing - ref_FT1_tmp{iGold}{iRef} = (ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = (ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = (refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = (refWgtROT{iGold}{iRef}); - end - end - end - - - sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) - tic; - - % Load the tomo into gpu - tomoName = tomoList{iTomo}; - %fprintf('gpu %d working on tomoName %s\n', iGPU, tomoName); - - tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); - % Load in the geometry for the tomogram, and get number of subTomos. - positionList = geometry_tmp.(tomoList{iTomo}); - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); - -% [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); - binShift = [0,0,0]; - nSubTomos = size(positionList,1); - - - - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; - wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); -% wgtName = sprintf('cache/%s_bin%d.wgt', tomoList{iTomo},... -% samplingRate); - maxWedgeMask = BH_unStackMontage4d(1:nCtfGroups,wgtName,... - ceil(sqrt(nCtfGroups)).*[1,1],''); - maxWedgeIfft = maxWedgeMask; - - for iWdg = 1:length(maxWedgeMask) - if ~isempty(maxWedgeMask{iWdg}) - maxWedgeIfft{iWdg} = ifftshift(maxWedgeIfft{iWdg}.^softenWeight); - maxWedgeMask{iWdg} = maxWedgeMask{iWdg}.^softenWeight; - end - end - - - fprintf('loaded %s.\n',wgtName); - - - % Can't clear inside the parfor, but make sure we don't have two tomograms - % in memory at once. - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - - if (flgCutOutVolumes) - volumeData = []; - else - [ volumeData, ~ ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,loadTomo); - if ( loadTomo ) - volHeader = struct(); - volHeader.('nX') = size(volumeData,1); - volHeader.('nY') = size(volumeData,2); - volHeader.('nZ') = size(volumeData,3); - else - volHeader = getHeader(volumeData); - end - end - - - % For now, set up for full grid-search only, as I intend to just do - % translational and in-plane searches for now anyhow. - - [~,iv1,iv2,iv3] = BH_resample3d(volMask_tmp,eye(3),[0,0,0],... - {'Bah',1,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv'); - inputVectors = {iv1,iv2,iv3}; - iv1 = []; iv2 = []; iv3 = []; - cccStorageBest = cell(nPeaks,1); - cccStorageRefine = cell(nPeaks,1); - for iPeak = 1:nPeaks - cccStorageBest{iPeak} = zeros(nSubTomos,10); - cccStorageRefine{iPeak}= zeros(nSubTomos,10); - end - % reset for each tomogram - wdgIDX = 0; - - for iSubTomo = 1:nSubTomos - breakPeak = 0; % for try catch on cut out vols - if (wdgIDX ~= positionList(iSubTomo,9)) - % Geometry is sorted on this value so that tranfers are minimized, - % as these can take up a lot of mem. For 9 ctf Groups on an 80s - % ribo at 2 Ang/pix at full sampling ~ 2Gb eache. - wdgIDX = positionList(iSubTomo,9); - fprintf('pulling the wedge %d onto the GPU\n',wdgIDX); - % Avoid temporar - iMaxWedgeMask = []; iMaxWedgeIfft = []; - iMaxWedgeMask = gpuArray(maxWedgeMask{wdgIDX}); - iMaxWedgeIfft = gpuArray(maxWedgeIfft{wdgIDX}); - end - - - [~,iw1,iw2,iw3] = BH_resample3d(iMaxWedgeMask, eye(3), [0,0,0], ... - {'Bah',1,'linear',1,wdgBinary_tmp}, ... - 'GPU', 'inv'); - inputWgtVectors = {iw1,iw2,iw3}; - iw1 = []; iw2 = []; iw3 = []; - - for iPeak = 1:nPeaks - if (breakPeak) - continue; - end - getInitialCCC = 1; - cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); - cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); - powerOut = zeros(nAngles(1).*nReferences(1),1,'gpuArray'); - - - - % Used in refinment loop - angCount = 1; - - % Check that the given subTomo is not to be ignored - classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); - particleIDX = positionList(iSubTomo, 4); - iGold = positionList(iSubTomo, 7); - - - if classVector{iGold}(1,:) == 0 - classPosition = 1; - flgAllClasses = true; - else - classPosition = find(classVector{iGold}(1,:) == classIDX); - flgAllClasses = false; - end - - - - if (classIDX ~= -9999) && ... % All previously ignored particles - ( flgAllClasses || ismember(classIDX, classVector{iGold}(1,:)) ) - - - center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; - angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); - - % Find range to extract, and check for domain error. - if (flgCutOutVolumes) - % Need some check that the windowsize has not changed! TODO TODO - - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(2*CUTPADDING+sizeWindow, ... - sizeWindow,maskRadius, center); - else - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... - sizeWindow,maskRadius, center); - end - - - - - if ischar(indVAL) - fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) - nIgnored = nIgnored + 1; - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - else - - - if (flgCutOutVolumes) - % Test with some generic padding , only to be used on bin 1 at - % first!!! TODO add a flag to check this. - try - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iparticle = gpuArray(getVolume(MRCImage(particleOUT_name),[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - catch - fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - breakPeak = 1; - continue; - end - else - - if ( loadTomo ) - iparticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3))); - - else - iparticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - end - - end - [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... - padVAL(2,1:3), 'GPU', 'singleTaper'); - - - - for iAngle = 1:size(angleStep,1) - - theta = angleStep(iAngle,1); - thetaInc = angleStep(iAngle,4); - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. - - phiInc = angleStep(iAngle,3); - - % To prevent only searching the same increments each time in a limited - % grid search, radomly offset the azimuthal angle by a random number - % between 0 and 1/2 the azimuthal increment. - - azimuthalRandomizer = (rand(1)-0.5)*phiInc; - - for iAzimuth = 0:angleStep(iAngle,2) - phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); - - for iInPlane = inPlaneSearch - psi = iInPlane; - psiInc = angleStep(iAngle,5); - %[phi,theta,psi-phi]; - - - RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - cccStorageTrans= zeros(1.*nReferences(1),10,'gpuArray'); - - for alignLoop = 1:2 - - - switch alignLoop - - case 1 - % This takes care of non-inter shift in the origin that is - % ignored during the windowing of the particle. - estPeakCoord = shiftVAL; - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - - case 2 - - - bestOfRefs = sortrows(gather(cccStorageTrans), -6); - %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); - - estPeakCoord = bestOfRefs(1,8:10); - - - - - % Assuming if class specific symmetry, then some not just 1 - if (flgSymmetry) - symmetry = classSymmetry{iGold}(1, classPosition); - %fprintf('Symmetry confirmation %d\n',symmetry); - [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... - estPeakCoord,... - {'Bah',symmetry,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv',inputVectors); - - - - if (getInitialCCC) - [ iTrimInitial ] = BH_resample3d(iparticle, ... - reshape(angles,3,3),... - shiftVAL,... - {'Bah',symmetry,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv',inputVectors); - - - -% % % powerInitial = sum(abs(iTrimInitial(volBinary_tmp))).^2; -% % % - - iWedgeInitial = BH_resample3d(iMaxWedgeMask, reshape(angles,3,3), [0,0,0], ... - {'Bah',symmetry,'linear',1,wdgBinary_tmp}, ... - 'GPU', 'inv',inputWgtVectors); - - - end - - iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - {'Bah',symmetry,'linear',1,wdgBinary_tmp}, ... - 'GPU', 'inv',inputWgtVectors); - - - - - - else - symmetry = 1; - % Transform the particle, and then trim to motif size - - [ iTrimParticle ] = BH_resample3d(iparticle, RotMat, ... - estPeakCoord, {'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'inv',inputVectors); - iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'inv',inputWgtVectors); - - - - if (getInitialCCC) - - iTrimInitial = BH_resample3d(iparticle, reshape(angles,3,3),... - shiftVAL,{'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'inv',inputVectors); - - - - iWedgeInitial = BH_resample3d(iMaxWedgeMask, reshape(angles,3,3), [0,0,0], ... - {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'inv',inputWgtVectors); - - end - end % Symmetry or not + interpolation - - -% % % powerOut(angCount) = sum(abs(iTrimParticle(volBinary_tmp))).^2; - - end - - - switch flgMultiRefAlignment - case 0 - refToAlign = 1; - case 1 - refToAlign = 1:max(nReferences(:)); - case 2 - refToAlign = classIDX; - otherwise - error('flgMultiRefAlignment is not 0,1,2') - end - - for iRef = refToAlign % 1:max(nReferences(:)) - - switch alignLoop - - case 1 - - - % use transpose of RotMat - - iRotRef = BH_resample3d(ref_FT2_tmp{iGold}{iRef}, RotMat', ... - estPeakCoord, {'Bah',1,'linear',1,peakBinary_tmp}, 'GPU', 'forward',inputVectors); - - iRotWdg = BH_resample3d(ref_WGT_rot{iGold}{iRef}, RotMat', ... - [0,0,0], {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - -% iRotRef = ... -% iRotRef(padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - % maybe I should be rotating peak mask here in case it has - % an odd shape, since we are leaving the proper frame - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef.*peakMask_tmp,... - bandpassFiltREF_tmp{iRef} ,peakBinary_tmp,... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*peakMask_tmp,... - bandpassFilt_tmp{iRef} ,peakBinary_tmp,padCalc,flgPrecision); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - peakMask_tmp, peakCOM,eraseMask_tmp); - - - cccStorageTrans(iRef,:) = [iRef, particleIDX, ... - phi, theta, psi - phi, ... - 0, 0, ... - peakCoord + estPeakCoord]; - case 2 - - % get starting point - if (getInitialCCC) - - initialRotPart_FT = BH_bandLimitCenterNormalize(... - iTrimInitial.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( initialRotPart_FT, ... - ref_FT1_tmp{iGold}{iRef}, ... - ifftshift(iWedgeInitial),... - ref_WGT_tmp{iGold}{iRef}, ... - wCCC_tmp{iRef}); - - - - - cccInitial(iRef,:) = [iRef, particleIDX, ... - 0,0,0, ... - iCCC, 1, ... - shiftVAL]; - - - initialRotPart_FT = []; - - - end - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - - - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{iGold}{iRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{iGold}{iRef}, ... - wCCC_tmp{iRef}); - - - - - - - % Note that no new translational estimate is made, so no - % need to multiply by RotMat - cccStorage2(angCount,:) = ... - [iRef, particleIDX, ... - phi, theta, psi , ... - iCCC, 1, ... - estPeakCoord]; - - - angCount = angCount + 1; - end - - - end % loop over references. - - - - end - % This volume won't be needed until the next subTomo is considered, - % which is also where getInitialCCC Boolean is set to True again. - iTrimInitial = []; - getInitialCCC = 0; - - end % in plane angles - end % azimuth - end % polar - -% % % fprintf('Power ratio is %3.3f\n',powerOut./powerInitial); - - cccPreRefineSort = sortrows(gather(cccStorage2),-6); - - if (length(refToAlign) > 1) - cccInitial = sortrows(gather(cccInitial), -6); - cccInitial = cccInitial(1,:); - else - cccInitial = gather(cccInitial(refToAlign,:)); - - end - - if cccInitial(1,6 ) > cccPreRefineSort(1,6) - cccPreRefineSort(1,:) = cccInitial(1,:); - end - - - - % This only seems to be a problem with cut out volumes. - % Normalization maybe? - if ~any(cccPreRefineSort(1,:)) - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); - continue - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - if (flgRefine) - - - - % Get the results from just this subTomo and sort on CCC - - rRef = cccPreRefineSort(1,1); - rPart = cccPreRefineSort(1,2); - rPhi = cccPreRefineSort(1,3); - rPhiInc = phiInc / 4; - rTheta= cccPreRefineSort(1,4); - rTheInc = thetaInc /2; - rPsi = cccPreRefineSort(1,5); - rPsiInc = psiInc /2; - % Confirm shiftVAL is doing what it should be - rXYZest = cccPreRefineSort(1,8:10); - - if (rTheInc) - % For a larger out of plane step, search a larger range in plane - psiRefineStep = floor(sqrt(rTheInc)); - else - psiRefineStep = 1; - end - - thetaRefineStep =1; - phiRefineStep=2; - totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; - totalRefineStep = prod((2.*totalRefineStep)+1); - - cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); - - if (rPsiInc == 0) - inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; - else - inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; - end - polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; - azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; - - searchList = zeros(totalRefineStep,3); - nSearch = 1; - for iPhi = azimuthalRefine - for iTheta = polarRefine - for iPsi = inPlaneRefine - % best iPsi is origin Psi - Phi, no need to subtract here. - - searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; - - nSearch = nSearch + 1; - end - end - end % end of building angle list - - for iRefine = 1:nSearch-1 - for alignLoop = 1:2 - if alignLoop == 1 - rXYZ = rXYZest; - elseif alignLoop == 2 - rXYZ = cccStorage3(iRefine,8:10); - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - - RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - - - - switch alignLoop - % This keeps seperate shifts due to windowing and binning from - % shifts found in CCC - case 1 - - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - case 2 - - % Assuming if class specific symmetry, then some not just 1 - if (flgSymmetry) - symmetry = classSymmetry{iGold}(1, classPosition); - - [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... - rXYZ,... - {'Bah',symmetry,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv',inputVectors); - - -% iTrimParticle = iTrimParticle(... -% padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - {'Bah',symmetry,'linear',1,wdgBinary_tmp},... - 'GPU', 'inv',inputWgtVectors); - - - else - symmetry = 1; - % Transform the particle, and then trim to motif size - - [ iTrimParticle ] = BH_resample3d(iparticle, RotMat, ... - rXYZ, {'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'inv',inputVectors); - - - iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'inv'); - - -% iTrimParticle = ... -% iTrimParticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - - end % Symmetry or not + interpolation - - - - - - end - - - - if alignLoop == 1 - - % use transpose of RotMat - try - iRotRef = BH_resample3d(ref_FT2_tmp{iGold}{rRef}, RotMat', ... - rXYZ, {'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); - catch - cccPreRefineSort(1,1) - end - iRotWdg = BH_resample3d(ref_WGT_rot{iGold}{rRef}, RotMat', ... - [0,0,0], {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - - - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef.*peakMask_tmp,... - bandpassFiltREF_tmp{rRef},peakBinary_tmp,... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*peakMask_tmp,... - bandpassFilt_tmp{rRef} ,peakBinary_tmp,padCalc,flgPrecision); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - peakMask_tmp, peakCOM,eraseMask_tmp); - - - % 2016-11-11 also took out (+ rXYZ) - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - 1, 1, ... - peakCoord+rXYZ]; - else - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{rRef},volBinary_tmp,... - padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{iGold}{rRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{iGold}{rRef}, ... - wCCC_tmp{iRef}); - - - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - iCCC, 1, ... - rXYZ] ; - end - end - - end - - sortRef = sortrows(gather(cccStorage3),-6); - cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); - - end % end of refinement loop - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % Get the final translational shift for the best scoring angular - % match. - try - if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) - bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); -% % % % Get the negative slope of the top ten CCC scores. -% % % topTen = fit([.1:.1:1]',sortRef(1:10,6),'linear'); -% % % bestRotPeak(1,7) = topTen(100)-topTen(101); - - else - bestRotPeak = cccPreRefineSort(1,:); - bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); -% % % rowNum = min(size(cccPreRefineSort,1),10*nPeaks); -% % % topX = 1- 0.1.*(10-rowNum); -% % % % Get the negative slope of the top ten CCC scores. -% % % topTen = fit([.1:.1:topX]',cccPreRefineSort(1:rowNum,6),'linear'); -% % % bestRotPeak(1,7) = topTen(100)-topTen(101); - - end - catch - fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); - cccStorageRefine{iPeak}(iSubTomo,:) - cccPreRefineSort(1,:) -% % % rowNum = min(size(cccPreRefineSort,1),10*nPeaks) -% % % topX = 1- 0.1.*(10-rowNum) -% % % fprintf('\nNow check the fits, first and second clause\n'); -% % % topTen = fit([.1:.1:1]',sortRef(1:10,6),'linear') -% % % fprintf('\nSecond\n'); -% % % topTen = fit([.1:.1:topX]',cccPreRefineSort(1:rowNum,6),'linear') -% % % error('Error in sorting the best peak in alignRaw'); - end - - finalRef = bestRotPeak(1,1); - finalPart = bestRotPeak(1,2); - finalPhi = bestRotPeak(1,3); - finalTheta= bestRotPeak(1,4); - finalPsi = bestRotPeak(1,5); - % Confirm shiftVAL is doing what it should be - finalrXYZest = bestRotPeak(1,8:10); - - RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - - - - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - - % use transpose of RotMat - %%% 2016-11-11 estPeakCoord should have been finalrXYZest in - %%% the last writing, but now switching to zeros - try - iRotRef = BH_resample3d(ref_FT2_tmp{iGold}{finalRef}, RotMat', ... - finalrXYZest, {'Bah',1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); - iRotWdg = BH_resample3d(ref_WGT_rot{iGold}{finalRef}, RotMat', ... - [0,0,0], {'Bah',1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - catch - fprintf('\n\nFinal ref,part,phi,theta,psi %f %f %f %f %f\n\n',... - bestRotPeak(:,1:5)); - bestRotPeak(1,1:5) - fprintf('BreakPeak %d\n',breakPeak); - error('errrorsoedfsdf') - end - -% iRotRef = ... -% iRotRef(padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef.*peakMask_tmp,... - bandpassFiltREF_tmp{finalRef} ,peakBinary_tmp,... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*peakMask_tmp,... - bandpassFilt_tmp{finalRef} ,peakBinary_tmp,padCalc,flgPrecision ); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - peakMask_tmp, peakCOM,eraseMask_tmp); - -% % % end - - - - - % Subtract shiftVAL since this is due to windowing, not the actual - % position. - cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... - peakCoord + finalrXYZest - shiftVAL]) ; - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - % It is probably more useful see the shifts in the particle - % reference frame vs. the avg which was the original - if (bh_global_print_shifts_in_particle_basis) - printShifts = zeros(3,3); - printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); - printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); - printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); - else - printShifts = [cccInitial(1,end-2:end); ... - cccPreRefineSort(1,end-2:end);... - cccStorageBest{iPeak}(iSubTomo,end-2:end)]; - end - - % Print out in Angstrom - printShifts = printShifts .* pixelSize; - - - deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); - if (deltaCCC < 0) && (abs(deltaCCC) > 0.15*cccInitial(1,6)) - fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); - fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - - end - - if (flgRefine) - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX,cccInitial(1,1:end-3),printShifts(1,:), ... - 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... - cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... - 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - - else - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX,cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - - end - - - end % if condition on newly ignored particles - - end - - if ~(rem(iSubTomo,100)) - timeClass = toc; - fprintf('\nworking on %d/%d subTomo from %s...%fs\n',... - iSubTomo,nSubTomos,tomoName,timeClass); - tic; - end - - - iParticle = []; - iSymParti = []; - iTrimParticle = []; - iAsym = []; - iTrimAsym = []; - iWedgeMask = []; - rotPart_FT = []; - rotParticle = []; - end % end loop over possible peaks - end % loop over subTomos - - - for iPeak = 1:nPeaks - - % Get rid of any zero entries left over from pre-initialization - if iPeak == 1 - nonZeroInits = ( cccStorageBest{iPeak}(:,2) ~= 0 ); - cccStorageBest{1}=cccStorageBest{1}(nonZeroInits,:); - sortCCC = zeros(size(cccStorageBest{1},1),10*nPeaks); - else - cccStorageBest{iPeak}=cccStorageBest{iPeak}(nonZeroInits,:); - end - - sortCCC(:,1+10*(iPeak-1):10+10*(iPeak-1)) = cccStorageBest{iPeak}; - end - -% % % % I think this is redundant now, but leaving until I double check. -% % % save('sortCCC.mat','sortCCC'); - [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); - - cccSortedandUnique = sortCCC(a,:); -% % % save('cccSortedandUnique.mat','cccSortedandUnique'); -% % % g = gather(geometry); -% % % save('TBL_geom.mat','g'); - - bestAngles_tmp.(tomoList{iTomo}) = gather(cccSortedandUnique); - - % save doesn't work in a parfor, so write out the results for each tomogram so that a - % run may be resumed if cancelled. - angOut = fopen(sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}),'w'); - - for iRow = 1:size( bestAngles_tmp.(tomoList{iTomo}),1) - for iPeak = 1:nPeaks - fprintf(angOut,'%d %d %6.3f %6.3f %6.3f %6.6f %6.6f %6.3f %6.3f %6.3f ', ... - bestAngles_tmp.(tomoList{iTomo})(iRow,1+10*(iPeak-1):10+10*(iPeak-1))); - end - fprintf(angOut,'\n'); - end - fclose(angOut); - - end % if clause to check for previous alignment - end % loop over tomos - bestAnglesResults{iParProc} = bestAngles_tmp; - geometryResults{iParProc} = geometry_tmp; -%profile off -%profsave -end % parfor - - - -save('bestAnglesResults.mat', 'bestAnglesResults'); -bestAngles = struct(); -for iParProc = 1:nParProcesses - for iTomo = iterList{iParProc} - geometry.(tomoList{iTomo}) = geometryResults{iParProc}.(tomoList{iTomo}); - bestAngles.(tomoList{iTomo}) = bestAnglesResults{iParProc}.(tomoList{iTomo}); - end -end -% save('bestAnglesTemp.mat', 'bestAngles'); - save('bestAngles.mat', 'bestAngles'); - - [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, nPeaks,rotConvention ); - masterTM.(cycleNumber).('RawAlign') = rawAlign; - masterTM.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); - -clear bestAngles rawAlign -subTomoMeta = masterTM; - -if ( flgReverseOrder || flgStartThird ) - fprintf('This reverse run will not write the metaData\n'); -else - save(pBH.('subTomoMeta'), 'subTomoMeta'); -end - -delete(gcp('nocreate')) -for iGPU = 1:nGPUs - gpuDevice(iGPU); -end - -end % end of alignRaw3d diff --git a/alignment/BH_alignReferences3d.m b/alignment/BH_alignReferences3d.m deleted file mode 100755 index bdfb9287..00000000 --- a/alignment/BH_alignReferences3d.m +++ /dev/null @@ -1,759 +0,0 @@ -efunction [ ] = BH_alignReferences3d( PARAMETER_FILE, CYCLE) -%Align an symmetrize references -% - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -if (nargin ~= 2) - error('args = PARAMETER_FILE, CYCLE') -end -startTime = clock; -CYCLE = EMC_str2double(CYCLE); - -% not for normal use, but under certain circumstances allow override of 0.5 cutoff for -% ref and allow to 0.143 -if (CYCLE < 0) - CYCLE = abs(CYCLE); - flgAlignCutoff = 0; -else - flgAlignCutoff = 1; -end - -cycleNumber = sprintf('cycle%0.3u', CYCLE); - -pBH = BH_parseParameterFile(PARAMETER_FILE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); - -maxGoldStandard = subTomoMeta.('maxGoldStandard'); -flgClassify = pBH.('flgClassify'); -try - flgMultiRefAlignment = pBH.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end - - -memoryLevel = pBH.('avgMemory'); - -% specify an angular shift to apply to the first reference, "" translational, -% identify the ref to align all other refs to, and whether to use bandpass from -% param (based on est from FSC on RawAlign average, or re-extract and calc FSC -% to design filter. The second option will only work in later iterations as the -% alignment of the raw images gets better, as symmetry will magnify small -% geometrical errors.) -bFactor = pBH.('Fsc_bfactor'); -samplingRate = pBH.('Ali_samplingRate'); -pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; -if pBH.('SuperResolution') - pixelSize = pixelSize * 2; -end -try - scaleCalcSize = pBH.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end -angleSearch = pBH.('Ref_angleSearch'); - -refName = pBH.('Ref_className'); - -peakSearch = floor(pBH.('particleRadius')./pixelSize); -peakCOM = [1,1,1].*peakCOM; -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); - - - -flgAngleShift{1}= pBH.('ref_AngleShift_odd'); -flgTransShift{1}= pBH.('ref_TransShift_odd'); -flgRefRef{1} = pBH.('ref_Ref_odd'); -refVectorFull{1}= pBH.('Ref_references_odd'); -features{1} = pBH.('Pca_coeffs_odd'); -% % % vol_geometry{1} = subTomoMeta.(cycleNumber).('ClusterResults').( ... -% % % sprintf('%s_%d_%d_nClass_%d_ODD', ... -% % % outputPrefix,features{1}(1,1), ... -% % % features{1}(1,end), refName)); - -flgAngleShift{2}= pBH.('ref_AngleShift_eve'); -flgTransShift{2}= pBH.('ref_TransShift_eve'); -flgRefRef{2} = pBH.('ref_Ref_eve'); -refVectorFull{2}= pBH.('Ref_references_eve'); -features{2} = pBH.('Pca_coeffs_eve'); -% % % vol_geometry{2} = subTomoMeta.(cycleNumber).('ClusterResults').( ... -% % % sprintf('%s_%d_%d_nClass_%d_EVE', ... -% % % outputPrefix,features{2}(1,1), ... -% % % features{2}(1,end), refName)); -% Merge alignments that could come from using different feature vectors in the -% classification, back into one metadata. -% % % vol_geometry = BH_mergeClassGeometry(vol_geometry{1},vol_geometry{2}); - % Copy to reset after initial extraction of primary references. -vol_geometry = subTomoMeta.(cycleNumber).('ClusterRefGeom'); - -vol_geometry_clean = vol_geometry; - - - - - - - - - -masterTM = subTomoMeta; clear subTomoMeta - - - -refVector = cell(2,1); -refGroup = cell(2,1); -refSym = cell(2,1); - -for iGold = 1:2 - refVectorFull{iGold} - % Sort low to high, because order is rearranged as such unstack - refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; - % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) - % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) - % axial symmetry to apply, negative value indicates creating a mirrored ref - % accros the corresponding axis - refSym{iGold} = refVectorFull{iGold}(2,:) -end - -% make sure the number of references match the unique groups in the classVector -% and also that the class/group pairs match the class/ref pairs. -nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]; - -uniqueSym = cell(2,1); -for iGold = 1:2 - [~,uniqueGroup,~] = unique(refGroup{iGold}); - uniqueSym{iGold} = refSym{iGold}(uniqueGroup) -end - - - - - -[ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Cls', samplingRate); - -[ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea(maskSize, maskRadius, scaleCalcSize) -padREF = [0,0,0;0,0,0]; - - -if any(peakSearch > maskRadius) - fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') - - peakSearch( (peakSearch > maskRadius) ) = ... - maskRadius( (peakSearch > maskRadius) ); -end - - - -% To properly average averages without re-extracting, the original number -% in each average must be taken in to account. - - -% Read in the references. -refIMG = cell(2,1); -imgCounts = cell(2,1); -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - - - % To allow more flexibility ref/ cls field prefixes are used, and rather than - % making something special for the naming of cls/ref locations which SHOULD be - % interchangeable, just try one then the other since this impacts nothing - % else. Use the unweighted average for the first pass - - try - imgNAME = sprintf('class_%d_Locations_Ref_%s', refName, halfSet); - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - [ refIMG{iGold} ] = BH_unStackMontage4d(refVector{iGold}, ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - catch - imgNAME = sprintf('class_%d_Locations_Cls_%s', refName, halfSet); - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - [ refIMG{iGold} ] = BH_unStackMontage4d(refVector{iGold}, ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - end - - occCell = BH_multi_isCell( refIMG{iGold} ); - nRefs = length(occCell); - tIMG = cell(nRefs,1); - for iRef = 1:nRefs - tIMG{iRef} = refIMG{iGold}{occCell(iRef)}; - end - refIMG{iGold} = tIMG; clear tIMG - - sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}; - sizeREF = sizeREF(2:2:6)' - -% % % % clear out empty cell contents, and return -% % % n = 1 ; tIMG = cell(nReferences(iGold)); -% % % for iP = 1:length(refTMP) -% % % if ~isempty(refTMP{iP}) -% % % tIMG{n} = refTMP{iP}; -% % % -% % % n = n + 1; -% % % end -% % % end -% % % -% % % refIMG{iGold} = tIMG ; clear tIMG refTMP -% % % - - -end - -% % Prevent divergent orientations -% [ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, 40 ); - -% % % [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc, padREF ] = ... -% % % BH_multi_validArea( maskRadius, sizeREF ) - - -% optimize the fft for the given size. Padding to the next power of 2 is usually -% slower given the dimensionality of the volume data. -fftPlanner = rand(sizeCalc); -fftw('planner', 'exhaustive'); -fftn(fftPlanner); -clear fftPlanner - - -% Make a mask, and apply to the average motif && save a masked, binned copy of -% the average for inspection. -mask = struct(); - - - [ volMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); - mask.('volMask') = gather(volMask); - - [ peakMask ] = gpuArray(BH_mask3d(maskType, sizeMask, peakSearch, maskCenter)); - mask.('peakMask') = gather(peakMask); - - peakBinary = (peakMask >= 0.01); - -% The bandpass here is from the full average - assuming each class is a -% substantial portion of the total population. - - bandpassFilt = cell(1,1); - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'cpu', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - for iRef = 1:1 - fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); - - % The class averages have roughly the same SNR as the references so apply any - % bFactor to them as well. - [ bandpassFilt{iRef}, ~ ] = BH_multi_cRef( fscINFO, radialGrid , bFactor,flgAlignCutoff); - end - clear radialGrid - mask.('bandpassFilt') = (bandpassFilt); - bandpassFilt{1} = gpuArray(bandpassFilt{1}); -% [maskMontage,~] = BH_montage4d({gather(volMask), gather(peakMask)},''); -% SAVE_IMG(MRCImage(maskMontage), sprintf('%s_maskMontage.mrc', outputPrefix)); - -%Improve the estimated center for the reference used to align any other -% optional references or sub references, applying any symmetry as well. Then -% re-extract these refs with the new transformation. -estShifts = cell(2,1); - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - halfNUM = iGold; - - % The first class in the refRef vector is used to align the primary reference - % for each reference group, to which the members of each group are - % subsequently aligned. - refRefIDX = find(refVector{iGold} == flgRefRef{iGold}(1)); - refRef = refIMG{iGold}{refRefIDX}; - % Only apply trans shifts from estimate, then later use angles as well. - refRef = BH_axialSymmetry(refRef,1, 0,'GPU',flgTransShift{iGold}(1,:)); - - refRef = refRef(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3) ); - - refAsym= refIMG{iGold}{refRefIDX}(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3) ); - - - - refAsym= gpuArray(refAsym); - - - - - refAsym_FT = BH_bandLimitCenterNormalize(refAsym.*peakMask, bandpassFilt{1}, ... - peakBinary,padCalc,'double'); - refRef_FT =conj(BH_bandLimitCenterNormalize(refRef.*peakMask, bandpassFilt{1},... - peakBinary,padCalc,'double')); - - - [ estPeakCoord ] = BH_multi_xcf_Translational( refAsym_FT, refRef_FT, ... - peakMask, peakCOM); - - estShifts{iGold} = gather(estPeakCoord); - fprintf('estPeak at %f %f %f\n estShifts now %f %f %f \n', estPeakCoord', estShifts{iGold}'); - clear refRef refAsym refRef_FT refAsym_FT - - - refRef = BH_axialSymmetry(refIMG{iGold}{refRefIDX}, refSym{iGold}(refRefIDX),... - flgAngleShift{iGold}(1), ... - 'GPU',estShifts{iGold}); - refRefRotAvg = BH_axialSymmetry(refIMG{iGold}{refRefIDX}, 120,... - flgAngleShift{iGold}(1), ... - 'GPU',estShifts{iGold}); - - if isa(refRef,'cell') - refRef = refRef{2}; - end - - SAVE_IMG(MRCImage(gather(refRef)),sprintf('initialAxialOffsetCheck_%s.mrc',halfSet)); - refWDG = NaN; - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % % - % To allow more flexibility ref/ cls field prefixes are used, and rather than - % making something special for the naming of cls/ref locations which SHOULD be - % interchangeable, just try one then the other since this impacts nothing - % else. - - try - imgClassNAME = sprintf('class_%d_Locations_%s_%s', refName, 'Ref',halfSet); - - - [ classIMG ] = BH_unStackMontage4d(1:refName, ... - masterTM.(cycleNumber).(imgClassNAME){1},... - masterTM.(cycleNumber).(imgClassNAME){2},... - sizeWindow); - catch - imgClassNAME = sprintf('class_%d_Locations_%s_%s', refName, 'Cls',halfSet); - - - [ classIMG ] = BH_unStackMontage4d(1:refName, ... - masterTM.(cycleNumber).(imgClassNAME){1},... - masterTM.(cycleNumber).(imgClassNAME){2},... - sizeWindow); - end - - - - - %%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't - %%%%%%%%%%%%%%%%%%%%% search at all in that dimension. - - [ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch(1,:)) - - - - % First align the reference reference to any sub-references in its group. After - % finding the best alignment extract from the tomograms, with any symmetry. - - % Second align the first member of each ref group to the new reference, and use - % alignment to resample the head of each ref group. - - % Third align each sub-ref to the head of each ref group, and finally extract - % these. - - - -% % Store the cross correlation score, peak location, and wedge weight - bestAnglesTotal = zeros(nReferences(iGold),10); - nCount = 1; - - for iClass = flgRefRef{iGold}(1,:) - tic; - % Load the class onto gpu - iClassImg = gpuArray(classIMG{iClass}); - - iClassWdg = NaN; - - - - [ alignOUT ] = run_alignment(angleStep, inPlaneSearch, ... - iClassImg, iClassWdg, ... - refRef, refWDG, refRefRotAvg, ... - volMask, bandpassFilt, padCalc, padWindow, ... - peakMask, peakCOM, iClass, ... - uniqueSym{iGold}); - - - bestAnglesTotal(nCount,:) = [gather(alignOUT(1,1:10))]; - - timeClass = toc; - fprintf('finished working on %d/%d classes...%fs\n',nCount, ... - length(flgRefRef{iGold}),timeClass); - nCount = nCount + 1; - - end % loop over (classes) - - bestAnglesTotal = gather(bestAnglesTotal); - - - [ vol_geometry ] = BH_refAlignmentsApply( vol_geometry, bestAnglesTotal,... - samplingRate, ... - flgRefRef{iGold}(1,:), flgRefRef{iGold}(2,:),halfNUM ); - - - - -end - -masterTM.(cycleNumber).('RefAlignment') = vol_geometry; -subTomoMeta = masterTM; -save(pBH.('subTomoMeta'), 'subTomoMeta'); - - -gpuDevice(1); -BH_average3d(PARAMETER_FILE,num2str(CYCLE),'RefAlignment'); - -% average3d puts info about the ref locations in the montage, so reload the -% metadata - -load(pBH.('subTomoMeta')); -masterTM = subTomoMeta; - -% reload masks onto gpu -volMask = gpuArray(mask.('volMask')); -peakMask = gpuArray(mask.('peakMask')); -for iRef = 1:length(mask.('bandpassFilt')) - bandpassFilt{iRef} = gpuArray(mask.('bandpassFilt'){iRef}); -end - -clear refRef refRefRotAvg initPeakMask wdgIMG refIMG - - -% Reset the geometry so that the original classes persist. -vol_geometry = vol_geometry_clean; -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Now we have two montages of references (if gold standard) which later I will -% add in an fsc comparision, which will be used to derive an ideal bandpass -% filter. For now, continue on to use these to further align "sub references" -% using the supplied bandpass filter. -refIMG = cell(2,1); -imgCounts = cell(2,1); - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - halfNUM = iGold; - - % Read in the new refs we just created. - newRefNAME = sprintf('class_%d_Locations_%s_%s', refName, 'REF',halfSet); - - fprintf('reading in new ref %s.\n',halfSet); - [ refIMG{iGold} ] = BH_unStackMontage4d(1:nRefs, ... - masterTM.(cycleNumber).(newRefNAME){1},... - masterTM.(cycleNumber).(newRefNAME){2},... - sizeWindow); - - imgCounts{iGold} = masterTM.(cycleNumber).(newRefNAME){3}; - - occCell = BH_multi_isCell( refIMG{iGold} ); - nRefs = length(occCell); - tIMG = cell(nRefs,1); - for iRef = 1:nRefs - tIMG{iRef} = refIMG{iGold}{occCell(iRef)}; - end - refIMG{iGold} = tIMG; clear tIMG - - % Save the intermediate "primaryRef" since this will make trouble shooting - % errors in symmetry/grouping until explicit error checks can be added to - % BH_parseParameterFile - CMD = sprintf('mv %s_filtered%d_REF_%s.mrc %s_primaryRef_%d_%s.mrc ', ... - outputPrefix, refName, halfSet,outputPrefix, refName, halfSet); - system(CMD, '-echo'); -end - - -[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, maxGoldStandard ); - -[ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch(2,:)) - -for iGold = 1:2 - - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - halfNUM = iGold; - - refImg = zeros([size(refIMG{iGold}{1}),nRefs],'single','gpuArray'); - refRotAvg = zeros([size(refIMG{iGold}{1}),nRefs], 'single','gpuArray'); - %refWdg = zeros([size(refIMG{iGold}{1}),nRefs],'single','gpuArray'); - refWdg = NaN - for iRef = 1:nRefs; - refRotAvg(:,:,:,iRef) = BH_axialSymmetry(refIMG{iGold}{iRef},120,0,'GPU',[0,0,0]); - refImg(:,:,:,iRef) = refIMG{iGold}{iRef}; - end - - - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - - % Store the cross correlation score, peak location, and wedge weight - bestAnglesTotal = zeros(nReferences(iGold),10); - nCount = 1; - - - imgClassNAME = sprintf('class_%d_Locations_%s_%s', refName, 'Ref',halfSet) - refName - - [ classIMG ] = BH_unStackMontage4d(1:refName, ... - masterTM.(cycleNumber).(imgClassNAME){1},... - masterTM.(cycleNumber).(imgClassNAME){2},... - sizeWindow); - - - - - refVector{iGold}(1,:) - for iClass = refVector{iGold}(1,:) - tic; - iClass - % Load the class onto gpu - iClassImg = gpuArray(classIMG{iClass}); - - iClassWdg = NaN; - - - - % only use the relevant class - iGroup = refGroup{iGold}(ismember(refVector{iGold},iClass)) - - - [ alignOUT ] = run_alignment(angleStep, inPlaneSearch, ... - iClassImg, iClassWdg, ... - refImg(:,:,:,iGroup), refWdg, ... - refRotAvg(:,:,:,iGroup), ... - volMask, bandpassFilt, padCalc, padWindow, ... - peakMask, peakCOM, iClass,uniqueSym{iGold}); - - - bestAnglesTotal(nCount,:) = [gather(iGroup),gather(alignOUT(1,2:10))]; - - timeClass = toc; - fprintf('finished working on %d/%d classes %s...%fs\n',nCount, ... - length(refVector{iGold}),halfSet,timeClass); - nCount = nCount + 1; - - end % loop over (classes) - - bestAnglesTotal = gather(bestAnglesTotal) - - - [ vol_geometry ] = BH_refAlignmentsApply( vol_geometry, bestAnglesTotal,... - samplingRate, ... - refVector{iGold}(1,:), refGroup{iGold}(1,:),halfNUM ); - - -end - - masterTM.(cycleNumber).('RefAlignment') = vol_geometry; - subTomoMeta = masterTM; - save(pBH.('subTomoMeta'), 'subTomoMeta'); - - BH_average3d(PARAMETER_FILE,num2str(CYCLE),'RefAlignment'); - - -fprintf('total execution time : %f seconds\n', etime(clock, startTime)); - - - -function [ alignOUT ] = run_alignment(angleStep, inPlaneSearch, ... - iClassImg, iClassWdg, ... - refRef, ref_WDG, refRefRotAvg, ... - volMask, bandpassFilt, padCalc, padWindow, ... - peakMask, peakCOM, iClass,uniqueSym) - - clear cccStorage1 cccStorage2 cccStorage3 cccStorage4 - - % Out of plane search - if any(angleStep(2:4)) - flgSearchDepth = 3; - - elseif any(angleStep(5)) - % in plane - flgSearchDepth = 2; - peakListTop10(angleStep(5).*nReferences,6) = gpuArray(0); - nPeak = 1; - for iRef = 1 - for iPsi = inPlaneSearch - peakListTop10(nPeak,1) = iRef; - nPeak = nPeak + 1; - end - end - else - error('specify at least an in plane search') - end - - if (flgSearchDepth == 3) - - [ cccStorage1 ] = BH_multi_angularSearch( angleStep, 0, 0, ... - iClassImg, iClassWdg, ... - refRefRotAvg, ref_WDG, ... - refRefRotAvg, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM, iClass, ... - uniqueSym); - - - cccStorage1(1:10,:) - - - % Second loop over top 10 peaks now using the non-rotationally averaged - % reference and including out of plane angles. - - % peakList is # rows = top peaks - % reference, phi, theta - % put zero peak at top of list - zeroPeak = sortrows(cccStorage1,[3, 4, 5]); - zeroPeak = [zeroPeak(1,:) ; cccStorage1 ]; - % if zero peak was already there, remove it so no duplicate - zeroPeak = unique(zeroPeak, 'stable', 'rows'); - peakListTop10 = [zeroPeak(1:10,1),zeroPeak(1:10,3:4),zeroPeak(1:10,8:10)] - end - - - [ cccStorage2 ] = BH_multi_angularSearch( angleStep, peakListTop10, ... - inPlaneSearch, ... - iClassImg, iClassWdg, ... - refRef, ref_WDG, ... - refRefRotAvg, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM, iClass, ... - uniqueSym); - - - cccStorage2 = unique(cccStorage2((cccStorage2(:,6) ~= 0),:), 'stable','rows'); - if size(cccStorage2, 1) > 9 - cccStorage2(1:10,:) - else - cccStorage2 - end - - % This is to save time assuming that we can get a good estimate of the - % particles shift by taking the average of the higher ranking alignments. In - % testing this was always within ~ half a pixel. If CCC scores are strangely - % low, suspect this as a break point. - - - % Get the top three peaks with unique phi, and theta - % Return [ref,phi,theta,psi,phistep,thetastep,psistep] - % Stable prevents any sorting - if (flgSearchDepth== 3) - [~,ia,~] = unique(cccStorage2(:,3:4),'stable' ,'rows'); - peakListTop3 = zeros(3,10); - else - [~,ia,~] = unique(cccStorage2(:,3:5),'stable' ,'rows'); - peakListTop3 = zeros(3,10); - end - - if numel(ia) >= 10 - TOP = 10; - else - TOP = numel(ia); - end - - - for top3 = 1:TOP - outOfPlaneAngle = cccStorage2(ia(top3),4); - angleIndex = find(angleStep(:,1)==outOfPlaneAngle,1,'first'); - if (flgSearchDepth == 3 ) - % Search around top 3 +/- 0.5 the original out of plane angular increment - peakListTop3(top3,:) = [cccStorage2(ia(top3),1), ... - cccStorage2(ia(top3),3:5),... - angleStep(angleIndex,3)./4,... - angleStep(angleIndex,4)./2,... - angleStep(angleIndex,5)./2, cccStorage2(ia(top3),8:10)]; - else - % Search around top 3 +/- 0.5 the original out of plane angular increment - peakListTop3(top3,:) = [cccStorage2(ia(top3),1), ... - cccStorage2(ia(top3),3:5),... - 0,... - 0,... - angleStep(1,5)./2, cccStorage2(ia(top3),8:10)]; - end - end - peakListTop3 - [ cccStorage3 ] = BH_multi_angularSearch( angleStep, peakListTop3, ... - 0, ... - iClassImg, iClassWdg, ... - refRef, ref_WDG, ... - refRefRotAvg, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM,iClass, ... - uniqueSym); - - - cccStorage3 = unique(cccStorage3((cccStorage3(:,6) ~= 0),:), 'stable','rows'); - if size(cccStorage3, 1) > 9 - cccStorage3(1:10,:) - else - cccStorage3 - end - - - if (flgSearchDepth == 3 ) - % Use previous increments/2 - - % Search around the top peak +/- 0.25 the orginal angular increment - topPeak = [cccStorage3(1,1), ... - cccStorage3(1,3:5), ... - cccStorage3(1,11:13)./2, ... - cccStorage3(1,8:10)] - else - topPeak = [cccStorage3(1,1), ... - cccStorage3(1,3:5), ... - 0, ... - 0,... - angleStep(1,5)./3, cccStorage3(1,8:10)] - end - - [ cccStorage4 ] = BH_multi_angularSearch( angleStep, topPeak, ... - 0, ... - iClassImg, iClassWdg, ... - refRef, ref_WDG, ... - refRefRotAvg, ... - volMask, bandpassFilt, ... - padCalc, padWindow, ... - peakMask, peakCOM, iClass, ... - uniqueSym); - - alignOUT = cccStorage4(1,:) - - - -end - diff --git a/metaData/BH_classAlignmentsApply.m b/metaData/BH_classAlignmentsApply.m deleted file mode 100755 index f0bde999..00000000 --- a/metaData/BH_classAlignmentsApply.m +++ /dev/null @@ -1,89 +0,0 @@ -function [UPDATED_GEOMETRY] = ... - BH_classAlignmentsApply( INPUT_GEOMETRY ,... - BEST_ANGLES, SAMPLING, CLASS_VECTOR, iGold, halfSet) -%Apply class alignments to the full set of subTomograms -% -% -% Input variables -% -% ALIGN_GEOM = string with mat file that has the geometry output from -% alignment. -% -% -% Output variables = none, the clustering geometry updated - eventually this -% -% -% -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% Goals & limitations -% -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% TODO -% 1)add check that new origin is not too close to the edge. -% 2)get rid of 'find' and switch from index to logical indexing -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - - - -inputGeometry = INPUT_GEOMETRY; -alignmentGeometry = BEST_ANGLES; -classVector = CLASS_VECTOR(1,:); - - -% Get the number of tomograms to proc - -tomoList = fieldnames(inputGeometry); -nTomograms = length(tomoList); - -for iTomo = 1:nTomograms - - positionList = inputGeometry.(tomoList{iTomo}); - newAlignment = alignmentGeometry; - - if strcmpi(halfSet,'ODD') || strcmpi(halfSet,'EVE') - includeList = find(ismember(positionList(:,26), classVector) & ... - ismember(positionList(:,7), iGold)); -% % % removeList = ~ismember(positionList(:,26), classVector) & ... -% % % ismember(positionList(:,7), iGold); -% % % positionList(removeList,26) = -9999; - else - includeList = find(ismember(positionList(:,26), classVector)); -% % % removeList = ~ismember(positionList(:,26), classVector); -% % % positionList(removeList,26) = -9999; - end - - for iParticle = includeList' - % assuming all classes are sequential, only discarded between cycles. - class = positionList(iParticle,26); - pIndex = find(newAlignment(:,2) == class); %%% 1-->2 - - if isempty(pIndex) - fprintf('pIndex is empty for %s pos %d class %d\n', tomoList{iTomo}, iParticle,class); - else - classRot = BH_defineMatrix(newAlignment(pIndex,3:5),'Bah','inv'); - - oldRot = reshape(positionList(iParticle,17:25),3,3); - - newRot = gather(reshape( oldRot * classRot, 1,9)); - - positionList(iParticle,17:25) = newRot; - - - - - inputGeometry.(tomoList{iTomo}) = positionList; - - end - - end - -UPDATED_GEOMETRY = inputGeometry; - -end - diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 473f7e34..3b823b8b 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -90,10 +90,6 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... fieldPrefix = 'Raw' case 'RawAlignment' fieldPrefix = 'Raw'; - case 'ClassAlignment' - fieldPrefix = 'Cls'; - case 'NoAlignment' - fieldPrefix = 'NoA'; case 'Cluster_cls' fieldPrefix = 'Cls'; STAGEofALIGNMENT = 'Cluster'; @@ -120,26 +116,6 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... case 'TiltAlignment' geometry = subTomoMeta.tiltGeometry; subTomoMeta.(cycleNumber).(sprintf('Pre_%s_tiltGeometry', OPERATION)) = geometry; - case 'ClassAlignment' - if (undoOP) - subTomoMeta.(cycleNumber).ClassAlignment = ... - subTomoMeta.(cycleNumber).(sprintf('Pre_%s_ClassAlignment', OPERATION)); - save(pBH.('subTomoMeta'), 'subTomoMeta'); - error('No Error, just exiting.\n') - else - geometry = subTomoMeta.(cycleNumber).ClassAlignment; - subTomoMeta.(cycleNumber).(sprintf('Pre_%s_ClassAlignment', OPERATION)) = geometry; - end - case 'NoAlignment' - if (undoOP) - subTomoMeta.(cycleNumber).geometry = ... - subTomoMeta.(cycleNumber).(sprintf('Pre_%s_geometry', OPERATION)); - save(pBH.('subTomoMeta'), 'subTomoMeta'); - error('No Error, just exiting.\n') - else - geometry = subTomoMeta.(cycleNumber).geometry; - subTomoMeta.(cycleNumber).(sprintf('Pre_%s_geometry', OPERATION)) = geometry; - end case 'RawAlignment' if (undoOP) subTomoMeta.(cycleNumber).RawAlign = ... @@ -180,7 +156,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... error('No Error, just exiting.\n') else - if strcmpi(fieldPrefix, 'cls') + if strcmpi(fieldPrefix, 'Cls') geometry = subTomoMeta.(cycleNumber).('ClusterClsGeom'); clusterGeom = 'ClusterClsGeom'; elseif strcmpi(fieldPrefix, 'ref') @@ -557,15 +533,10 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... case 'TiltAlignment' masterTM.tiltGeometry = geometry; - case 'ClassAlignment' - masterTM.(cycleNumber).ClassAlignment = geometry; - case 'NoAlignment' - masterTM.(cycleNumber).geometry = geometry; + case 'RawAlignment' masterTM.(cycleNumber).RawAlign = geometry; case 'Cluster' - % cN = sprintf('%s_%d_%d_nClass_%d_%s',outputPrefix,classCoeffs{halfNUM(1)}(1,1), ... - % classCoeffs{halfNUM(1)}(1,end), className, halfSet) masterTM.(cycleNumber).(clusterGeom) = geometry; diff --git a/metaData/BH_skipClassAlignment.m b/metaData/BH_skipClassAlignment.m index ec387288..0e7d4425 100755 --- a/metaData/BH_skipClassAlignment.m +++ b/metaData/BH_skipClassAlignment.m @@ -38,80 +38,35 @@ load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); -%className = pBH.('Cls_className'); -%refName = pBH.('Ref_className'); -if strcmpi(STAGEofALIGNMENT, 'ClassAlignment') + +if strcmpi(STAGEofALIGNMENT, 'RawAlignment') if (flgMultiRefAlignment && ~flgClassify) - subTomoMeta.(cycleNumber).('ClassAlignment') = ... + subTomoMeta.(cycleNumber).('RawAlign') = ... subTomoMeta.(cycleNumber).('Avg_geometry'); - - elseif (flgMultiRefAlignment && flgClassify) - subTomoMeta.(cycleNumber).('ClassAlignment') = ... + subTomoMeta.(cycleNumber).('RawAlign') = ... subTomoMeta.(cycleNumber).('ClusterClsGeom'); - else try - subTomoMeta.(cycleNumber).('ClassAlignment') = ... + subTomoMeta.(cycleNumber).('RawAlign') = ... subTomoMeta.(cycleNumber).('ClusterClsGeom'); catch - subTomoMeta.(cycleNumber).('ClassAlignment') = ... + subTomoMeta.(cycleNumber).('RawAlign') = ... subTomoMeta.(cycleNumber).('ClusterRefGeom'); end - % if (flgGold) - % features1 = pBH.('Pca_coeffs_odd'); - % features2 = pBH.('Pca_coeffs_eve'); - % GEOM1 = sprintf('%s_%d_%d_nClass_%d_ODD',outputPrefix,features1(1,1), ... - % features1(1,end), className); - % GEOM2 = sprintf('%s_%d_%d_nClass_%d_EVE',outputPrefix,features2(1,1), ... - % features2(1,end), className); - % - % % if class averages weren't extracted, just references, check and use these - % % instead. build in tool "isfield" doesn't work for dynamic names, so use - % % eval, which will work if the field exists, otherwise throw an error. - % - % try - % %checkClassAvg = isfield(subTomoMeta, sprintf( '(%s).(%s).(%s)', cycleNumber, 'ClusterResults',GEOM1)); - % eval(sprintf( 'subTomoMeta.(''%s'').(''%s'').(''%s'')', cycleNumber, 'ClusterResults',GEOM1)); - % catch - % - % GEOM1 = sprintf('%s_%d_%d_nClass_%d_ODD',outputPrefix,features1(1,1), ... - % features1(1,end), refName); - % GEOM2 = sprintf('%s_%d_%d_nClass_%d_EVE',outputPrefix,features2(1,1), ... - % features2(1,end), refName); %% -% -% end -% -% -% -% [ GEOM_OUT ] = BH_mergeClassGeometry( subTomoMeta.(cycleNumber).('ClusterResults').(GEOM1), ... -% subTomoMeta.(cycleNumber).('ClusterResults').(GEOM2));% -% -% else -% features1 = pBH.('Pca_coeffs_odd'); -% GEOM1 = sprintf('%s_%d_%d_nClass_%d_STD',outputPrefix,features1(1,1), ... -% features1(1,end), className) -% GEOM_OUT = subTomoMeta.(cycleNumber).('ClusterResults').(GEOM1); -% end -% -% subTomoMeta.(cycleNumber).('ClassAlignment') = GEOM_OUT; end -elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') - subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('ClassAlignment'); else - error(['STAGEofALIGNMENT to skip may be RawAlignment or ClassAlignment,'],... + error(['STAGEofALIGNMENT to skip may be RawAlignment'],... ['the former requires the latter to exist.\n']); end save(pBH.('subTomoMeta'), 'subTomoMeta'); - end diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index dc30ed05..67fef996 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -188,7 +188,6 @@ nReferences = length(classVector(1,:)) -% % % % imageName{1} = sprintf('class_%d_Locations_%s_ODD_NoWgt', className,fieldPrefix); imageName{2} = sprintf('class_%d_Locations_%s_EVE_NoWgt', className,fieldPrefix); weightName{1} = sprintf('class_%d_Locations_%s_ODD_Wgt', className,fieldPrefix); @@ -197,12 +196,8 @@ refVector{1} =1; refVector{2}= 1; -% % % % nReferences = 1; outputPrefix = sprintf('%s_Raw', outputPrefix); - - case 'ClassAlignment' - error('Fsc calculation for class averages is not implemented.') case 'NoAlignment' savePrefix = 'Raw'; if (flgClassify) diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 16b70eec..2b499980 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -283,7 +283,6 @@ fprintf(['\nUsage: emClarity skip param.m iter\n']); else emC_testParse(varargin{2}) - BH_skipClassAlignment(varargin{2},varargin{3},'ClassAlignment','1'); BH_skipClassAlignment(varargin{2},varargin{3},'RawAlignment','1'); end case 'rescale' diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 3a7dc58d..e98121a4 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -319,17 +319,7 @@ classVector{2} = [0;1]; end - - - case 'ClassAlignment' - fieldPrefix = 'Cls'; - - classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); - classVector{2} = pBH.(sprintf('%s_classes_eve',fieldPrefix)); - - samplingRate = pBH.(sprintf('Cls_samplingRate')); - classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); - className = pBH.(sprintf('%s_className',fieldPrefix)); + case 'FinalAlignment' % Special case for the final cycle. @@ -533,8 +523,6 @@ eachTomo = false; flgEstSNR = 0; switch STAGEofALIGNMENT - case 'ClassAlignment' - geometry = subTomoMeta.(cycleRead).ClassAlignment; case 'RawAlignment' if ( CYCLE ) geometry = subTomoMeta.(cycleRead).RawAlign; From d46a1a254cbee43f8eb86c61e7f67cfcea0f73df Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 08:24:01 -0500 Subject: [PATCH 005/151] Remove SOA RefAlignment --- metaData/BH_geometryAnalysis.m | 5 +-- statistics/BH_fscGold_class.m | 17 ----------- testScripts/emClarity.m | 4 +-- transformations/BH_average3d.m | 56 +++++----------------------------- 4 files changed, 10 insertions(+), 72 deletions(-) diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 3b823b8b..a71dce2b 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -93,11 +93,8 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... case 'Cluster_cls' fieldPrefix = 'Cls'; STAGEofALIGNMENT = 'Cluster'; - case 'Cluster_ref' - fieldPrefix = 'Ref'; - STAGEofALIGNMENT = 'Cluster'; otherwise - error('STAGEofALIGNMENT incorrect, should be [Tilt,Class,No,Raw]Alignment, not %s', ... + error('STAGEofALIGNMENT incorrect, should be Cluster_cls or [Tilt,Raw]Alignment, not %s', ... STAGEofALIGNMENT); end diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 67fef996..03bcf038 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -218,23 +218,6 @@ outputPrefix = sprintf('%s_NoA', outputPrefix); case 'Cluster' error('Fsc calculation for cluster results is not implemented.') - case 'RefAlignment' - savePrefix = 'REF'; - imageName{1} = sprintf('class_%d_Locations_REF_ODD_NoWgt', refName); - imageName{2} = sprintf('class_%d_Locations_REF_EVE_NoWgt', refName); - weightName{1} = sprintf('class_%d_Locations_REF_ODD_Wgt', refName); - weightName{2} = sprintf('class_%d_Locations_REF_EVE_Wgt', refName); - - outputPrefix = sprintf('%s_Ref', outputPrefix); - refVector{1} = 1; %pBH.('ref_Ref_odd')(1,:); - refVector{2} = 1; %pBH.('ref_Ref_eve')(1,:); - - fieldPrefix = 'REF'; - if length(refVector{1}) ~= length(refVector{2}) - error('Fsc ref vectors are not the same length.') - else - nReferences = length(refVector{1}); - end case 'SnrEstimate' savePrefix = 'SNR'; flgEstSNR = 1; diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 2b499980..df4c652a 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -349,7 +349,7 @@ 'cycle number\n',... 'stage of alignment\n',... ' raw (post raw alignment)\n',... - ' cluster_ref or cluster_cls (post classification)\n']); + ' cluster_cls (post classification)\n']); else emC_testParse(varargin{2}) BH_average3d(varargin{2}, varargin{3}, varargin{4}); @@ -362,7 +362,7 @@ 'cycle number\n',... 'stage of alignment\n',... ' raw (post raw alignment)\n',... - ' cluster_ref or cluster_cls (post classification)\n']); + ' cluster_cls (post classification)\n']); elseif length(varargin) == 4 emC_testParse(varargin{2}) BH_fscGold_class(varargin{2}, varargin{3}, varargin{4}); diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index e98121a4..ea2b52a6 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -377,41 +377,7 @@ className = pBH.(sprintf('%s_className',fieldPrefix)); if flgClassify < 0 flgGold = 0; - end - case 'Cluster_ref' - STAGEofALIGNMENT = 'Cluster'; - ClusterGeomNAME = 'ClusterRefGeom'; - fieldPrefix = 'Ref'; - - classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); - classVector{2} = pBH.(sprintf('%s_classes_eve',fieldPrefix)); - - classCoeffs{1} = pBH.('Pca_coeffs'); - classCoeffs{2} = pBH.('Pca_coeffs'); - - samplingRate = pBH.('Cls_samplingRate'); - className = pBH.(sprintf('%s_className',fieldPrefix)); - if flgClassify < 0 - flgGold = 0; - end - case 'RefAlignment' - - fieldPrefix = 'REF'; - - refVector{1} = pBH.(sprintf('%s_references_odd','Ref')); - refVector{2} = pBH.(sprintf('%s_references_eve','Ref')); - - samplingRate = pBH.(sprintf('%s_samplingRate','Cls')); - % note the exchange of class to ref - - for iGold = 1:2 - % get just unique group names, which are subbed in for class id - [iGroup, groupIDX, ~] = unique(refVector{iGold}(3,:)); - classVector{iGold} = [ iGroup ; refVector{iGold}(2, groupIDX) ]; - end - classVector{1} - classVector{2} - className = pBH.(sprintf('%s_className','Ref')); + end case 'SnrEstimate' @@ -568,12 +534,7 @@ subTomoMeta.(cycleRead).('KmsSampling') = samplingRate; doNotTrim = true; - case 'RefAlignment' - - - geometry = subTomoMeta.(cycleRead).('RefAlignment') - subTomoMeta.(cycleRead).('Cls_Sampling') = samplingRate; - doNotTrim = true; + case 'SnrEstimate' flgEstSNR = 1; if ( CYCLE ) @@ -1928,13 +1889,11 @@ gpuDevice(gpuList(iGPU)); end -% The final condition is setting up RefAlignment which with classification -% would have been averaging with 'Cluster' == Stageof -if strcmpi(STAGEofALIGNMENT, 'RawAlignment') || ... - strcmpi(STAGEofALIGNMENT, 'RefAlignment') + +if strcmpi(STAGEofALIGNMENT, 'RawAlignment') BH_fscGold_class(PARAMETER_FILE, num2str(CYCLE), STAGEofALIGNMENT); -elseif strcmpi(STAGEofALIGNMENT, 'Alignment') - BH_fscGold_class(PARAMETER_FILE, num2str(CYCLE), 'RawAlignment'); +else + error('This block should not be reached'); end @@ -1987,8 +1946,7 @@ % This is slow ass when using cones and class averages and wouldn't be too % hard to put into parallel. Do that once the next manuscript is finished. -% % % % if (flgMultiRefAlignment && ~flgClassify && ~strcmpi(STAGEofALIGNMENT, 'RefAlignment')) - if (~flgMultiRefAlignment && ~flgClassify && ~strcmpi(STAGEofALIGNMENT, 'RefAlignment')) + if (~flgMultiRefAlignment && ~flgClassify ) nClassesReWgt = 1; else From 6b4671b594723ad22df77ef7134643930f4226b5 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 08:35:15 -0500 Subject: [PATCH 006/151] Replace all pBH parameter structs with emc in prep for moving all optional parameters into the parser function --- alignment/BH_alignRaw3d_v2.m | 92 ++++++++++----------- alignment/BH_runAutoAlign.m | 28 +++---- alignment/BH_templateSearch3d.m | 44 +++++----- alignment/BH_templateSearch3d_2.m | 54 ++++++------- ctf/BH_ctf_Correct.m | 8 +- ctf/BH_ctf_Correct3d.m | 46 +++++------ ctf/BH_ctf_Estimate.m | 62 +++++++-------- ctf/BH_ctf_Refine2.m | 20 ++--- ctf/BH_ctf_Updatefft.m | 26 +++--- logicals/BH_multi_maskCheck.m | 12 +-- masking/BH_weightMask_dpRUN.m | 6 +- metaData/BH_geometryAnalysis.m | 32 ++++---- metaData/BH_geometryInitialize.m | 18 ++--- metaData/BH_parseParameterFile.m | 13 ++- metaData/BH_removeDuplicates.m | 18 ++--- metaData/BH_skipClassAlignment.m | 12 +-- metaData/EMC_parseParameterFile.py | 30 ------- metaData/EMC_str2double.py | 21 ----- metaData/EMC_str2double_test.py | 21 ----- statistics/BH_clusterPub.m | 46 +++++------ statistics/BH_fscGold_class.m | 46 +++++------ statistics/BH_pcaPub.m | 74 ++++++++--------- synthetic/BH_synthetic_mapBack.m | 124 ++++++++++++++--------------- synthetic/BH_to_cisTEM_mapBack.m | 38 ++++----- testScripts/emClarity.m | 44 +++++----- transformations/BH_average3d.m | 119 +++++++++++++-------------- 26 files changed, 491 insertions(+), 563 deletions(-) delete mode 100644 metaData/EMC_parseParameterFile.py delete mode 100644 metaData/EMC_str2double.py delete mode 100644 metaData/EMC_str2double_test.py diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 085a3a76..064ab8bc 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -63,32 +63,32 @@ -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); cycleNumber = sprintf('cycle%0.3u', CYCLE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; reconScaling = 1; try - nPeaks = pBH.('nPeaks'); + nPeaks = emc.('nPeaks'); catch nPeaks = 1; end try - track_stats = pBH.('track_stats'); + track_stats = emc.('track_stats'); catch track_stats = false; end try - flgCutOutVolumes=pBH.('flgCutOutVolumes') + flgCutOutVolumes=emc.('flgCutOutVolumes') catch flgCutOutVolumes=0 end try - tmpVal = pBH.('whitenPS'); + tmpVal = emc.('whitenPS'); if (numel(tmpVal) == 3) wiener_constant = tmpVal(3); else @@ -106,25 +106,25 @@ end try - use_v2_SF3D = pBH.('use_v2_SF3D') + use_v2_SF3D = emc.('use_v2_SF3D') catch use_v2_SF3D = true; end try - symmetry_op = pBH.('symmetry'); + symmetry_op = emc.('symmetry'); catch error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); end try - use_new_grid_search = pBH.('use_new_grid_search'); + use_new_grid_search = emc.('use_new_grid_search'); catch use_new_grid_search = true; end try - force_no_symmetry = pBH.('force_no_symmetry'); + force_no_symmetry = emc.('force_no_symmetry'); catch force_no_symmetry = false; end @@ -136,18 +136,18 @@ maxGoldStandard = subTomoMeta.('maxGoldStandard'); -nGPUs = pBH.('nGPUs') +nGPUs = emc.('nGPUs') -flgClassify= pBH.('flgClassify'); +flgClassify= emc.('flgClassify'); try - flgMultiRefAlignment=pBH.('flgMultiRefAlignment'); + flgMultiRefAlignment=emc.('flgMultiRefAlignment'); catch flgMultiRefAlignment = 0; end try - updateClassByBestReferenceScore = pBH.('updateClassByBestReferenceScore'); + updateClassByBestReferenceScore = emc.('updateClassByBestReferenceScore'); catch updateClassByBestReferenceScore = false; end @@ -156,39 +156,39 @@ end try - flgCenterRefCOM = pBH.('flgCenterRefCOM'); + flgCenterRefCOM = emc.('flgCenterRefCOM'); catch flgCenterRefCOM = 1; end % FIXME: unused, fix experimental options option try - flgSymmetrizeSubTomos = pBH.('flgSymmetrizeSubTomos'); + flgSymmetrizeSubTomos = emc.('flgSymmetrizeSubTomos'); catch flgSymmetrizeSubTomos = 0; end -flgRaw_shapeMask = 0;%= pBH.('experimentalOpts')(3) -samplingRate = pBH.('Ali_samplingRate'); +flgRaw_shapeMask = 0;%= emc.('experimentalOpts')(3) +samplingRate = emc.('Ali_samplingRate'); -pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; -if pBH.('SuperResolution') +pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; +if emc.('SuperResolution') pixelSize = pixelSize * 2; end -flgPrecision = 'single'; %pBH.('flgPrecision'); -angleSearch = pBH.('Raw_angleSearch'); -peakSearch = (pBH.('particleRadius')./pixelSize); +flgPrecision = 'single'; %emc.('flgPrecision'); +angleSearch = emc.('Raw_angleSearch'); +peakSearch = (emc.('particleRadius')./pixelSize); peakCOM = [1,1,1].*3; -className = pBH.('Raw_className'); +className = emc.('Raw_className'); try - loadTomo = pBH.('loadTomo') + loadTomo = emc.('loadTomo') catch loadTomo = 0; end try - eraseMaskType = pBH.('Peak_mType'); - eraseMaskRadius = pBH.('Peak_mRadius')./pixelSize; + eraseMaskType = emc.('Peak_mType'); + eraseMaskRadius = emc.('Peak_mRadius')./pixelSize; fprintf('Further restricting peak search to radius %f %f %f\n',... eraseMaskRadius); eraseMask = 1; @@ -201,7 +201,7 @@ % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search try - doHelical = pBH.('doHelical'); + doHelical = emc.('doHelical'); catch doHelical = 0; end @@ -212,7 +212,7 @@ rotConvention try - bFactor = pBH.('Fsc_bfactor'); + bFactor = emc.('Fsc_bfactor'); catch bFactor = 0; end @@ -222,40 +222,40 @@ end try - scaleCalcSize = pBH.('scaleCalcSize'); + scaleCalcSize = emc.('scaleCalcSize'); catch scaleCalcSize = 1.5; end % % % % if (flgClassify || flgMultiRefAlignment) if (flgClassify) - refName = pBH.('Ref_className'); + refName = emc.('Ref_className'); else - refName = pBH.('Raw_className'); + refName = emc.('Raw_className'); end -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); +outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); -classVector{1} = pBH.('Raw_classes_odd')(1,:); +classVector{1} = emc.('Raw_classes_odd')(1,:); -classVector{2} = pBH.('Raw_classes_eve')(1,:); +classVector{2} = emc.('Raw_classes_eve')(1,:); % % % % if (flgClassify || flgMultiRefAlignment) if (flgClassify) geometry = subTomoMeta.(cycleNumber).ClassAlignment; - refVectorFull{1}= [pBH.('Ref_references_odd');1] - refVectorFull{2}= [pBH.('Ref_references_eve');1] + refVectorFull{1}= [emc.('Ref_references_odd');1] + refVectorFull{2}= [emc.('Ref_references_eve');1] elseif (flgMultiRefAlignment) geometry = subTomoMeta.(cycleNumber).ClusterRefGeom; - refVectorFull{1}= [pBH.('Raw_classes_odd');classVector{1} ] - refVectorFull{2}= [pBH.('Raw_classes_eve');classVector{2} ] + refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ] + refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ] else geometry = subTomoMeta.(cycleNumber).Avg_geometry; - refVectorFull{1} = [pBH.('Raw_classes_odd');1]; - refVectorFull{2} = [pBH.('Raw_classes_eve');1]; + refVectorFull{1} = [emc.('Raw_classes_odd');1]; + refVectorFull{2} = [emc.('Raw_classes_eve');1]; end @@ -311,14 +311,14 @@ [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Ali', pixelSize) + BH_multi_maskCheck(emc, 'Ali', pixelSize) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) try - flgLimitToOneProcess = pBH.('flgLimitToOneProcess'); + flgLimitToOneProcess = emc.('flgLimitToOneProcess'); catch flgLimitToOneProcess = 0; end @@ -331,7 +331,7 @@ elseif (flgLimitToOneProcess) limitToOne = flgLimitToOneProcess; else - limitToOne = pBH.('nCpuCores'); + limitToOne = emc.('nCpuCores'); end [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); @@ -684,7 +684,7 @@ inPlaneSearch = gridSearch.parameter_map.psi try - symmetry_constrained_search = pBH.('symmetry_constrained_search'); + symmetry_constrained_search = emc.('symmetry_constrained_search'); fprintf('Using symmetry constrained search\n'); catch symmetry_constrained_search = false; @@ -1953,7 +1953,7 @@ clear bestAngles rawAlign subTomoMeta = masterTM; - save(pBH.('subTomoMeta'), 'subTomoMeta'); + save(emc.('subTomoMeta'), 'subTomoMeta'); diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index 5577d546..78a75adc 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -3,82 +3,82 @@ % sadf % TODO add options for experimenting. -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); skip_tilts = 0; if nargin > 6 skip_tilts = EMC_str2double(varargin{1}); end -pixelSize = pBH.('PIXEL_SIZE').*10^10; +pixelSize = emc.('PIXEL_SIZE').*10^10; imgRotation = EMC_str2double(imgRotation); try - RESOLUTION_CUTOFF = pBH.('autoAli_max_resolution'); + RESOLUTION_CUTOFF = emc.('autoAli_max_resolution'); catch RESOLUTION_CUTOFF=18; end % Min and max sampling rate in Ang/Pix (for patch tracking) try - MIN_SAMPLING_RATE = pBH.('autoAli_min_sampling_rate'); + MIN_SAMPLING_RATE = emc.('autoAli_min_sampling_rate'); catch MIN_SAMPLING_RATE = 10.0; end try - MAX_SAMPLING_RATE = pBH.('autoAli_max_sampling_rate'); + MAX_SAMPLING_RATE = emc.('autoAli_max_sampling_rate'); catch MAX_SAMPLING_RATE = 4.0; end try - PATCH_SIZE_FACTOR = pBH.('autoAli_patch_size_factor'); + PATCH_SIZE_FACTOR = emc.('autoAli_patch_size_factor'); catch PATCH_SIZE_FACTOR = 4; end % Check this first to allow only patch tracking even if there are beads try - REFINE_ON_BEADS = pBH.('autoAli_refine_on_beads'); + REFINE_ON_BEADS = emc.('autoAli_refine_on_beads'); catch REFINE_ON_BEADS = false; end % Check this first to allow only patch tracking even if there are beads try - BORDER_SIZE_PIXELS = pBH.('autoAli_patch_tracking_border'); + BORDER_SIZE_PIXELS = emc.('autoAli_patch_tracking_border'); catch BORDER_SIZE_PIXELS = 64; end % Check this first to allow only patch tracking even if there are beads try - N_ITERS_NO_ROT = pBH.('autoAli_n_iters_no_rotation'); + N_ITERS_NO_ROT = emc.('autoAli_n_iters_no_rotation'); catch N_ITERS_NO_ROT = 3; end try - PATCH_OVERLAP = pBH.('autoAli_patch_overlap'); + PATCH_OVERLAP = emc.('autoAli_patch_overlap'); catch PATCH_OVERLAP = 0.5; end try - ITERATIONS_PER_BIN = pBH.('autoAli_iterations_per_bin'); + ITERATIONS_PER_BIN = emc.('autoAli_iterations_per_bin'); catch ITERATIONS_PER_BIN = 3; end % FIXME this should probably be specified in Ang try - FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(pBH.('autoAli_max_shift_in_angstroms')./pixelSize); + FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(emc.('autoAli_max_shift_in_angstroms')./pixelSize); catch FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(40 ./ pixelSize); end try - DIVIDE_SHIFT_LIMIT_BY = pBH.('autoAli_max_shift_factor'); + DIVIDE_SHIFT_LIMIT_BY = emc.('autoAli_max_shift_factor'); catch DIVIDE_SHIFT_LIMIT_BY = 1; % int(max_shift / (iter^DIVI...)) + 1 @@ -86,7 +86,7 @@ % Now get the bead diameter, if it is zeros override the default to refine % on beads after patch tracking. -beadDiameter = pBH.('beadDiameter') * 10^10; +beadDiameter = emc.('beadDiameter') * 10^10; if beadDiameter == 0 REFINE_ON_BEADS = false; end diff --git a/alignment/BH_templateSearch3d.m b/alignment/BH_templateSearch3d.m index e64d1cda..e6d0bbcf 100755 --- a/alignment/BH_templateSearch3d.m +++ b/alignment/BH_templateSearch3d.m @@ -35,9 +35,9 @@ SYMMETRY = EMC_str2double(SYMMETRY); startTime = clock ; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR % clear subTomoMeta % Make sure we get a CTF corrected stack @@ -46,22 +46,22 @@ mapBackIter = 0; shouldBeCTF = -1 end -samplingRate = pBH.('Tmp_samplingRate'); +samplingRate = emc.('Tmp_samplingRate'); try - tmpDecoy = pBH.('templateDecoy') + tmpDecoy = emc.('templateDecoy') catch tmpDecoy = 0 end try - scale_mip = pBH.('scale_mip'); + scale_mip = emc.('scale_mip'); catch scale_mip = false; end try - max_tries = pBH.('max_peaks'); + max_tries = emc.('max_peaks'); catch max_tries = 10000; end @@ -69,30 +69,30 @@ if ( cmdLineThresh ) peakThreshold = cmdLineThresh; fprintf('\nOverride peakThreshold from paramfile (%d) with cmd line arg (%d)\n\n',... - cmdLineThresh, pBH.('Tmp_threshold')); + cmdLineThresh, emc.('Tmp_threshold')); else - peakThreshold = pBH.('Tmp_threshold'); + peakThreshold = emc.('Tmp_threshold'); end -latticeRadius = pBH.('particleRadius'); +latticeRadius = emc.('particleRadius'); try - targetSize = pBH.('Tmp_targetSize') + targetSize = emc.('Tmp_targetSize') catch targetSize = [512,512,512]; end -angleSearch = pBH.('Tmp_angleSearch'); +angleSearch = emc.('Tmp_angleSearch'); statsRadius = 1; convTMPNAME = sprintf('convmap_wedgeType_%d_bin%d',wedgeType,samplingRate) try - eraseMaskType = pBH.('Peak_mType'); + eraseMaskType = emc.('Peak_mType'); catch eraseMaskType = 'sphere'; end try - eraseMaskRadius = pBH.('Peak_mRadius'); + eraseMaskRadius = emc.('Peak_mRadius'); catch eraseMaskRadius = 0.75.*latticeRadius; end @@ -102,13 +102,13 @@ reconScaling = 1; try - nPeaks = pBH.('nPeaks'); + nPeaks = emc.('nPeaks'); catch nPeaks = 1; end -pixelSizeFULL = pBH.('PIXEL_SIZE').*10^10; -if pBH.('SuperResolution') +pixelSizeFULL = emc.('PIXEL_SIZE').*10^10; +if emc.('SuperResolution') pixelSizeFULL = pixelSizeFULL * 2; end @@ -118,7 +118,7 @@ try - wantedCut = pBH.('lowResCut'); + wantedCut = emc.('lowResCut'); catch wantedCut = 28; end @@ -170,7 +170,7 @@ % The template will be padded later, trim for now to minimum so excess % iterations can be avoided. fprintf('size of provided template %d %d %d\n',size(template)); -trimTemp = BH_multi_padVal(size(template),ceil(2.*max(pBH.('Ali_mRadius')./pixelSizeFULL))); +trimTemp = BH_multi_padVal(size(template),ceil(2.*max(emc.('Ali_mRadius')./pixelSizeFULL))); template = BH_padZeros3d(template, trimTemp(1,:),trimTemp(2,:),'cpu','singleTaper'); SAVE_IMG(MRCImage(template),'template_trimmed.mrc'); clear trimTemp @@ -226,7 +226,7 @@ % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search try - doHelical = pBH.('doHelical'); + doHelical = emc.('doHelical'); catch doHelical = 0; end @@ -400,7 +400,7 @@ clear wedgeMask try - doMedFilt = pBH.('Tmp_medianFilter'); + doMedFilt = emc.('Tmp_medianFilter'); if ~ismember(doMedFilt,[3,5,7]) error('Tmp_medianFilter can only be 3,5, or 7'); else @@ -1294,8 +1294,8 @@ % [ subTomoMeta ] = BH_fscSplit( preFscSplit ); % subTomoMeta.('currentCycle') = 0; -% save(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); -% save(sprintf('./convmap/%s.mat~', pBH.('subTomoMeta')), 'subTomoMeta'); +% save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); +% save(sprintf('./convmap/%s.mat~', emc.('subTomoMeta')), 'subTomoMeta'); %save('test.pos','a','-ascii'); diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 4af18a8d..d68f75cc 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -35,7 +35,7 @@ startTime = clock ; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); % Currently hardcoded to always expect a tomogram constructed with ctf correction % using emClarity ctf3d paramN.m templateSearch @@ -44,7 +44,7 @@ mapBackIter = 0; else try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR catch % TODO: is there a better check on whether we are using ctf3d templateSearch vs ctf 3d? @@ -52,16 +52,16 @@ shouldBeCTF = false; end end - samplingRate = pBH.('Tmp_samplingRate'); + samplingRate = emc.('Tmp_samplingRate'); try - tmpDecoy = pBH.('templateDecoy') + tmpDecoy = emc.('templateDecoy') catch tmpDecoy = 0 end try - super_sample = pBH.('super_sample'); + super_sample = emc.('super_sample'); if (super_sample > 0) [~,v] = system('cat $IMOD_DIR/VERSION'); v = split(v,'.'); @@ -80,7 +80,7 @@ end try - tmpVal = pBH.('whitenPS'); + tmpVal = emc.('whitenPS'); if (numel(tmpVal) == 3) wiener_constant = tmpVal(3); else @@ -90,39 +90,39 @@ wiener_constant = 0.0; end - peakThreshold = pBH.('Tmp_threshold'); + peakThreshold = emc.('Tmp_threshold'); -latticeRadius = pBH.('particleRadius'); +latticeRadius = emc.('particleRadius'); try - targetSize = pBH.('Tmp_targetSize') + targetSize = emc.('Tmp_targetSize') catch targetSize = [512,512,512]; end -angleSearch = pBH.('Tmp_angleSearch'); +angleSearch = emc.('Tmp_angleSearch'); convTMPNAME = sprintf('convmap_wedgeType_%d_bin%d',wedgeType,samplingRate) try - use_new_grid_search = pBH.('use_new_grid_search'); + use_new_grid_search = emc.('use_new_grid_search'); catch use_new_grid_search = true; end try - symmetry = pBH.('symmetry'); + symmetry = emc.('symmetry'); catch error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); end try - eraseMaskType = pBH.('Peak_mType'); + eraseMaskType = emc.('Peak_mType'); catch eraseMaskType = 'sphere'; end try - eraseMaskRadius = pBH.('Peak_mRadius'); + eraseMaskRadius = emc.('Peak_mRadius'); catch eraseMaskRadius = 1.0.*latticeRadius; end @@ -132,27 +132,27 @@ reconScaling = 1; try - nPeaks = pBH.('nPeaks'); + nPeaks = emc.('nPeaks'); catch nPeaks = 1; end ignore_threshold = false; try - max_tries = pBH.('max_peaks'); + max_tries = emc.('max_peaks'); catch max_tries = 10000; end try - over_ride = pBH.('Override_threshold_and_return_N_peaks') + over_ride = emc.('Override_threshold_and_return_N_peaks') ignore_threshold = true; fprintf('Override_threshold_and_return_N_peaks set to true, returning exactly %d peaks\n', over_ride); peakThreshold = over_ride; end -pixelSizeFULL = pBH.('PIXEL_SIZE').*10^10; -if pBH.('SuperResolution') +pixelSizeFULL = emc.('PIXEL_SIZE').*10^10; +if emc.('SuperResolution') pixelSizeFULL = pixelSizeFULL * 2; end @@ -161,7 +161,7 @@ % For testing print_warning=false; try - wantedCut = pBH.('lowResCut'); + wantedCut = emc.('lowResCut'); fprintf('lowResCut is deprecated and will be removed in future versions.\n') fprintf('please switch to Tmp_bandpass\n\n'); bp_vals = [1e-3,600,wantedCut]; @@ -171,7 +171,7 @@ end try - bp_vals = pBH.('Tmp_bandpass'); + bp_vals = emc.('Tmp_bandpass'); if numel(bp_vals) ~= 3 error('Tmp_bandpass is [filter at zero freq, res high-pass cutoff, res low-pass cutoff]'); end @@ -185,7 +185,7 @@ end try - stats_diameter_fraction = pBH.('diameter_fraction_for_local_stats') + stats_diameter_fraction = emc.('diameter_fraction_for_local_stats') catch stats_diameter_fraction = 1 end @@ -195,7 +195,7 @@ mean_r_mask = 0; reference_mask = []; try - scale_mip = pBH.('scale_mip'); + scale_mip = emc.('scale_mip'); catch scale_mip = false; end @@ -273,7 +273,7 @@ % The template will be padded later, trim for now to minimum so excess % iterations can be avoided. fprintf('size of provided template %d %d %d\n',size(template)); -trimTemp = BH_multi_padVal(size(template),ceil(2.0.*max(pBH.('Ali_mRadius')./pixelSizeFULL))); +trimTemp = BH_multi_padVal(size(template),ceil(2.0.*max(emc.('Ali_mRadius')./pixelSizeFULL))); % template = BH_padZeros3d(template, trimTemp(1,:),trimTemp(2,:),'cpu','singleTaper'); % SAVE_IMG(MRCImage(template),'template_trimmed.mrc'); clear trimTemp @@ -322,7 +322,7 @@ % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search try - doHelical = pBH.('doHelical'); + doHelical = emc.('doHelical'); catch doHelical = 0; end @@ -496,7 +496,7 @@ nTomograms = prod(nIters); try - test_local = pBH.('test_local'); + test_local = emc.('test_local'); catch test_local = false; end @@ -518,7 +518,7 @@ try - doMedFilt = pBH.('Tmp_medianFilter'); + doMedFilt = emc.('Tmp_medianFilter'); if ~ismember(doMedFilt,[3,5,7]) error('Tmp_medianFilter can only be 3,5, or 7'); else diff --git a/ctf/BH_ctf_Correct.m b/ctf/BH_ctf_Correct.m index 6678cd35..d7bf27e3 100755 --- a/ctf/BH_ctf_Correct.m +++ b/ctf/BH_ctf_Correct.m @@ -10,10 +10,10 @@ % tilt information. % -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; catch mapBackIter = 0; @@ -27,7 +27,7 @@ ITER_LIST = {STACK_LIST}; else flgParallel = 1; - nGPUs = pBH.('nGPUs'); + nGPUs = emc.('nGPUs'); STACK_LIST_tmp = fieldnames(subTomoMeta.mapBackGeometry); STACK_LIST_tmp = STACK_LIST_tmp(~ismember(STACK_LIST_tmp,'tomoName')); @@ -45,7 +45,7 @@ end end -pixelSize = pBH.('PIXEL_SIZE'); +pixelSize = emc.('PIXEL_SIZE'); !mkdir -p ctfStacks try diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 347e5523..9db779b8 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -11,7 +11,7 @@ % Read in 2dCtf stacks to trouble shoot PosControl2d=0; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); % Apply a Wiener filter with this many zeros during Ctf multiplication global bh_global_turn_on_phase_plate @@ -24,7 +24,7 @@ tiltStart = 1; try - flgEraseBeads_aferCTF = pBH.('erase_beads_after_ctf'); + flgEraseBeads_aferCTF = emc.('erase_beads_after_ctf'); catch flgEraseBeads_aferCTF = false; % If false they SHOULD be erased in ctf estimate/update, but since the user could change parameter, include here. end @@ -32,7 +32,7 @@ % Test David's new super sampling in reconstruction. No check that this % version (currently 4.10.40) is properly sourced. try - super_sample = pBH.('super_sample'); + super_sample = emc.('super_sample'); if (super_sample > 0) [~,v] = system('cat $IMOD_DIR/VERSION'); v = split(v,'.'); @@ -52,7 +52,7 @@ try - expand_lines = pBH.('expand_lines'); + expand_lines = emc.('expand_lines'); if isempty(super_sample) || expand_lines == false expand_lines = ''; else @@ -107,7 +107,7 @@ try % -1, whiten before ctf, 1 whiten after - test both. - usr_flgWhitenPS = pBH.('whitenPS'); + usr_flgWhitenPS = emc.('whitenPS'); if (numel(usr_flgWhitenPS) == 3) flgWhitenPS = usr_flgWhitenPS; else @@ -123,7 +123,7 @@ try - applyExposureFilter = pBH.('applyExposureFilter') + applyExposureFilter = emc.('applyExposureFilter') catch applyExposureFilter = 1; end @@ -131,14 +131,14 @@ % This will be set false if the reconstruction is for template matching or % for tomoCPR try - useSurfaceFit = pBH.('useSurfaceFit') + useSurfaceFit = emc.('useSurfaceFit') catch useSurfaceFit = 1 end try % Not for normal use, pass the total dose less first frame to flip values. - invertDose = pBH.('invertDose') + invertDose = emc.('invertDose') catch invertDose = 0; end @@ -152,7 +152,7 @@ -tmpCache= pBH.('fastScratchDisk'); +tmpCache= emc.('fastScratchDisk'); if strcmpi(tmpCache, 'ram') if isempty(getenv('EMC_CACHE_MEM')) @@ -203,7 +203,7 @@ if (recWithoutMat) useSurfaceFit = false; if (loadSubTomoMeta) - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; masterTM = subTomoMeta; clear subTomoMeta CYCLE = masterTM.currentCycle; @@ -212,7 +212,7 @@ CYCLE = 0; end else - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; masterTM = subTomoMeta; clear subTomoMeta CYCLE = masterTM.currentCycle; @@ -233,13 +233,13 @@ try - flgDampenAliasedFrequencies = pBH.('flgDampenAliasedFrequencies') + flgDampenAliasedFrequencies = emc.('flgDampenAliasedFrequencies') catch flgDampenAliasedFrequencies = 0 end try - flg2dCTF = pBH.('flg2dCTF'); + flg2dCTF = emc.('flg2dCTF'); catch flg2dCTF = 0; end @@ -248,13 +248,13 @@ % Part of the experiment with template matching using higher res info, also % allow for a median filter post CTF correction, pre reconstruction to % further denoise prior to template matching. - flgMedianFilter = pBH.('ctfMedianFilter'); + flgMedianFilter = emc.('ctfMedianFilter'); catch flgMedianFilter = 0; end -% ctf3dDepth=pBH.('defocusErrorEst') +% ctf3dDepth=emc.('defocusErrorEst') %mean in case cones. @@ -263,7 +263,7 @@ samplingRate = reconstructionParameters(2); else if (loadSubTomoMeta) - samplingRate = pBH.('Ali_samplingRate'); + samplingRate = emc.('Ali_samplingRate'); % This number is used to roughly balance the trade off between % achievable resolution, and run time during reconstruction as % determined by the thickness of each 3d slab reconstructed. Given that @@ -275,9 +275,9 @@ end else % For template search - samplingRate = pBH.('Tmp_samplingRate'); + samplingRate = emc.('Tmp_samplingRate'); try - resTarget = pBH.('lowResCut'); + resTarget = emc.('lowResCut'); catch resTarget = 12; end @@ -287,7 +287,7 @@ end try - max_ctf3dDepth = pBH.('max_ctf3dDepth'); + max_ctf3dDepth = emc.('max_ctf3dDepth'); catch max_ctf3dDepth = 500*10^-9; end @@ -300,7 +300,7 @@ fprintf('Using a target resolution of %2.2f Angstroms\n',resTarget); -nGPUs = pBH.('nGPUs'); +nGPUs = emc.('nGPUs'); % Optionally specify gpu idxs if numel(nGPUs) == 1 gpuList = 1:nGPUs; @@ -309,17 +309,17 @@ nGPUs = length(gpuList); end -pixelSize = pBH.('PIXEL_SIZE').*10^10 .* samplingRate; +pixelSize = emc.('PIXEL_SIZE').*10^10 .* samplingRate; % if (recWithoutMat) % reconstructionParameters(1) = ')(i) =(1) ./ pixelSize; % end -if pBH.('SuperResolution') +if emc.('SuperResolution') pixelSize = pixelSize * 2; end -eraseRadius = ceil(1.5.*(pBH.('beadDiameter')./pBH.('PIXEL_SIZE').*0.5) / samplingRate); +eraseRadius = ceil(1.5.*(emc.('beadDiameter')./emc.('PIXEL_SIZE').*0.5) / samplingRate); nTomosPerTilt = 0; recGeom = 0; diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index d749d96f..3161afb3 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -25,7 +25,7 @@ collectionORDER = sprintf('fixedStacks/%s.order',STACK_BASENAME); end -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); gpuIDX = BH_multi_checkGPU(-1); gDev = gpuDevice(gpuIDX); @@ -57,7 +57,7 @@ end skipFitting = 0; try - PHASE_PLATE_SHIFT = pBH.('PHASE_PLATE_SHIFT').*pi + PHASE_PLATE_SHIFT = emc.('PHASE_PLATE_SHIFT').*pi catch PHASE_PLATE_SHIFT = [0,0] end @@ -67,14 +67,14 @@ flgStandardOrdeDoCalc = 1; try - flgCosineDose = pBH.('oneOverCosineDose'); - startingAngle = pBH.('startingAngle'); - startingDirection = pBH.('startingDirection'); - doseSymmetricIncrement = pBH.('doseSymmetricIncrement'); - doseAtMinTilt = pBH.('doseAtMinTilt'); + flgCosineDose = emc.('oneOverCosineDose'); + startingAngle = emc.('startingAngle'); + startingDirection = emc.('startingDirection'); + doseSymmetricIncrement = emc.('doseSymmetricIncrement'); + doseAtMinTilt = emc.('doseAtMinTilt'); flgOldDose = 0; - tltOrder = calc_dose_scheme(pBH,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT); + tltOrder = calc_dose_scheme(emc,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT); catch @@ -92,11 +92,11 @@ -PIXEL_SIZE = pBH.('PIXEL_SIZE'); -Cs = pBH.('Cs'); -VOLTAGE = pBH.('VOLTAGE'); -AMPCONT = pBH.('AMPCONT'); -SuperResolution = pBH.('SuperResolution'); +PIXEL_SIZE = emc.('PIXEL_SIZE'); +Cs = emc.('Cs'); +VOLTAGE = emc.('VOLTAGE'); +AMPCONT = emc.('AMPCONT'); +SuperResolution = emc.('SuperResolution'); if (SuperResolution) if SuperResolution == 1 @@ -141,7 +141,7 @@ Cs = 1e-6; end -CUM_e_DOSE = pBH.('CUM_e_DOSE'); +CUM_e_DOSE = emc.('CUM_e_DOSE'); % test astigmatism vals flgAstigmatism = 1; if (flgAstigmatism ~=1 && flgAstigmatism ~= 0) @@ -157,31 +157,31 @@ eraseSigma = 3; -eraseRadius = ceil(1.2.*(pBH.('beadDiameter')./PIXEL_SIZE.*0.5)); +eraseRadius = ceil(1.2.*(emc.('beadDiameter')./PIXEL_SIZE.*0.5)); flgImodErase = 0 % Assuming that the first CTF zero is always less than this value FIXED_FIRSTZERO = PIXEL_SIZE / (70*10^-10) ; -highCutoff = PIXEL_SIZE/pBH.('defCutOff'); +highCutoff = PIXEL_SIZE/emc.('defCutOff'); % I still use the def for underfocus < 0 as this places the origin at the % focal plan in the microscope rather than on the specimen. Which makes % more sense to me. -defEST = -1.*pBH.('defEstimate').*10^6 -defWIN = pBH.('defWindow').*10^6 +defEST = -1.*emc.('defEstimate').*10^6 +defWIN = emc.('defWindow').*10^6 tiltRange = [-1]; backGroundBuffer = 0.9985; try - deltaZTolerance = pBH.('deltaZTolerance'); + deltaZTolerance = emc.('deltaZTolerance'); catch deltaZTolerance = 100e-9; end try - zShift = abs(pBH.('zShift')); + zShift = abs(emc.('zShift')); catch zShift = 150e-9; end @@ -191,7 +191,7 @@ end try - maxNumberOfTiles = pBH.('ctfMaxNumberOfTiles'); + maxNumberOfTiles = emc.('ctfMaxNumberOfTiles'); catch maxNumberOfTiles = 10000; @@ -205,7 +205,7 @@ % Tile size & overlap try - tileSize = pBH.('ctfTileSize'); + tileSize = emc.('ctfTileSize'); catch tileSize = floor(680e-10 / PIXEL_SIZE); end @@ -219,7 +219,7 @@ % Size to padTile to should be even, large, and preferably a power of 2 try - paddedSize = pBH.('paddedSize'); + paddedSize = emc.('paddedSize'); catch paddedSize = 768; end @@ -369,7 +369,7 @@ outputStackName = sprintf('aliStacks/%s%s',stackNameOUT,extension) try - erase_beads_after_ctf = pBH.('erase_beads_after_ctf'); + erase_beads_after_ctf = emc.('erase_beads_after_ctf'); catch erase_beads_after_ctf = false; end @@ -546,7 +546,7 @@ % Redefining things down hear is a stupid thing to do. Fix this if you % keep the optino for cropping. FIXED_FIRSTZERO = pixelOUT / 70 ; - highCutoff = (pixelOUT*10^-10)/pBH.('defCutOff'); + highCutoff = (pixelOUT*10^-10)/emc.('defCutOff'); else @@ -1337,13 +1337,13 @@ end -function [ tltOrder ] = calc_dose_scheme(pBH,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT) +function [ tltOrder ] = calc_dose_scheme(emc,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT) - flgCosineDose = pBH.('oneOverCosineDose'); - startingAngle = pBH.('startingAngle'); - startingDirection = pBH.('startingDirection'); - doseSymmetricIncrement = pBH.('doseSymmetricIncrement'); - doseAtMinTilt = pBH.('doseAtMinTilt'); + flgCosineDose = emc.('oneOverCosineDose'); + startingAngle = emc.('startingAngle'); + startingDirection = emc.('startingDirection'); + doseSymmetricIncrement = emc.('doseSymmetricIncrement'); + doseAtMinTilt = emc.('doseAtMinTilt'); nPrjs = length(rawTLT); tltOrder = zeros(nPrjs,5); diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index 7fecf1f5..f530312d 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -4,16 +4,16 @@ % mean % Load in the tomo and tilt info -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; clear subTomoMeta catch mapBackIter = 0; end try - testNoRefine = pBH.('force_no_defocus_stretch'); + testNoRefine = emc.('force_no_defocus_stretch'); catch testNoRefine = false; end @@ -26,14 +26,14 @@ % improve the thon rings on tilted data. Zero will produce the "normal" % process, 1 will use the same area as the min tilt, try - fraction_of_extra_tilt_data = pBH.('fraction_of_extra_tilt_data') + fraction_of_extra_tilt_data = emc.('fraction_of_extra_tilt_data') catch fraction_of_extra_tilt_data = 0.25 end % set the search ranges - should change ctf_est to save the parameters used so % this can be loaded automatically. -nWorkers = min(pBH.('nCpuCores'),7*pBH.('nGPUs')); +nWorkers = min(emc.('nCpuCores'),7*emc.('nGPUs')); nWorkers = BH_multi_parallelWorkers(nWorkers) @@ -50,12 +50,12 @@ pathName = '.'; end -PIXEL_SIZE = pBH.('PIXEL_SIZE'); +PIXEL_SIZE = emc.('PIXEL_SIZE'); -Cs = pBH.('Cs'); -VOLTAGE = pBH.('VOLTAGE'); -AMPCONT = pBH.('AMPCONT'); +Cs = emc.('Cs'); +VOLTAGE = emc.('VOLTAGE'); +AMPCONT = emc.('AMPCONT'); ctfParams = [PIXEL_SIZE*10^10,VOLTAGE./1000,Cs.*1000,AMPCONT]; @@ -83,7 +83,7 @@ % Size to padTile to should be even, large, and preferably a power of 2 try - paddedSize = pBH.('paddedSize'); + paddedSize = emc.('paddedSize'); catch paddedSize = 768; end diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index b13b6ad3..b4a0f280 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -3,30 +3,30 @@ global bh_global_do_2d_fourier_interp; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); flgSkipUpdate = 0; % To avoid accidently masking any failures in subsequent update, clean out % all stacks and reconstructions from the local cache. try - eucentric_minTilt = pBH.('eucentric_minTilt'); + eucentric_minTilt = emc.('eucentric_minTilt'); catch eucentric_minTilt = 15; end try - flgShiftEucentric = pBH.('eucentric_fit'); + flgShiftEucentric = emc.('eucentric_fit'); catch flgShiftEucentric = 0; end try % Should be negative, but to test. - defShiftSign = pBH.('testFlipSign'); + defShiftSign = emc.('testFlipSign'); catch defShiftSign = -1; end try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; catch mapBackIter = 0; @@ -45,7 +45,7 @@ PARAMETER_FILE,subTomoMeta.currentCycle, ... subTomoMeta.currentCycle); % fprintf('%s/n',updateCMD); - nGPUs = pBH.('nGPUs'); + nGPUs = emc.('nGPUs'); ITER_LIST = cell(nGPUs,1); [STACK_LIST, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); @@ -61,7 +61,7 @@ end % BH_geometryAnalysis(sprintf('%s',PARAMETER_FILE),sprintf('%d',subTomoMeta.currentCycle),'TiltAlignment','UpdateTilts',sprintf('[%d,0,0]',subTomoMeta.currentCycle),'STD') try - conserveDiskSpace = pBH.('conserveDiskSpace'); + conserveDiskSpace = emc.('conserveDiskSpace'); catch conserveDiskSpace = 0; end @@ -161,8 +161,8 @@ eraseStack = sprintf('rm cache/%s_*.fixed',STACK_PRFX); eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); % Converte bead diameter to pixels and add a little to be safe. - PIXEL_SIZE = pBH.('PIXEL_SIZE'); - SuperResolution = pBH.('SuperResolution'); + PIXEL_SIZE = emc.('PIXEL_SIZE'); + SuperResolution = emc.('SuperResolution'); % Don't apply any fourier cropping of super-res data if only updating, % as it would already be done. @@ -176,9 +176,9 @@ PIXEL_SIZE = 2.* PIXEL_SIZE; end - eraseSigma = 3;%pBH.('beadSigma'); + eraseSigma = 3;%emc.('beadSigma'); - eraseRadius = ceil(1.2.*(pBH.('beadDiameter')./PIXEL_SIZE.*0.5)); + eraseRadius = ceil(1.2.*(emc.('beadDiameter')./PIXEL_SIZE.*0.5)); flgImodErase = 0 % FIXME, this should be stored from previous mask calc and accessed there. @@ -298,7 +298,7 @@ oldStackName = sprintf('%s/%s%s',outputDirectory,PRJ_OLD,INPUT_CELL{iStack,5}); try - erase_beads_after_ctf = pBH.('erase_beads_after_ctf'); + erase_beads_after_ctf = emc.('erase_beads_after_ctf'); catch erase_beads_after_ctf = false; end @@ -641,7 +641,7 @@ end end end - save(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); end if ( flgParallel ) diff --git a/logicals/BH_multi_maskCheck.m b/logicals/BH_multi_maskCheck.m index 7e10bf9d..5175ded5 100755 --- a/logicals/BH_multi_maskCheck.m +++ b/logicals/BH_multi_maskCheck.m @@ -1,21 +1,21 @@ -function [ maskType, maskSize, maskRadius, maskCenter ] = BH_multi_maskCheck( pBH, PREFIX, pixelSize, varargin ) +function [ maskType, maskSize, maskRadius, maskCenter ] = BH_multi_maskCheck( emc, PREFIX, pixelSize, varargin ) %Check that any masked sub-regions are in bounds with proper apodization given %the offset. % if nargin > 3 && strcmpi(varargin{1},'FSC') maskCenter = [0,0,0]; - maskRadius = round(pBH.('particleRadius') ./ pixelSize); + maskRadius = round(emc.('particleRadius') ./ pixelSize); else - maskCenter = pBH.(sprintf('%s_mCenter',PREFIX)) ./ pixelSize; - maskRadius = round(pBH.(sprintf('%s_mRadius',PREFIX)) ./ pixelSize); + maskCenter = emc.(sprintf('%s_mCenter',PREFIX)) ./ pixelSize; + maskRadius = round(emc.(sprintf('%s_mRadius',PREFIX)) ./ pixelSize); end -maskType = pBH.(sprintf('%s_mType',PREFIX)); +maskType = emc.(sprintf('%s_mType',PREFIX)); % Although the minimum roll-off is set for 6 pixels on each edge, use 14 rather % than 12 to make sure a shift in the origin doesn't preculde the volume from % analysis. -maskSize = 2.*floor(pBH.('particleRadius') ./ pixelSize)+14; +maskSize = 2.*floor(emc.('particleRadius') ./ pixelSize)+14; % center can be floats. maskSize = max([maskSize;2.*maskRadius+14],[],1); diff --git a/masking/BH_weightMask_dpRUN.m b/masking/BH_weightMask_dpRUN.m index c573fac2..839bd0b3 100644 --- a/masking/BH_weightMask_dpRUN.m +++ b/masking/BH_weightMask_dpRUN.m @@ -26,10 +26,10 @@ gpuIDX = BH_multi_checkGPU(useGPU); gDev = gpuDevice(gpuIDX); -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); cycleNumber = sprintf('cycle%0.3d',CYCLE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); geom = subTomoMeta.(cycleNumber).Avg_geometry; @@ -39,7 +39,7 @@ aVal = subTomoMeta.(cycleNumber).fitFSC.ResampleRaw1; wgtSize = subTomoMeta.(cycleNumber).class_0_Locations_REF_ODD_Wgt{2}{1}(2:2:6); -samplingRate = pBH.('Ali_samplingRate'); +samplingRate = emc.('Ali_samplingRate'); tomoList = fieldnames(geom); if ( tiltStart ) diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index a71dce2b..46e6f01c 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -54,14 +54,14 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... cycleNumber = sprintf('cycle%0.3u', CYCLE); -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); try - conserveDiskSpace = pBH.('conserveDiskSpace'); + conserveDiskSpace = emc.('conserveDiskSpace'); catch conserveDiskSpace = 0; end try - percentCut = pBH.('removeBottomPercent'); + percentCut = emc.('removeBottomPercent'); catch percentCut = 0.0; end @@ -99,16 +99,16 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... end -samplingRate = pBH.(sprintf('%s_samplingRate','Ali')); +samplingRate = emc.(sprintf('%s_samplingRate','Ali')); -className = pBH.(sprintf('%s_className',fieldPrefix)); +className = emc.(sprintf('%s_className',fieldPrefix)); -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); +outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); %try - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); switch STAGEofALIGNMENT case 'TiltAlignment' geometry = subTomoMeta.tiltGeometry; @@ -117,7 +117,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... if (undoOP) subTomoMeta.(cycleNumber).RawAlign = ... subTomoMeta.(cycleNumber).(sprintf('Pre_%s_RawAlign', OPERATION)); - save(pBH.('subTomoMeta'), 'subTomoMeta'); + save(emc.('subTomoMeta'), 'subTomoMeta'); error('No Error, just exiting.\n') else geometry = subTomoMeta.(cycleNumber).RawAlign; @@ -126,18 +126,18 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... case 'Cluster' try - classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); + classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); catch - classVector{1} = pBH.(sprintf('%s_classes',fieldPrefix)); + classVector{1} = emc.(sprintf('%s_classes',fieldPrefix)); end - classVector{2} = pBH.(sprintf('%s_classes_eve',fieldPrefix)); + classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); try - classCoeffs{1} = pBH.('Pca_coeffs_odd'); - classCoeffs{2} = pBH.('Pca_coeffs_eve'); + classCoeffs{1} = emc.('Pca_coeffs_odd'); + classCoeffs{2} = emc.('Pca_coeffs_eve'); catch - classCoeffs{1} = pBH.('Pca_coeffs'); + classCoeffs{1} = emc.('Pca_coeffs'); end cN = sprintf('%s_%d_%d_nClass_%d_%s',outputPrefix,classCoeffs{halfNUM(1)}(1,1), ... @@ -149,7 +149,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... if (undoOP) subTomoMeta.(cycleNumber).ClusterResults.(cN) = ... subTomoMeta.(cycleNumber).(sprintf('Pre_%s_ClusterResults', OPERATION)).(cN); - save(pBH.('subTomoMeta'), 'subTomoMeta'); + save(emc.('subTomoMeta'), 'subTomoMeta'); error('No Error, just exiting.\n') else @@ -543,7 +543,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... end subTomoMeta = masterTM; - save(pBH.('subTomoMeta'), 'subTomoMeta'); + save(emc.('subTomoMeta'), 'subTomoMeta'); end diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index d83cb891..3d21efbc 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -22,7 +22,7 @@ angleSgn=1; convention='Bah'; direction='fwd'; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); if nargin > 1 if length(varargin) == 1 mapBackIter = EMC_str2double(varargin{1}); @@ -87,20 +87,20 @@ end end -nGPUs = pBH.('nGPUs'); +nGPUs = emc.('nGPUs'); % This will have to do until a better approach based on PSF of positions % in projection space linked together can be used to define groups that % don't have co-mingled resolution. When splitOnTOmos, always run in serial, so % that we have the best chance of distributing the defocus variateion/tomoqualtiy. -splitOnTomos = pBH.('fscGoldSplitOnTomos'); +splitOnTomos = emc.('fscGoldSplitOnTomos'); if (splitOnTomos) nGPUs = 1; fprintf('override nGPUs to just 1 for initial step to evenly split crowded tomos because fscGoldSplitOnTomos is true') end -nOrientations=1;%nOrientations = pBH.('pseudoMLnumber'); +nOrientations=1;%nOrientations = emc.('pseudoMLnumber'); nCTFgroups = 9; try - nPeaks = pBH.('nPeaks'); + nPeaks = emc.('nPeaks'); catch nPeaks = 1; end @@ -113,13 +113,13 @@ % 40 may be too conservative, especially for low defocus tomos try - lowResCut = pBH.('lowResCut'); + lowResCut = emc.('lowResCut'); catch lowResCut = 40; end -maxGoldStandard = lowResCut;%pBH.('Tmp_bandpassFilter')(3); -dupSampling = pBH.('Tmp_samplingRate') +maxGoldStandard = lowResCut;%emc.('Tmp_bandpassFilter')(3); +dupSampling = emc.('Tmp_samplingRate') dupRadius = 2; dupTolerance = (2.*dupRadius)+1; dupMask = zeros(dupTolerance, dupTolerance, dupTolerance, 'single'); @@ -547,7 +547,7 @@ subTomoMeta.('currentTomoCPR') = mapBackIter; subTomoMeta.('currentResForDefocusError') = lowResCut; subTomoMeta.('maxGoldStandard') = maxGoldStandard; -save(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); if (bh_global_do_profile) diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 7e3fcc9a..80f9f6a8 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -1,4 +1,4 @@ -function [ pStruct ] = BH_parseParameterFile( PARAMETER_FILE ) +function [ emc ] = BH_parseParameterFile( PARAMETER_FILE ) %Parse a parameter file & check for valid parameters. % experimental @@ -18,7 +18,7 @@ end clear p -pStruct = struct(); +emc = struct(); % Check that all paramters are name: value pairs stringValues = {'subTomoMeta'; ... 'Ali_mType';'Cls_mType';'Cls_mType';'Raw_mType';'Fsc_mType'; ... @@ -33,13 +33,18 @@ else if any(strcmp(stringValues, pNameVal{1})) - pStruct.(pNameVal{1}) = pNameVal{2}; + emc.(pNameVal{1}) = pNameVal{2}; else - pStruct.(pNameVal{1}) = EMC_str2double(pNameVal{2}); + emc.(pNameVal{1}) = EMC_str2double(pNameVal{2}); end end end +% Now check for optional parameters + +if ~isfield(emc, 'nPeaks') + emc.('nPeaks') = 1; +end end diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index 3dd59f72..d99ff053 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -21,15 +21,15 @@ CYCLE = EMC_str2double(CYCLE); cycleNumber = sprintf('cycle%0.3u', CYCLE); -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); -dupSampling = ceil(10e-10 / pBH.('PIXEL_SIZE')); +dupSampling = ceil(10e-10 / emc.('PIXEL_SIZE')); -pixelSize = pBH.('PIXEL_SIZE').*dupSampling.*10^10; -if pBH.('SuperResolution') +pixelSize = emc.('PIXEL_SIZE').*dupSampling.*10^10; +if emc.('SuperResolution') pixelSize = pixelSize * 2; end -latticeRadius = pBH.('particleRadius'); +latticeRadius = emc.('particleRadius'); dupRadius = max(1,floor(0.2*min(latticeRadius)/pixelSize)); dupTolerance = (2.*dupRadius)+1; @@ -42,13 +42,13 @@ end try - nPeaks = pBH.('nPeaks'); + nPeaks = emc.('nPeaks'); catch nPeaks = 1; end % Backup the current geometry - system(sprintf('cp %s.mat preDupRemoval_%s.mat',pBH.('subTomoMeta'),pBH.('subTomoMeta'))); - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + system(sprintf('cp %s.mat preDupRemoval_%s.mat',emc.('subTomoMeta'),emc.('subTomoMeta'))); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); geometry = subTomoMeta.(cycleNumber).RawAlign; % % % mapExt = subTomoMeta.mapExt; % % % mapPath= subTomoMeta.mapPath; @@ -167,7 +167,7 @@ subTomoMeta = masterTM; subTomoMeta.(cycleNumber).RawAlign = geometry; -save(pBH.('subTomoMeta'), 'subTomoMeta'); +save(emc.('subTomoMeta'), 'subTomoMeta'); end diff --git a/metaData/BH_skipClassAlignment.m b/metaData/BH_skipClassAlignment.m index 0e7d4425..75c4f20b 100755 --- a/metaData/BH_skipClassAlignment.m +++ b/metaData/BH_skipClassAlignment.m @@ -27,17 +27,17 @@ % explicity copied here. cycleNumber = sprintf('cycle%0.3u', EMC_str2double(CYCLE)); -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); -flgClassify = pBH.('flgClassify'); +flgClassify = emc.('flgClassify'); try - flgMultiRefAlignment = pBH.('flgMultiRefAlignment'); + flgMultiRefAlignment = emc.('flgMultiRefAlignment'); catch flgMultiRefAlignment = 0; end -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); +outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); if strcmpi(STAGEofALIGNMENT, 'RawAlignment') @@ -65,7 +65,7 @@ ['the former requires the latter to exist.\n']); end -save(pBH.('subTomoMeta'), 'subTomoMeta'); +save(emc.('subTomoMeta'), 'subTomoMeta'); end diff --git a/metaData/EMC_parseParameterFile.py b/metaData/EMC_parseParameterFile.py deleted file mode 100644 index 8bcc76d8..00000000 --- a/metaData/EMC_parseParameterFile.py +++ /dev/null @@ -1,30 +0,0 @@ -import EMC_str2double as emc - -def EMC_parseParameterFile(PARAMETER_FILE): - # Open the parameter file - with open(PARAMETER_FILE, 'r') as fileID: - # Read the file into a list of strings, ignoring comments and empty lines - p = [line.strip() for line in fileID if line.strip() and not line.startswith('%')] - - # Check that all parameters are name: value pairs - stringValues = ['subTomoMeta', 'Ali_mType', 'Cls_mType', 'Cls_mType', 'Raw_mType', 'Fsc_mType', - 'Pca_distMeasure', 'Kms_mType', 'flgPrecision', 'Tmp_xcfScale', 'fastScratchDisk', - 'Tmp_eraseMaskType', 'startingDirection', 'Peak_mType', 'symmetry'] - pStruct = {} - for line in p: - try: - name, value = line.split('=', 1) - name = name.strip() - value = value.strip() - # also strip any trailing ';' or ',' - if value[-1] in [';', ',']: - value = value[:-1] - if name in stringValues: - pStruct[name] = value - else: - pStruct[name] = emc.EMC_str2double(value) - except: - err_msg = f"BH_parseParameterFile: invalid parameter line!\nReceived: {line}" - raise ValueError(err_msg) - - return pStruct \ No newline at end of file diff --git a/metaData/EMC_str2double.py b/metaData/EMC_str2double.py deleted file mode 100644 index 1d117b27..00000000 --- a/metaData/EMC_str2double.py +++ /dev/null @@ -1,21 +0,0 @@ -from numpy import ones - -def EMC_str2double(input_str): - try: - output_double = float(input_str) - except ValueError: - try: - output_double = eval(input_str) - if output_double is None: - raise ValueError - except ValueError: - try: - # check for ones() is in the string and use the numpy function to replace it - if 'ones(' in input_str: - input_str = input_str.replace('ones(', 'ones((') - input_str = input_str.replace(')', ', dtype=float)') - print(input_str) - except ValueError: - err_msg = f"EMC_str2double: input string is not a number!\nReceived: {input_str}" - raise ValueError(err_msg) - return output_double \ No newline at end of file diff --git a/metaData/EMC_str2double_test.py b/metaData/EMC_str2double_test.py deleted file mode 100644 index 85f28078..00000000 --- a/metaData/EMC_str2double_test.py +++ /dev/null @@ -1,21 +0,0 @@ -import numpy as np -import EMC_str2double as emc - -def test_EMC_str2double(): - # Test valid input - assert emc.EMC_str2double('3.14') == 3.14 - assert emc.EMC_str2double('-2.718') == -2.718 - assert emc.EMC_str2double('1e6') == 1e6 - assert emc.EMC_str2double('1.23e-4') == 1.23e-4 - assert emc.EMC_str2double('inf') == np.inf - assert emc.EMC_str2double('-inf') == -np.inf - assert np.isnan(emc.EMC_str2double('nan')) - - - try: - emc.EMC_str2double('not a number') - except ValueError as e: - assert str(e) == "EMC_str2double: input string is not a number!\nReceived: not a number" - -# Run test -test_EMC_str2double() \ No newline at end of file diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index cc988fec..b42a16f3 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -47,11 +47,11 @@ CYCLE = EMC_str2double(CYCLE); cycleNumber = sprintf('cycle%0.3u', CYCLE); -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); test_multi_ref_diffmap=true; -flgClassify = pBH.('flgClassify'); +flgClassify = emc.('flgClassify'); %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of %%% these features are re-introduced, this will need to be reverted. @@ -63,18 +63,18 @@ end try - nPeaks = pBH.('nPeaks'); + nPeaks = emc.('nPeaks'); catch nPeaks = 1; end -nRows = length(pBH.('pcaScaleSpace')); +nRows = length(emc.('pcaScaleSpace')); featureVector = cell(2,1); if flgGold - featureVector{1,1} = pBH.('Pca_coeffs_odd'); - featureVector{2,1} = pBH.('Pca_coeffs_eve'); + featureVector{1,1} = emc.('Pca_coeffs_odd'); + featureVector{2,1} = emc.('Pca_coeffs_eve'); else - featureVector{1,1} = pBH.('Pca_coeffs') + featureVector{1,1} = emc.('Pca_coeffs') featureVector{1,1} end @@ -83,62 +83,62 @@ error('There should be a set of indices for each pcaScaleSpace, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') end -clusterVector= pBH.('Pca_clusters'); +clusterVector= emc.('Pca_clusters'); try - kDIST = pBH.('Pca_distMeasure'); + kDIST = emc.('Pca_distMeasure'); catch kDIST = 'sqeuclidean'; end try - kREP = pBH.('Pca_nReplicates'); + kREP = emc.('Pca_nReplicates'); catch kREP = 256; end try - flgRefineKmeans = pBH.('Pca_refineKmeans') + flgRefineKmeans = emc.('Pca_refineKmeans') catch flgRefineKmeans = false end -nCores = BH_multi_parallelWorkers(pBH.('nCpuCores')); +nCores = BH_multi_parallelWorkers(emc.('nCpuCores')); % try -% relativeScale = pBH.('Pca_relativeScale') +% relativeScale = emc.('Pca_relativeScale') % catch % relativeScale= ones(size(clusterVector,1),1); % end try - flgFlattenEigs = pBH.('Pca_flattenEigs') + flgFlattenEigs = emc.('Pca_flattenEigs') catch flgFlattenEigs=1 end try - coverSteps = pBH.('Pca_som_coverSteps'); + coverSteps = emc.('Pca_som_coverSteps'); catch coverSteps = 100; end try - initNeighbor = pBH.('Pca_som_initNeighbor'); + initNeighbor = emc.('Pca_som_initNeighbor'); catch initNeighbor = 3; end try - topologyFcn = pBH.('Pca_som_topologyFcn'); + topologyFcn = emc.('Pca_som_topologyFcn'); catch % 'hextop' (default) | 'randtop' | 'gridtop' | 'tritop' topologyFcn = 'hextop'; end -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); masterTM = subTomoMeta; clear subTomoMeta try - flgMultiRefAlignment = pBH.('flgMultiRefAlignment'); + flgMultiRefAlignment = emc.('flgMultiRefAlignment'); catch flgMultiRefAlignment = 0; end @@ -168,8 +168,8 @@ randSet = [1,2]; end - coeffMatrix = sprintf('%s_%s_%s_pcaFull.mat',cycleNumber,pBH.('subTomoMeta'),halfSet); - outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); + coeffMatrix = sprintf('%s_%s_%s_pcaFull.mat',cycleNumber,emc.('subTomoMeta'),halfSet); + outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); % Get the number of tomograms to process. tomoList = fieldnames(geometry_clean); nTomograms = length(tomoList); @@ -402,7 +402,7 @@ newClass(class == ndx(i)) = i; end - fileOUT = fopen(sprintf('%s_%s_ClassIDX.txt',pBH.('subTomoMeta'),cycleNumber), 'a'); + fileOUT = fopen(sprintf('%s_%s_ClassIDX.txt',emc.('subTomoMeta'),cycleNumber), 'a'); fprintf(fileOUT, '\n\n%s, %s, %s\n','position','idx','count'); for iClass = 1:nClusters fprintf(fileOUT, '%d, %d\n',iClass,sum(newClass == iClass)); @@ -471,7 +471,7 @@ subTomoMeta = masterTM; - save(pBH.('subTomoMeta'), 'subTomoMeta'); + save(emc.('subTomoMeta'), 'subTomoMeta'); %save(sprintf('%s_pca.mat',OUTPUT_PREFIX), 'nTOTAL','U', 'S', 'V', 'coeffs') fprintf('Total execution time on set %s: %f seconds\n', halfSet,etime(clock, startTime)); diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 03bcf038..3c126397 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -31,49 +31,49 @@ % Put all out put in a subdirectory. system('mkdir -p FSC'); -pBH = BH_parseParameterFile(PARAMETER_FILE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +emc = BH_parseParameterFile(PARAMETER_FILE); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); masterTM = subTomoMeta; cycleNumber = sprintf('cycle%0.3u', CYCLE); prevCycleNumber = sprintf('cycle%0.3u',CYCLE-1); -flgCones = pBH.('flgCones'); -flgClassify= pBH.('flgClassify') +flgCones = emc.('flgCones'); +flgClassify= emc.('flgClassify') try - flgMultiRefAlignment = pBH.('flgMultiRefAlignment'); + flgMultiRefAlignment = emc.('flgMultiRefAlignment'); catch flgMultiRefAlignment = 0; end try - scaleCalcSize = pBH.('scaleCalcSize'); + scaleCalcSize = emc.('scaleCalcSize'); catch scaleCalcSize = 1.5; end try - flgFscShapeMask = pBH.('flgFscShapeMask'); + flgFscShapeMask = emc.('flgFscShapeMask'); catch flgFscShapeMask = 1; end try - shape_mask_lowpass = pBH.('shape_mask_lowpass'); + shape_mask_lowpass = emc.('shape_mask_lowpass'); catch shape_mask_lowpass = 14; end try - shape_mask_threshold = pBH.('shape_mask_threshold'); + shape_mask_threshold = emc.('shape_mask_threshold'); catch shape_mask_threshold = 2.4; end try % Apply the mask with the given parameters, save and exit. - shape_mask_test = pBH.('shape_mask_test'); + shape_mask_test = emc.('shape_mask_test'); catch shape_mask_test = false; end @@ -81,20 +81,20 @@ % Estimating the particle volume still occasionaly goes awry. Place a cap and return a cautionary message. try - minimumParticleVolume = pBH.('minimumparticleVolume'); + minimumParticleVolume = emc.('minimumparticleVolume'); catch minimumParticleVolume = 0.1; end -try fscWithChimera = pBH.('fscWithChimera'); +try fscWithChimera = emc.('fscWithChimera'); catch fscWithChimera = 0; end -outputPrefix = sprintf('./FSC/%s_%s', cycleNumber, pBH.('subTomoMeta')); -samplingRate = pBH.('Ali_samplingRate'); +outputPrefix = sprintf('./FSC/%s_%s', cycleNumber, emc.('subTomoMeta')); +samplingRate = emc.('Ali_samplingRate'); -pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; -if pBH.('SuperResolution') +pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; +if emc.('SuperResolution') pixelSize = pixelSize * 2; end @@ -131,9 +131,9 @@ % Note this is taken from the class section, not Fsc -refName = pBH.('Cls_className');% pBH.('Ref_className'); +refName = emc.('Cls_className');% emc.('Ref_className'); -peakSearch = floor(pBH.('particleRadius')./pixelSize); +peakSearch = floor(emc.('particleRadius')./pixelSize); peakCOM =3; global bh_global_MTF @@ -179,8 +179,8 @@ className = 0; classVector = [0;1]; else - className = pBH.(sprintf('Raw_className')); - classVector = pBH.(sprintf('Raw_classes_odd')); + className = emc.(sprintf('Raw_className')); + classVector = emc.(sprintf('Raw_classes_odd')); fieldPrefix = 'REF'; end @@ -243,10 +243,10 @@ % [ maskType, maskSize, maskRadius, maskCenter ] = ... -% BH_multi_maskCheck(pBH, 'Ali', pixelSize,'FSC') +% BH_multi_maskCheck(emc, 'Ali', pixelSize,'FSC') [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Ali', pixelSize) + BH_multi_maskCheck(emc, 'Ali', pixelSize) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc] = ... BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) @@ -1101,7 +1101,7 @@ end subTomoMeta = masterTM; -save(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); clearvars -except refWGT end diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 3c1cc67e..03d8c72d 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -140,19 +140,19 @@ cycleNumber = sprintf('cycle%0.3u', CYCLE); -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); reconScaling = 1; %%% Put this in the param file later - the input values should be in angstrom %%% and are the relevant scale spaces for classification. -pcaScaleSpace = pBH.('pcaScaleSpace'); +pcaScaleSpace = emc.('pcaScaleSpace'); nScaleSpace = numel(pcaScaleSpace); -samplingRate = pBH.('Cls_samplingRate'); -refSamplingRate= pBH.('Ali_samplingRate'); -randomSubset = pBH.('Pca_randSubset'); -maxEigs = pBH.('Pca_maxEigs'); -pixelSize = pBH.('PIXEL_SIZE').*10^10.*samplingRate; -refPixelSize = pBH.('PIXEL_SIZE').*10^10.*refSamplingRate; +samplingRate = emc.('Cls_samplingRate'); +refSamplingRate= emc.('Ali_samplingRate'); +randomSubset = emc.('Pca_randSubset'); +maxEigs = emc.('Pca_maxEigs'); +pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; +refPixelSize = emc.('PIXEL_SIZE').*10^10.*refSamplingRate; % FIMXE: Probably remove this incomplete idea if (refSamplingRate ~= samplingRate) @@ -160,37 +160,37 @@ end % FIXME: SuperResolution should be deprecated -if pBH.('SuperResolution') +if emc.('SuperResolution') pixelSize = pixelSize * 2; refPixelSize = refPixelSize * 2; end -nCores = BH_multi_parallelWorkers(pBH.('nCpuCores')); +nCores = BH_multi_parallelWorkers(emc.('nCpuCores')); pInfo = parcluster(); -nTempParticles = pBH.('PcaGpuPull'); +nTempParticles = emc.('PcaGpuPull'); try - scaleCalcSize = pBH.('scaleCalcSize'); + scaleCalcSize = emc.('scaleCalcSize'); catch scaleCalcSize = 1.5; end try - use_v2_SF3D = pBH.('use_v2_SF3D') + use_v2_SF3D = emc.('use_v2_SF3D') catch use_v2_SF3D = true end -outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); -%%%flgGold = pBH.('flgGoldStandard'); +outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); +%%%flgGold = emc.('flgGoldStandard'); try - nPeaks = pBH.('nPeaks'); + nPeaks = emc.('nPeaks'); catch nPeaks = 1; end -flgNorm = 1;% pBH.('flgNormalizeWMDs'); +flgNorm = 1;% emc.('flgNormalizeWMDs'); try - flgPcaShapeMask = pBH.('flgPcaShapeMask'); + flgPcaShapeMask = emc.('flgPcaShapeMask'); catch flgPcaShapeMask = 1; end @@ -199,19 +199,19 @@ % mask since we are concerned with densities that are likely damped during % averaging due to low occupancy. try - shape_mask_lowpass = pBH.('shape_mask_lowpass'); + shape_mask_lowpass = emc.('shape_mask_lowpass'); catch shape_mask_lowpass = 14 + 10; end try - shape_mask_threshold = pBH.('shape_mask_threshold'); + shape_mask_threshold = emc.('shape_mask_threshold'); catch shape_mask_threshold = 2.4 - 0.4; end try - tmpVal = pBH.('whitenPS'); + tmpVal = emc.('whitenPS'); if (numel(tmpVal) == 3) wiener_constant = tmpVal(3); else @@ -223,18 +223,18 @@ try % Apply the mask with the given parameters, save and exit. - shape_mask_test = pBH.('shape_mask_test'); + shape_mask_test = emc.('shape_mask_test'); catch shape_mask_test = false; end try - test_updated_bandpass = pBH.('test_updated_bandpass'); + test_updated_bandpass = emc.('test_updated_bandpass'); catch test_updated_bandpass = false; end -flgClassify = pBH.('flgClassify'); +flgClassify = emc.('flgClassify'); % Removed flgGold everywhere else, but keep ability to classify full data set at % the end (after all alignment is finished.) @@ -250,11 +250,11 @@ end -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; try - flgMultiRefAlignment = pBH.('flgMultiRefAlignment'); + flgMultiRefAlignment = emc.('flgMultiRefAlignment'); catch flgMultiRefAlignment = 0; end @@ -276,7 +276,7 @@ try - flgCutOutVolumes = pBH.('flgCutOutVolumes'); + flgCutOutVolumes = emc.('flgCutOutVolumes'); catch flgCutOutVolumes = 0; end @@ -302,10 +302,10 @@ [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Cls', pixelSize); + BH_multi_maskCheck(emc, 'Cls', pixelSize); [ preMaskType, preMaskSize, preMaskRadius, preMaskCenter ] = ... - BH_multi_maskCheck(pBH, 'Ali', refPixelSize); + BH_multi_maskCheck(emc, 'Ali', refPixelSize); % This is prob not a good way to make sure the mask size matches::w maskSize=preMaskSize; @@ -322,7 +322,7 @@ if (test_multi_ref_diffmap) - refName = pBH.('Raw_className'); + refName = emc.('Raw_className'); else refName = 0; end @@ -353,10 +353,10 @@ %% Added for test_multi_ref_diffmap %% refVector = cell(2,1); refGroup = cell(2,1); -classVector{1} = pBH.('Raw_classes_odd')(1,:); -classVector{2} = pBH.('Raw_classes_eve')(1,:); -refVectorFull{1}= [pBH.('Raw_classes_odd');classVector{1} ] -refVectorFull{2}= [pBH.('Raw_classes_eve');classVector{2} ] +classVector{1} = emc.('Raw_classes_odd')(1,:); +classVector{2} = emc.('Raw_classes_eve')(1,:); +refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ] +refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ] for iGold = 1:2 % Sort low to high, because order is rearranged as such unstack refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; @@ -487,14 +487,14 @@ end try - symmetry = pBH.('symmetry'); + symmetry = emc.('symmetry'); fprintf('\n\tWarning: As of emClarity 1.7.0.12 the symmetry parameter is applied to the volume and mask in PCA!\n') catch error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); end try - constrain_symmetry = pBH.('Pca_constrain_symmetry'); + constrain_symmetry = emc.('Pca_constrain_symmetry'); catch constrain_symmetry = false; end @@ -1007,7 +1007,7 @@ masterTM.(cycleNumber).('newIgnored_PCA').(halfSet) = gather(nIgnored); subTomoMeta = masterTM; - save(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); for iScale = 1:nScaleSpace dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 5bfe7a7e..ebc3d3aa 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -18,40 +18,40 @@ flgRunAlignments = true; COLOR_MAP= '0'; -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); try - flgColorMap = pBH.('flgColorMap'); + flgColorMap = emc.('flgColorMap'); catch flgColorMap = 0; end try - preShift = pBH.('preShift'); + preShift = emc.('preShift'); catch preShift = [-0.5,-0.5,0.5]; end try - postShift = pBH.('postShift'); + postShift = emc.('postShift'); catch postShift = [-0.5,-0.5]; end try - prjVectorShift = pBH.('prjVectorShift')'; + prjVectorShift = emc.('prjVectorShift')'; catch prjVectorShift = [0.5,0.5,1.0]'; end try - pixelShift = pBH.('pixelShift'); + pixelShift = emc.('pixelShift'); catch pixelShift = -1; end try - pixelMultiplier = pBH.('pixelMultiplier'); + pixelMultiplier = emc.('pixelMultiplier'); catch pixelMultiplier = 1; end @@ -87,13 +87,13 @@ reconScaling = 1; -samplingRate = pBH.('Ali_samplingRate'); +samplingRate = emc.('Ali_samplingRate'); % used to determine the number of fiducials/patch for local area. -MOL_MASS = pBH.('particleMass'); +MOL_MASS = emc.('particleMass'); molMass = MOL_MASS.*(25/samplingRate); try - tomoCPR_random_subset = pBH.('tomoCPR_randomSubset') + tomoCPR_random_subset = emc.('tomoCPR_randomSubset') catch tomoCPR_random_subset = -1 end @@ -101,7 +101,7 @@ %%%%%%%%%%% Parameters I am currently experimenting with as of Jan 2018 try - rmsScale = pBH.('rmsScale'); + rmsScale = emc.('rmsScale'); catch % Larger RMS downweights the contribution of the tomogram. The use of % the molecular mass in MDa fits with experiments from the ribosome that @@ -113,13 +113,13 @@ end try - probabilityPeakiness = pBH.('probPeakiness'); + probabilityPeakiness = emc.('probPeakiness'); catch probabilityPeakiness = 0; end try - useAverageDefocus = pBH.('useAverageDefocus'); + useAverageDefocus = emc.('useAverageDefocus'); catch % While it seems like using the per fiducial defocus max for the % refinement makes sense, have the default be the average of the @@ -128,7 +128,7 @@ end try - whitenProjections = pBH.('whitenProjections'); + whitenProjections = emc.('whitenProjections'); catch whitenProjections = 0; end @@ -141,94 +141,94 @@ %%% Playing around with the model n_surfaces=2; try - rot_option_global = pBH.('rot_option_global'); + rot_option_global = emc.('rot_option_global'); catch rot_option_global = 1; end try - rot_option_local = pBH.('rot_option_local'); + rot_option_local = emc.('rot_option_local'); catch rot_option_local = 1; end try - rot_default_grouping_global = pBH.('rot_default_grouping_global'); + rot_default_grouping_global = emc.('rot_default_grouping_global'); catch rot_default_grouping_global = 3; end try - rot_default_grouping_local = pBH.('rot_default_grouping_local'); + rot_default_grouping_local = emc.('rot_default_grouping_local'); catch rot_default_grouping_local = 3; end try - mag_option_global = pBH.('mag_option_global'); + mag_option_global = emc.('mag_option_global'); catch mag_option_global = 1; end try - mag_option_local = pBH.('mag_option_local'); + mag_option_local = emc.('mag_option_local'); catch mag_option_local = 1; end try - mag_default_grouping_global = pBH.('mag_default_grouping_global'); + mag_default_grouping_global = emc.('mag_default_grouping_global'); catch mag_default_grouping_global = 5; end try - mag_default_grouping_local = pBH.('mag_default_grouping_local'); + mag_default_grouping_local = emc.('mag_default_grouping_local'); catch mag_default_grouping_local = 5; end try - tilt_option_global = pBH.('tilt_option_global'); + tilt_option_global = emc.('tilt_option_global'); catch tilt_option_global = 5; end try - tilt_option_local = pBH.('tilt_option_local'); + tilt_option_local = emc.('tilt_option_local'); catch tilt_option_local = 5; end try - tilt_default_grouping_global = pBH.('tilt_default_grouping_global'); + tilt_default_grouping_global = emc.('tilt_default_grouping_global'); catch tilt_default_grouping_global = 5; end try - tilt_default_grouping_local = pBH.('tilt_default_grouping_local'); + tilt_default_grouping_local = emc.('tilt_default_grouping_local'); catch tilt_default_grouping_local = 5; end try - nFiducialsPerPatch = pBH.('n_fiducials_per_patch'); + nFiducialsPerPatch = emc.('n_fiducials_per_patch'); catch % TODO how smooth should the solutions really be - should multiple % results be run and compared? nFiducialsPerPatch = ceil(100./sqrt(molMass)); end try - target_patch_size = pBH.('target_patch_size'); + target_patch_size = emc.('target_patch_size'); catch target_patch_size = 500; end try - peak_mask_fraction = pBH.('peak_mask_fraction'); + peak_mask_fraction = emc.('peak_mask_fraction'); catch peak_mask_fraction = 0.4; end try - min_overlap = pBH.('min_overlap'); + min_overlap = emc.('min_overlap'); catch min_overlap = 0.5; end try - k_factor_scaling = pBH.('k_factor_scaling'); + k_factor_scaling = emc.('k_factor_scaling'); catch k_factor_scaling = nan; end try - shift_z_to_to_centroid = pBH.('shift_z_to_to_centroid'); + shift_z_to_to_centroid = emc.('shift_z_to_to_centroid'); catch shift_z_to_to_centroid = true; end @@ -236,7 +236,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% try - use_PCF = pBH.('use_PCF') + use_PCF = emc.('use_PCF') catch use_PCF = 0; end @@ -252,7 +252,7 @@ % so override the tmpCache. tmpCache = ''; else - tmpCache= pBH.('fastScratchDisk'); + tmpCache= emc.('fastScratchDisk'); end if strcmpi(tmpCache, 'ram') @@ -272,10 +272,10 @@ end % % % nWorkers = EMC_str2double(nWORKERS) -nGPUs = pBH.('nGPUs'); +nGPUs = emc.('nGPUs'); pInfo = parcluster(); gpuScale=3*samplingRate; -nWorkers = min(nGPUs*gpuScale,pBH.('nCpuCores')); % 18 +nWorkers = min(nGPUs*gpuScale,emc.('nCpuCores')); % 18 fprintf('Using %d workers as max of %d %d*nGPUs and %d nWorkers visible\n', ... nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); @@ -304,17 +304,17 @@ system(sprintf('mkdir -p %s',tmpCache)); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; % Add error check onrange for reasonable values. -ctfRange = pBH.('tomoCprDefocusRange')*10^10; -ctfInc = pBH.('tomoCprDefocusStep')*10^10; +ctfRange = emc.('tomoCprDefocusRange')*10^10; +ctfInc = emc.('tomoCprDefocusStep')*10^10; -calcCTF = pBH.('tomoCprDefocusRefine'); +calcCTF = emc.('tomoCprDefocusRefine'); [tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); @@ -353,18 +353,18 @@ refVol = cell(2,1); -refName = pBH.('Raw_className'); +refName = emc.('Raw_className'); try - symmetry = pBH.('symmetry'); + symmetry = emc.('symmetry'); catch error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); end -classVector{1} = pBH.('Raw_classes_odd')(1,:); -classSymmetry{1}= pBH.('Raw_classes_odd')(2,:); -classVector{2} = pBH.('Raw_classes_eve')(1,:); -classSymmetry{2}= pBH.('Raw_classes_eve')(2,:); +classVector{1} = emc.('Raw_classes_odd')(1,:); +classSymmetry{1}= emc.('Raw_classes_odd')(2,:); +classVector{2} = emc.('Raw_classes_eve')(1,:); +classSymmetry{2}= emc.('Raw_classes_eve')(2,:); nRefs = length(classVector{1}) @@ -403,18 +403,18 @@ % try % refNameODD = sprintf('%s_%s_class0_REF_ODD.mrc', ... -% cycleNumber,pBH.('subTomoMeta')); +% cycleNumber,emc.('subTomoMeta')); % refNameEVE = sprintf('%s_%s_class0_REF_EVE.mrc', ... -% cycleNumber,pBH.('subTomoMeta')); +% cycleNumber,emc.('subTomoMeta')); % refODD = getVolume(MRCImage(refNameODD)); % refEVE = getVolume(MRCImage(refNameEVE)); % catch % fprintf('\nDid not find either %s or %s, trying Raw prefix\n',refNameODD,refNameEVE); % try % refNameODD = sprintf('%s_%s_class0_Raw_ODD.mrc', ... -% cycleNumber,pBH.('subTomoMeta')); +% cycleNumber,emc.('subTomoMeta')); % refNameEVE = sprintf('%s_%s_class0_Raw_EVE.mrc', ... -% cycleNumber,pBH.('subTomoMeta')); +% cycleNumber,emc.('subTomoMeta')); % % refODD = getVolume(MRCImage(refNameODD)); % refEVE = getVolume(MRCImage(refNameEVE)); @@ -424,7 +424,7 @@ % end try - conserveDiskSpace = pBH.('conserveDiskSpace'); + conserveDiskSpace = emc.('conserveDiskSpace'); catch conserveDiskSpace = 0; end @@ -501,15 +501,15 @@ end % The model is scaled to full sampling prior to passing to tiltalign, % make sure the header in the synthetic stack is set appropriately. - fullPixelSize = pBH.('PIXEL_SIZE').*10^10; - if pBH.('SuperResolution') + fullPixelSize = emc.('PIXEL_SIZE').*10^10; + if emc.('SuperResolution') fullPixelSize = fullPixelSize * 2; end pixelSize = fullPixelSize.*samplingRate; try - eraseMaskType = pBH.('Peak_mType'); - eraseMaskRadius = pBH.('Peak_mRadius')./pixelSize; + eraseMaskType = emc.('Peak_mType'); + eraseMaskRadius = emc.('Peak_mRadius')./pixelSize; fprintf('Further restricting peak search to radius %f %f %f\n',... eraseMaskRadius); eraseMask = 1; @@ -519,14 +519,14 @@ end - [ ~,~,maskRadius,~ ] = BH_multi_maskCheck(pBH,'Ali',pixelSize) - PARTICLE_RADIUS = floor(max(pBH.('particleRadius')./pixelSize)); + [ ~,~,maskRadius,~ ] = BH_multi_maskCheck(emc,'Ali',pixelSize) + PARTICLE_RADIUS = floor(max(emc.('particleRadius')./pixelSize)); - %PARTICLE_RADIUS = floor(mean(pBH.('particleRadius')./pixelSize)); + %PARTICLE_RADIUS = floor(mean(emc.('particleRadius')./pixelSize)); % TODO, is this too restricted? peakSearchRad = floor(peak_mask_fraction*PARTICLE_RADIUS.*[1,1]); try - lowPassCutoff = pBH.('tomoCprLowPass'); + lowPassCutoff = emc.('tomoCprLowPass'); fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n.',... lowPassCutoff); catch @@ -548,7 +548,7 @@ min_res_for_ctf_fitting = 10.0; if (calcCTF) try - min_res_for_ctf_fitting = pBH.('min_res_for_ctf_fitting'); + min_res_for_ctf_fitting = emc.('min_res_for_ctf_fitting'); catch end @@ -615,7 +615,7 @@ end fprintf('init with %d workers\n',nWorkers); - outputPrefix = sprintf('%s_%s', cycleNumber, pBH.('subTomoMeta')); + outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); @@ -2257,7 +2257,7 @@ subTomoMeta.('tomoCPR_run_in_cycle') = [subTomoMeta.currentTomoCPR,CYCLE]; end - save(pBH.('subTomoMeta'), 'subTomoMeta'); + save(emc.('subTomoMeta'), 'subTomoMeta'); end end diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index f76cc452..679f3880 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -68,18 +68,18 @@ cycleNumber = sprintf('cycle%0.3u', CYCLE); -pBH = BH_parseParameterFile(PARAMETER_FILE); +emc = BH_parseParameterFile(PARAMETER_FILE); reconScaling = 1; -samplingRate = 1; % Always working at full binning. pBH.('Ali_samplingRate'); +samplingRate = 1; % Always working at full binning. emc.('Ali_samplingRate'); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); resForFitting = 1.3*mean(subTomoMeta.currentResForDefocusError); -nGPUs = pBH.('nGPUs'); +nGPUs = emc.('nGPUs'); pInfo = parcluster(); gpuScale=3*samplingRate -nWorkers = min(nGPUs*gpuScale,pBH.('nCpuCores')); % 18 +nWorkers = min(nGPUs*gpuScale,emc.('nCpuCores')); % 18 fprintf('Using %d workers as max of %d %d*nGPUs and %d nWorkers visible\n', ... nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); @@ -89,7 +89,7 @@ system(sprintf('mkdir -p %s',tmpCache)); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; @@ -172,12 +172,12 @@ % The model is scaled to full sampling prior to passing to tiltalign, % make sure the header in the synthetic stack is set appropriately. - fullPixelSize = pBH.('PIXEL_SIZE').*10^10; + fullPixelSize = emc.('PIXEL_SIZE').*10^10; pixelSize = fullPixelSize.*samplingRate; - PARTICLE_RADIUS = floor(max(pBH.('particleRadius')./pixelSize)); + PARTICLE_RADIUS = floor(max(emc.('particleRadius')./pixelSize)); [~,tiltBaseName,~] = fileparts(tiltList{1}); @@ -646,10 +646,10 @@ sigma = 10.0; score = 10.0; % TODO test with scaled CCC score? scoreChange = 0.0; - pixelSize = pBH.('PIXEL_SIZE') * 10^10; - micVoltage = pBH.('VOLTAGE') * 10^-3; - micCS = pBH.('Cs') * 10^3; - ampContrast = pBH.('AMPCONT') * 10^0; + pixelSize = emc.('PIXEL_SIZE') * 10^10; + micVoltage = emc.('VOLTAGE') * 10^-3; + micCS = emc.('Cs') * 10^3; + ampContrast = emc.('AMPCONT') * 10^0; beamTiltX = 0.0; beamTiltY = 0.0; beamTiltShiftX = 0.0; @@ -701,7 +701,7 @@ SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',baseFile),pixelSize); - maxThreads = pBH.('nCpuCores'); + maxThreads = emc.('nCpuCores'); %%%%%%%%%%%%%%%%%%%%%%%%%% % Initial reconstruction @@ -745,8 +745,8 @@ 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : '%2.2d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,pBH.('PIXEL_SIZE')*10^10, ... - pBH.('particleMass')*10^3, 0.0, mean(pBH.('Ali_mRadius')), maxThreads); + symmetry,emc.('PIXEL_SIZE')*10^10, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); fprintf(recScript, '\neof\n'); @@ -814,8 +814,8 @@ 'no\n',...Threshold input reconstruction [No] : '%2.2d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_REFINE3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,pBH.('PIXEL_SIZE')*10^10, ... - pBH.('particleMass')*10^3, 0.0, mean(pBH.('Ali_mRadius')), ... + symmetry,emc.('PIXEL_SIZE')*10^10, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... resForFitting,resForFitting,maxThreads); fprintf(refineScript, '\neof\n'); @@ -867,8 +867,8 @@ 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : '%2.2d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'), baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,pBH.('PIXEL_SIZE')*10^10, ... - pBH.('particleMass')*10^3, 0.0, mean(pBH.('Ali_mRadius')), maxThreads); + symmetry,emc.('PIXEL_SIZE')*10^10, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); fprintf(recScript, '\neof\n'); diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index df4c652a..d93eb3cf 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -141,7 +141,7 @@ if nArgs > 1 && ~(emcHelp || emcProgramHelp) switch varargin{1} case 'ctf' - pBH = emC_testParse(varargin{3}); + emc = emC_testParse(varargin{3}); case 'rescale' % nothing to parse multiGPUs = 0; @@ -162,11 +162,11 @@ multiGPUs = 0; otherwise - pBH = emC_testParse(varargin{2}); + emc = emC_testParse(varargin{2}); end if ( multiGPUs ) % wanted num gpus - nGPUs_wanted = pBH.('nGPUs'); + nGPUs_wanted = emc.('nGPUs'); cudaStart = getenv('CUDA_VISIBLE_DEVICES') nGPUs_visible = gpuDeviceCount; if nGPUs_visible < nGPUs_wanted @@ -683,12 +683,12 @@ -function [ pBH ] = emC_testParse( paramTest ) +function [ emc ] = emC_testParse( paramTest ) % Try to parse the parameter file make sure it's okay. % % Add some actual error handling here to help trouble shoot. try - pBH = BH_parseParameterFile( paramTest ); + emc = BH_parseParameterFile( paramTest ); @@ -702,7 +702,7 @@ global bh_global_window_cutoff; try - bh_global_window_cutoff = pBH.('windowCutoff'); + bh_global_window_cutoff = emc.('windowCutoff'); catch bh_global_window_cutoff = -2; end @@ -742,7 +742,7 @@ global bh_global_turn_on_phase_plate; try - bh_global_turn_on_phase_plate = pBH.('phakePhasePlate'); + bh_global_turn_on_phase_plate = emc.('phakePhasePlate'); catch bh_global_turn_on_phase_plate = 0; end @@ -752,14 +752,14 @@ %%%% ready global bh_global_do_2d_fourier_interp; try - bh_global_do_2d_fourier_interp = pBH.('useFourierInterp'); + bh_global_do_2d_fourier_interp = emc.('useFourierInterp'); catch bh_global_do_2d_fourier_interp = 1; end global bh_global_save_tomoCPR_diagnostics; try - bh_global_save_tomoCPR_diagnostics = pBH.('tomoCprDiagnostics'); + bh_global_save_tomoCPR_diagnostics = emc.('tomoCprDiagnostics'); catch bh_global_save_tomoCPR_diagnostics = 0; end @@ -772,14 +772,14 @@ %%%%% For profiling global bh_global_do_profile; try - bh_global_do_profile = pBH.('doProfile'); + bh_global_do_profile = emc.('doProfile'); catch bh_global_do_profile = false; end try - bh_global_fast_scratch_disk = pBH.('fastScratchDisk'); + bh_global_fast_scratch_disk = emc.('fastScratchDisk'); catch bh_global_fast_scratch_disk=''; end @@ -787,7 +787,7 @@ try - bh_global_ram_disk = pBH.('ramDisk'); + bh_global_ram_disk = emc.('ramDisk'); catch bh_global_ram_disk = '/dev/shm'; end @@ -804,27 +804,27 @@ try - bh_global_binary_mask_low_pass = pBH.('setMaskLowPass'); + bh_global_binary_mask_low_pass = emc.('setMaskLowPass'); catch % These seem to be okay for higher-resolution data (EMPIAR ribo sets) bh_global_binary_mask_low_pass = 14; end try - bh_global_binary_mask_threshold = pBH.('setMaskThreshold'); + bh_global_binary_mask_threshold = emc.('setMaskThreshold'); catch bh_global_binary_mask_threshold = 2.5; end try - bh_global_binary_pcaMask_threshold = pBH.('setPcaMaskThreshold'); + bh_global_binary_pcaMask_threshold = emc.('setPcaMaskThreshold'); catch bh_global_binary_pcaMask_threshold = 0.5; end global bh_global_kFactorScaling; try - bh_global_kFactorScaling = pBH.('kFactorScaling'); + bh_global_kFactorScaling = emc.('kFactorScaling'); catch bh_global_kFactorScaling = 1.0; end @@ -832,7 +832,7 @@ try - bh_global_vol_est_scaling = pBH.('setParticleVolumeScaling'); + bh_global_vol_est_scaling = emc.('setParticleVolumeScaling'); catch % The low pass version of the map used for the estimate overestimates % the molecular volume at the hydration radius of the underlying atoms. @@ -848,21 +848,21 @@ % 0 - off, 2 original (matches closely measured MTF), 1 stronger % Anthing else, float, iX = scalar, dX = cap val e.g. % opiton 1 100.04 and 2 (default) is 25.06 - bh_global_MTF = pBH.('mtfVal'); + bh_global_MTF = emc.('mtfVal'); catch bh_global_MTF = 2; end global bh_global_print_shifts_in_particle_basis; try - bh_global_print_shifts_in_particle_basis = pBH.('printShiftsInParticleBasis'); + bh_global_print_shifts_in_particle_basis = emc.('printShiftsInParticleBasis'); catch bh_global_print_shifts_in_particle_basis = true; end global bh_global_zero_lag_score; try - bh_global_zero_lag_score = pBH.('useZeroLagScore'); + bh_global_zero_lag_score = emc.('useZeroLagScore'); catch bh_global_zero_lag_score = false; end @@ -870,12 +870,12 @@ global bh_global_ML_compressByFactor; global bh_global_ML_angleTolerance; try - bh_global_ML_compressByFactor = pBH.('ML_compressByFactor'); + bh_global_ML_compressByFactor = emc.('ML_compressByFactor'); catch bh_global_ML_compressByFactor = 2.0; end try - bh_global_ML_angleTolerance = pBH.('ML_angleTolerance'); + bh_global_ML_angleTolerance = emc.('ML_angleTolerance'); catch bh_global_ML_angleTolerance = 5; end diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index ea2b52a6..e3a2eb09 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -104,17 +104,13 @@ cycleNumber = sprintf('cycle%0.3u', CYCLE) -pBH = BH_parseParameterFile(PARAMETER_FILE); -load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); +emc = BH_parseParameterFile(PARAMETER_FILE); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); reconScaling = 1; -try - nPeaks = pBH.('nPeaks'); -catch - nPeaks = 1; -end + try - tmpVal = pBH.('whitenPS'); + tmpVal = emc.('whitenPS'); if (numel(tmpVal) == 3) wiener_constant = tmpVal(3); else @@ -125,7 +121,7 @@ end try - fscBfactor = pBH.('Fsc_bfactor'); + fscBfactor = emc.('Fsc_bfactor'); catch fscBfactor = 40; end @@ -134,7 +130,7 @@ if (CYCLE) try - flgQualityWeight = pBH.('flgQualityWeight'); + flgQualityWeight = emc.('flgQualityWeight'); catch flgQualityWeight = 5; end @@ -145,20 +141,20 @@ % Experimental downweighting of higher frequency info farther from focus. % Could also consider filtering pre reconstruction try - flgFilterDefocus = pBH.('filterDefocus'); + flgFilterDefocus = emc.('filterDefocus'); fprintf('\nFiltering by defocus using exp[-(%d*(argmax(def-1,0,5).*q)^%d)]\n',flgFilterDefocus); catch flgFilterDefocus = 0; end try - flgCutOutVolumes = pBH.('flgCutOutVolumes'); + flgCutOutVolumes = emc.('flgCutOutVolumes'); catch flgCutOutVolumes = 0; end try - projectVolumes = pBH.('flgProjectVolumes'); + projectVolumes = emc.('flgProjectVolumes'); catch projectVolumes = false; end @@ -179,14 +175,14 @@ end try - track_stats = pBH.('track_stats'); + track_stats = emc.('track_stats'); catch track_stats = false; end % Note this will be set to false unless we are averging after an update try - flgShiftEucentric = pBH.('eucentric_fit'); + flgShiftEucentric = emc.('eucentric_fit'); catch flgShiftEucentric = 0; end @@ -195,7 +191,7 @@ % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search try - doHelical = pBH.('doHelical'); + doHelical = emc.('doHelical'); catch doHelical = 0; end @@ -219,22 +215,22 @@ end fprintf('track stats is %d\n',track_stats) -flgClassify= pBH.('flgClassify'); +flgClassify= emc.('flgClassify'); %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of %%% these features are re-introduced, this will need to be reverted. if ( flgClassify ); flgClassify = -1 ; end try - flgMultiRefAlignment = pBH.('flgMultiRefAlignment'); + flgMultiRefAlignment = emc.('flgMultiRefAlignment'); catch flgMultiRefAlignment = 0; end flgGold=1; -pixelSize = pBH.('PIXEL_SIZE').*10^10; -if pBH.('SuperResolution') +pixelSize = emc.('PIXEL_SIZE').*10^10; +if emc.('SuperResolution') pixelSize = pixelSize * 2; end -nGPUs = pBH.('nGPUs'); +nGPUs = emc.('nGPUs'); % Optionally specify gpu idxs if numel(nGPUs) == 1 gpuList = 1:nGPUs; @@ -246,19 +242,19 @@ try - loadTomo = pBH.('loadTomo') + loadTomo = emc.('loadTomo') catch loadTomo = 0 end try - scaleCalcSize = pBH.('scaleCalcSize'); + scaleCalcSize = emc.('scaleCalcSize'); catch scaleCalcSize = 1.5; end try - use_v2_SF3D = pBH.('use_v2_SF3D') + use_v2_SF3D = emc.('use_v2_SF3D') catch use_v2_SF3D = true end @@ -301,15 +297,15 @@ end - classVector{1} = pBH.(sprintf('%s_classes_odd','Raw')); - classVector{2} = pBH.(sprintf('%s_classes_eve','Raw')); + classVector{1} = emc.(sprintf('%s_classes_odd','Raw')); + classVector{2} = emc.(sprintf('%s_classes_eve','Raw')); - className = pBH.(sprintf('%s_className','Raw')); - samplingRate = pBH.('Ali_samplingRate'); + className = emc.(sprintf('%s_className','Raw')); + samplingRate = emc.('Ali_samplingRate'); if (flgMultiRefAlignment && (test_multi_ref_diffmap || ~flgClassify)) - className = pBH.(sprintf('Raw_className')) - saveClassSum = pBH.(sprintf('Raw_className')) + className = emc.(sprintf('Raw_className')) + saveClassSum = emc.(sprintf('Raw_className')) elseif (flgMultiRefAlignment && flgClassify) fprintf('\n\nMutliRef and Classify enabled.\n'); fprintf('Only creating the global class average for PCA\n\n.'); @@ -327,15 +323,15 @@ % Goal is to re-extract odd-half, applying the xform found in fscGold fieldPrefix = 'Raw' - classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); - classVector{2} = pBH.(sprintf('%s_classes_eve',fieldPrefix)); + classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); + classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); - className = pBH.(sprintf('%s_className',fieldPrefix)); - samplingRate = pBH.('Ali_samplingRate'); + className = emc.(sprintf('%s_className',fieldPrefix)); + samplingRate = emc.('Ali_samplingRate'); if (flgClassify) - %samplingRate = pBH.('Pca_samplingRate'); + %samplingRate = emc.('Pca_samplingRate'); else - %samplingRate = pBH.('Raw_samplingRate'); + %samplingRate = emc.('Raw_samplingRate'); fieldPrefix = 'REF' end @@ -367,14 +363,14 @@ ClusterGeomNAME = 'ClusterClsGeom'; fieldPrefix = 'Cls'; - classVector{1} = pBH.(sprintf('%s_classes_odd',fieldPrefix)); - classVector{2} = pBH.(sprintf('%s_classes_eve',fieldPrefix)); + classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); + classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); - classCoeffs{1} = pBH.('Pca_coeffs'); - classCoeffs{2} = pBH.('Pca_coeffs'); + classCoeffs{1} = emc.('Pca_coeffs'); + classCoeffs{2} = emc.('Pca_coeffs'); - samplingRate = pBH.(sprintf('Cls_samplingRate')); - className = pBH.(sprintf('%s_className',fieldPrefix)); + samplingRate = emc.(sprintf('Cls_samplingRate')); + className = emc.(sprintf('%s_className',fieldPrefix)); if flgClassify < 0 flgGold = 0; end @@ -390,7 +386,7 @@ classVector{2} = [1:25;ones(1,25)]; className = 25; - samplingRate = pBH.(sprintf('%s_samplingRate','Ali')); + samplingRate = emc.(sprintf('%s_samplingRate','Ali')); otherwise error('STAGEofALIGNMENT incorrect') @@ -399,24 +395,24 @@ fprintf('StOAlign = %s, fieldPrefix = %s\n', STAGEofALIGNMENT, fieldPrefix); -flgCones = pBH.('flgCones'); +flgCones = emc.('flgCones'); try % if > 1 keep this many subtomos % if < 1 keep this fraction - cccCutOff = pBH.('flgCCCcutoff'); + cccCutOff = emc.('flgCCCcutoff'); catch cccCutOff = 0.0; end -cutPrecision = 'single'; %pBH.('flgPrecision'); +cutPrecision = 'single'; %emc.('flgPrecision'); try - interpOrder = pBH.('interpOrder'); + interpOrder = emc.('interpOrder'); catch interpOrder = 1; end try - flgLimitToOneProcess = pBH.('flgLimitToOneProcess'); + flgLimitToOneProcess = emc.('flgLimitToOneProcess'); catch flgLimitToOneProcess = 0; end @@ -431,7 +427,7 @@ elseif (flgLimitToOneProcess) limitToOne = flgLimitToOneProcess; else - limitToOne = pBH.('nCpuCores'); + limitToOne = emc.('nCpuCores'); interpOrder = 1; end @@ -444,10 +440,10 @@ if ~(ismember(interpOrder,[1,4])) error('interpolationOrder must be 1,,4 - linear,sinc'); end -outputPrefix = sprintf('%s_%s',cycleNumber, pBH.('subTomoMeta')); +outputPrefix = sprintf('%s_%s',cycleNumber, emc.('subTomoMeta')); pixelSize = pixelSize .* samplingRate; -peakSearch = floor(0.85.*pBH.('particleRadius')./pixelSize) +peakSearch = floor(0.85.*emc.('particleRadius')./pixelSize) peakCOM = [1,1,1].*3; @@ -467,7 +463,7 @@ cycleRead = sprintf('cycle%0.3u', CYCLE - 1); % Save a backup of the cycles total geometry - save(sprintf('%s_%s_backup.mat',cycleRead,pBH.('subTomoMeta')), ... + save(sprintf('%s_%s_backup.mat',cycleRead,emc.('subTomoMeta')), ... 'subTomoMeta'); end @@ -555,7 +551,7 @@ masterTM = subTomoMeta; clear subTomoMeta try - symmetry = pBH.('symmetry'); + symmetry = emc.('symmetry'); catch error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); end @@ -591,7 +587,7 @@ if (flgClassify) [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Ali', pixelSize); + BH_multi_maskCheck(emc, 'Ali', pixelSize); % These are used when 'Cluster' is called, to take the masking parameters % from focused PCA/Classification, to produce a montage with reduced % Z-dimension & low pass filtering to be used in decision making but not @@ -600,11 +596,11 @@ [~, pcaMaskSize, pcaMaskRadius, pcaMaskCenter ] = ... - BH_multi_maskCheck(pBH, 'Cls', pixelSize); + BH_multi_maskCheck(emc, 'Cls', pixelSize); else [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(pBH, 'Ali', pixelSize); + BH_multi_maskCheck(emc, 'Ali', pixelSize); end @@ -737,7 +733,7 @@ maxCCC = 0; try - spike_prior = pBH.('spike_prior') + spike_prior = emc.('spike_prior') catch spike_prior = false end @@ -802,7 +798,7 @@ [ normal_vect, chi2 ] = BH_fit_ellipsoidal_prior(pixelSize .* particle_coords(positions_to_analyze,3:5), ... - pBH.('particleRadius')(3), ... + emc.('particleRadius')(3), ... radial_shrink_factor, ... display_fit); @@ -1875,7 +1871,7 @@ subTomoMeta.('currentCycle') = gather(CYCLE); -save(pBH.('subTomoMeta'), 'subTomoMeta'); +save(emc.('subTomoMeta'), 'subTomoMeta'); @@ -1912,7 +1908,7 @@ if ~( flgEstSNR ) - load(sprintf('%s.mat', pBH.('subTomoMeta')), 'subTomoMeta'); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); masterTM = subTomoMeta; %%%%%%%%%%%%%55 Reweight now that the FSC is calculated @@ -1947,7 +1943,6 @@ % This is slow ass when using cones and class averages and wouldn't be too % hard to put into parallel. Do that once the next manuscript is finished. if (~flgMultiRefAlignment && ~flgClassify ) - nClassesReWgt = 1; else nClassesReWgt = maxClasses; @@ -2076,7 +2071,7 @@ if (flgCutOutVolumes && doCut) subTomoMeta.('volumesAreCutOut') = 1; end - save(pBH.('subTomoMeta'), 'subTomoMeta'); + save(emc.('subTomoMeta'), 'subTomoMeta'); end From e65636ef2d9548150372c274655e825ad1ec3bde Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 08:52:41 -0500 Subject: [PATCH 007/151] Move nPeaks to parser --- alignment/BH_alignRaw3d_v2.m | 39 +- alignment/BH_templateSearch3d.m | 1307 ----------------------------- alignment/BH_templateSearch3d_2.m | 18 +- metaData/BH_geometryInitialize.m | 25 +- metaData/BH_parseParameterFile.m | 22 + metaData/BH_removeDuplicates.m | 14 +- statistics/BH_clusterPub.m | 11 +- statistics/BH_pcaPub.m | 19 +- transformations/BH_average3d.m | 146 +--- 9 files changed, 89 insertions(+), 1512 deletions(-) delete mode 100755 alignment/BH_templateSearch3d.m diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 064ab8bc..6c25e09e 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -69,11 +69,6 @@ mapBackIter = subTomoMeta.currentTomoCPR; reconScaling = 1; -try - nPeaks = emc.('nPeaks'); -catch - nPeaks = 1; -end try track_stats = emc.('track_stats'); @@ -961,9 +956,9 @@ 'GPU', 'inv'); inputVectors = {iv1,iv2,iv3}; iv1 = []; iv2 = []; iv3 = []; - cccStorageBest = cell(nPeaks,1); - cccStorageRefine = cell(nPeaks,1); - for iPeak = 1:nPeaks + cccStorageBest = cell(emc.nPeaks,1); + cccStorageRefine = cell(emc.nPeaks,1); + for iPeak = 1:emc.nPeaks cccStorageBest{iPeak} = zeros(nSubTomos,10); cccStorageRefine{iPeak}= zeros(nSubTomos,10); end @@ -1002,7 +997,7 @@ - for iPeak = 1:nPeaks + for iPeak = 1:emc.nPeaks if (track_stats) measure_noise = true; @@ -1667,31 +1662,19 @@ try if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); -% % % % Get the negative slope of the top ten CCC scores. -% % % topTen = fit([.1:.1:1]',sortRef(1:10,6),'linear'); -% % % bestRotPeak(1,7) = topTen(100)-topTen(101); + else bestRotPeak = cccPreRefineSort(1,:); bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); -% % % rowNum = min(size(cccPreRefineSort,1),10*nPeaks); -% % % topX = 1- 0.1.*(10-rowNum); -% % % % Get the negative slope of the top ten CCC scores. -% % % topTen = fit([.1:.1:topX]',cccPreRefineSort(1:rowNum,6),'linear'); -% % % bestRotPeak(1,7) = topTen(100)-topTen(101); + end catch fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); cccStorageRefine{iPeak}(iSubTomo,:) cccPreRefineSort(1,:) -% % % rowNum = min(size(cccPreRefineSort,1),10*nPeaks) -% % % topX = 1- 0.1.*(10-rowNum) -% % % fprintf('\nNow check the fits, first and second clause\n'); -% % % topTen = fit([.1:.1:1]',sortRef(1:10,6),'linear') -% % % fprintf('\nSecond\n'); -% % % topTen = fit([.1:.1:topX]',cccPreRefineSort(1:rowNum,6),'linear') -% % % error('Error in sorting the best peak in alignRaw'); + end finalRef = bestRotPeak(1,1); @@ -1882,13 +1865,13 @@ end % loop over subTomos - for iPeak = 1:nPeaks + for iPeak = 1:emc.nPeaks % Get rid of any zero entries left over from pre-initialization if iPeak == 1 nonZeroInits = ( cccStorageBest{iPeak}(:,2) ~= 0 ); cccStorageBest{1}=cccStorageBest{1}(nonZeroInits,:); - sortCCC = zeros(size(cccStorageBest{1},1),10*nPeaks); + sortCCC = zeros(size(cccStorageBest{1},1),10*emc.nPeaks); else cccStorageBest{iPeak}=cccStorageBest{iPeak}(nonZeroInits,:); end @@ -1912,7 +1895,7 @@ angOut = fopen(sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}),'w'); for iRow = 1:size( bestAngles_tmp.(tomoList{iTomo}),1) - for iPeak = 1:nPeaks + for iPeak = 1:emc.nPeaks fprintf(angOut,'%d %d %6.3f %6.3f %6.3f %6.6f %6.6f %6.3f %6.3f %6.3f ', ... bestAngles_tmp.(tomoList{iTomo})(iRow,1+10*(iPeak-1):10+10*(iPeak-1))); end @@ -1946,7 +1929,7 @@ % save('bestAnglesTemp.mat', 'bestAngles'); save('bestAngles.mat', 'bestAngles'); - [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, nPeaks, rotConvention, updateWeights, updateClassByBestReferenceScore); + [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, emc.nPeaks, rotConvention, updateWeights, updateClassByBestReferenceScore); masterTM.(cycleNumber).('RawAlign') = rawAlign; masterTM.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); masterTM.('updatedWeights') = true; diff --git a/alignment/BH_templateSearch3d.m b/alignment/BH_templateSearch3d.m deleted file mode 100755 index e6d0bbcf..00000000 --- a/alignment/BH_templateSearch3d.m +++ /dev/null @@ -1,1307 +0,0 @@ -function [hAvg, hRms, avgRange, rmsRange] = BH_templateSearch3d( PARAMETER_FILE,... - tomoName,tomoNumber,TEMPLATE, ... - SYMMETRY, wedgeType, varargin) - - -%3d template matching - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -precision = 'single'; -precisionTaper = 'singleTaper'; - -if length(varargin) == 1 - % Allow for an override of the max number, useful when only a few tomos - % have a strong feature like carbon that is hard to avoid. - cmdLineThresh = 0; - gpuIDX = EMC_str2double(varargin{1}); -elseif length(varargin) == 2 - cmdLineThresh = EMC_str2double(varargin{1}); - gpuIDX = EMC_str2double(varargin{2}); -end - tomoNumber = EMC_str2double(tomoNumber); - - - [ useGPU ] = BH_multi_checkGPU( gpuIDX ) - - - - -gpuDevice(useGPU); - -SYMMETRY = EMC_str2double(SYMMETRY); -startTime = clock ; - -emc = BH_parseParameterFile(PARAMETER_FILE); -try - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR -% clear subTomoMeta - % Make sure we get a CTF corrected stack - shouldBeCTF = 1 -catch - mapBackIter = 0; - shouldBeCTF = -1 -end -samplingRate = emc.('Tmp_samplingRate'); - -try - tmpDecoy = emc.('templateDecoy') -catch - tmpDecoy = 0 -end - -try - scale_mip = emc.('scale_mip'); -catch - scale_mip = false; -end - -try - max_tries = emc.('max_peaks'); -catch - max_tries = 10000; -end - -if ( cmdLineThresh ) - peakThreshold = cmdLineThresh; - fprintf('\nOverride peakThreshold from paramfile (%d) with cmd line arg (%d)\n\n',... - cmdLineThresh, emc.('Tmp_threshold')); -else - peakThreshold = emc.('Tmp_threshold'); -end - -latticeRadius = emc.('particleRadius'); -try - targetSize = emc.('Tmp_targetSize') -catch - targetSize = [512,512,512]; -end -angleSearch = emc.('Tmp_angleSearch'); - -statsRadius = 1; - -convTMPNAME = sprintf('convmap_wedgeType_%d_bin%d',wedgeType,samplingRate) - -try - eraseMaskType = emc.('Peak_mType'); -catch - eraseMaskType = 'sphere'; -end -try - eraseMaskRadius = emc.('Peak_mRadius'); -catch - eraseMaskRadius = 0.75.*latticeRadius; -end - - -nPreviousSubTomos = 0; - -reconScaling = 1; -try - nPeaks = emc.('nPeaks'); -catch - nPeaks = 1; -end - -pixelSizeFULL = emc.('PIXEL_SIZE').*10^10; -if emc.('SuperResolution') - pixelSizeFULL = pixelSizeFULL * 2; -end - -pixelSize = pixelSizeFULL.*samplingRate; - - - - -try - wantedCut = emc.('lowResCut'); -catch - wantedCut = 28; -end - -TLT = load(sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tomoName,mapBackIter+1)); -def = mean(-1.*TLT(:,15))*10^6; %TODO if you switch to POSITIVEDEFOCUS this will be wrong -firstZero = -0.2*def^2 +5.2*def +11; - -% Take the lower of firstZero lowResCut or Nyquist -lowResCut = max(wantedCut, firstZero); - -if pixelSize*2 > lowResCut - fprintf('\nLimiting to Nyquist (%f) instead of user requested lowResCut %f Angstrom\n',pixelSize*2,lowResCut); - lowResCut = pixelSize*2; -else - fprintf('\nUsing max (%f) of specified resolution cutoff of %f and first ctf zero %f Angstrom\n',lowResCut, wantedCut, firstZero); -end - - - -mapPath = './cache'; -mapName = sprintf('%s_%d_bin%d',tomoName,tomoNumber,samplingRate); -mapExt = '.rec'; - -sprintf('recon/%s_recon.coords',tomoName) -[ recGeom, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName) ); - -reconCoords = recGeom(tomoNumber,:); -clear recGeom - - -[ tomogram ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... - reconCoords, mapBackIter, samplingRate,... - shouldBeCTF*gpuIDX, reconScaling,1); - - -% We'll handle image statistics locally, but first place the global environment -% into a predictible range - - - -[template, tempPath, tempName, tempExt] = ... - BH_multi_loadOrBin( TEMPLATE, 1, 3 ); - - - - - -% The template will be padded later, trim for now to minimum so excess -% iterations can be avoided. -fprintf('size of provided template %d %d %d\n',size(template)); -trimTemp = BH_multi_padVal(size(template),ceil(2.*max(emc.('Ali_mRadius')./pixelSizeFULL))); -template = BH_padZeros3d(template, trimTemp(1,:),trimTemp(2,:),'cpu','singleTaper'); -SAVE_IMG(MRCImage(template),'template_trimmed.mrc'); -clear trimTemp -fprintf('size after trim to sqrt(2)*max(lattice radius) %d %d %d\n',size(template)); - -if isempty(mapPath) ; mapPath = '.' ; end -if isempty(tempPath) ; tempPath = '.' ; end -% Check to see if only tilt angles are supplied, implying a y-axis tilt scheme, -% or otherwise, assume a general geometry as in protomo. -% % % tiltGeometry = load(RAWTLT); -RAWTLT = sprintf('fixedStacks/ctf/%s_ali1_ctf.tlt',tomoName); -tiltGeometry = load(RAWTLT); -% subTomoMeta.('tiltGeometry').(mapName) = tiltGeometry; - -% Make sure the template and is an even sized image -template = padarray(template, mod(size(template),2),0, 'post'); -template = template - mean(template(:)); - -templateBIN = BH_reScale3d(template,'',sprintf('%f',1/samplingRate),'cpu'); -templateBIN = templateBIN - mean(templateBIN(:)); -templateBIN = templateBIN ./rms(templateBIN(:)); - - -sizeTemp = size(template) -sizeTempBIN = size(templateBIN) - - - -statsRadiusAng = statsRadius.*[2,2,2].*max(latticeRadius); -statsRadius = ceil(statsRadiusAng./pixelSize); -latticeRadius = (0.75 .* latticeRadius) ./ (pixelSize); -latticeRadius = floor(latticeRadius); -latticeRadius = latticeRadius + mod(latticeRadius, 2); - -eraseMaskRadius = floor((eraseMaskRadius) ./ (pixelSize)); -eraseMaskRadius = eraseMaskRadius + mod(eraseMaskRadius,2); - -fprintf('\ntomograms normalized in %f Angstrom cubic window\n',statsRadiusAng(1)); - -fprintf('\nlatticeRadius = %dx%dx%d pixels\n\n', latticeRadius); -fprintf('\neraseMaskType %s, eraseMaskRadius %dx%dx%d pixels\n',eraseMaskType,eraseMaskRadius); - % For wedgeMask -particleThickness = latticeRadius(3); - -gpuDevice(useGPU); - - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -% Initialize a whole mess of control variables and storage volumes. % -%Out of plane range inc (starts from 1.* inc) -rotConvention = 'Bah'; -% Check and override the rotational convention to get helical averaging. -% Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = emc.('doHelical'); -catch - doHelical = 0; -end -if ( doHelical ) - rotConvention = 'Helical' -end - -rotConvention - -[ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch) - - - -[ OUTPUT ] = BH_multi_iterator( [targetSize; ... - size(tomogram);... - sizeTempBIN; ... - 2.*latticeRadius], 'convolution' ); - - - -tomoPre = OUTPUT(1,:); -tomoPost = OUTPUT(2,:); -sizeChunk = OUTPUT(3,:); -validArea = OUTPUT(4,:); -validCalc = OUTPUT(5,:); -nIters = OUTPUT(6,:); - -%[ padVal ] = BH_multi_padVal( sizeTemp, sizeChunk ); -%tempPre = padVal(1,:); -%tempPost = padVal(2,:); - -[ padBIN ] = BH_multi_padVal( sizeTempBIN, sizeChunk ); -[ trimValid ] = BH_multi_padVal(sizeChunk, validArea); - -if ( tmpDecoy ) - % This is probably sample dependent. should search a small range and find - % the maximum rate of change in the ccc - - % the -1 searches for the next smallest fast fourier size - templateBIN = gpuArray(templateBIN); - - - - decoyTest = BH_reScale3d(templateBIN,'',tmpDecoy,'GPU'); - decoyTrim = BH_multi_padVal(size(decoyTest),size(templateBIN)); - decoyTest = fftn(BH_padZeros3d(decoyTest,decoyTrim(1,:),decoyTrim(2,:),'GPU','single')); - decoyShift = -1.*gather(BH_multi_xcf_Translational(decoyTest,conj(fftn(templateBIN)),'',[3,3,3])); - decoyNorm = gather(sum(abs(decoyTest(:)))./sum(abs(fftn(templateBIN(:))))); - padDecoy = BH_multi_padVal(size(decoyTest),sizeChunk) + decoyTrim; - clear decoyTest - templateBIN = gather(templateBIN); - fprintf('tmpDecoy %f normFactor %f and shift by %2.2f %2.2f %2.2f\n',tmpDecoy,decoyNorm,decoyShift); - -end - - -fprintf('\n-----\nProcessing in chunks\n\n'); -fprintf('tomo prepadding %d %d %d\n', tomoPre); -fprintf('tomo postpadding %d %d %d\n', tomoPost); -fprintf('size to process %d %d %d\n', sizeChunk); -fprintf('valid Area %d %d %d\n', validArea); -fprintf('valid Calc %d %d %d\n', validCalc); -fprintf('# of iterations %d %d %d\n', nIters); -fprintf('-----\n'); - -size(tomogram) - -% [ tomogram ] = BH_padZeros3d(tomogram, tomoPre, tomoPost, ... -% 'cpu', 'singleTaper'); -tomogram = padarray(tomogram,tomoPre,'symmetric','pre'); -tomogram = padarray(tomogram,tomoPost,'symmetric','post'); -sizeTomo = size(tomogram); - - -[ validAreaMask ] = gather(BH_mask3d('rectangle',sizeChunk,validCalc./2,[0,0,0])); -[ vA ] = BH_multi_padVal( validArea, sizeChunk ); -% This would need to be changed to take a mask size and not just a radius. -% Currently, this would not produce the correct results for odd size area -% % % fftMask = BH_fftShift(validArea,sizeChunk,0); - -% Array for storing chunk results -RESULTS_peak = zeros(sizeTomo, 'single'); -RESULTS_angle= zeros(sizeTomo, 'single'); - -if (scale_mip) - RESULTS_sum = zeros(sizeTomo,'single'); - RESULTS_sum_sq = zeros(sizeTomo,'single'); -end - -if ( tmpDecoy ) - RESULTS_decoy = RESULTS_peak; -end -% Loop over tomogram -% Set this up second - - -% optimize fft incase a power of two is not used, this will make things run ok. -opt = zeros(sizeChunk, precision,'gpuArray'); -fftw('planner','patient'); -fftn(opt); -clear opt ans - -% Temp while testing new dose weighting -TLT = tiltGeometry; -nPrjs = size(TLT,1); - - -kVal = 0; - -% % [ OUTPUT ] = BH_multi_iterator( [sizeTempBIN;kVal.*[1,1,1]], 'extrapolate' ); -[ OUTPUT ] = BH_multi_iterator( [sizeChunk;kVal.*[1,1,1]], 'extrapolate' ); - - - -switch wedgeType - case 1 - % make a binary wedge - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'binaryWedgeGPU',particleThickness,... - 1, 1, samplingRate); - case 2 - % make a non-CTF wedge - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 2, 1, samplingRate); - case 3 - % make a CTF without exposure weight - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 3, 1, samplingRate); - case 4 - % make a wedge with full-ctf - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 4, 1, samplingRate); - otherwise - error('wedgeType must be 1-4'); -end - -wedgeMask = (ifftshift(wedgeMask)); -% -% % Now just using the mask to calculate the power remaining in the template, -% % without actually applying. -% wedgeMask = gather(find(ifftshift(wedgeMask))); - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Preprocess the tomogram - -tomoIDX = 1; -nTomograms = prod(nIters); - - -tomoStack = zeros([sizeChunk,nTomograms], 'single'); - -% backgroundVol = zeros(sizeChunk,'single'); -tomoCoords= zeros(nTomograms, 3, 'uint16'); - -% % % [ tomoBandpass ] = BH_bandpass3d(sizeChunk, 0,maxSizeForHighPass, ... -% % % lowResCut,'cpu', pixelSize ); -% In switching to the full 3D-sampling function the high pass is -% already incorporated in the CTF. Still include one for very low -% resolution to deal with gradients in the tomos. -[ tomoBandpass ] = BH_bandpass3d(sizeChunk, 10e-4,800, ... - lowResCut,'cpu', pixelSize ); - -% if ~(shouldBeCTF) -% tomoBandpass = wedgeMask .* tomoBandpass; -% end - -clear wedgeMask - -try - doMedFilt = emc.('Tmp_medianFilter'); - if ~ismember(doMedFilt,[3,5,7]) - error('Tmp_medianFilter can only be 3,5, or 7'); - else - fprintf('Using median filter, size %d',doMedFilt); - end -catch - doMedFilt =0 -end - -calcStats = 0; -if calcStats - maskStack = false([sizeChunk,nTomograms]); - calcMask = 0; -else - calcMask = 1; -end -firstStats = 1; -flgOOM = 0; - -fullX = 0; -fullX2 = 0; -fullnX = 0; - -for iX = 1:nIters(1) - cutX = 1 + (iX-1).*validArea(1); - for iY = 1:nIters(2) - cutY = 1 + (iY-1).*validArea(2); - for iZ = 1:nIters(3) - cutZ = 1 + (iZ-1).*validArea(3); - - fprintf('preprocessing tomo_chunk %d/%d col %d/%d row %d/%d plane idx%d\n' , ... - iY,nIters(2),iX,nIters(1),iZ,nIters(3),tomoIDX) - - - % Cut out chunk and zero pad - double would be more accurate, but for - % template matching which is fairly crude anyhow, this should be okay, and - % allows much larger chunks to be processed. - - - tomoChunk = tomogram(cutX:cutX+sizeChunk(1)-1,... - cutY:cutY+sizeChunk(2)-1,... - cutZ:cutZ+sizeChunk(3)-1); - - - tomoChunk = real(ifftn(fftn(tomoChunk).*tomoBandpass)); - - - - if doMedFilt - if ( flgOOM ) - tomoChunk = (medfilt3(tomoChunk,doMedFilt.*[1,1,1])); - else - tomoChunk = gpuArray(medfilt3(tomoChunk,doMedFilt.*[1,1,1])); - end - else - if ( flgOOM ) - % Leave on CPU - else - tomoChunk = gpuArray(tomoChunk); - statsRadius = gather(statsRadius); - end - end - - % Handle all mean centering and rms normalization in local window - - [ averageMask, flgOOM ] = BH_movingAverage(tomoChunk, statsRadius); - - if isa(tomoChunk(1),'gpuArray') && flgOOM - tomoChunk = gather(tomoChunk); - end - - - - tomoChunk= tomoChunk - averageMask; clear averageMask - - [ rmsMask ] = BH_movingRMS(tomoChunk, statsRadius); - - - - if ( shouldBeCTF == 1 ) - tomoStack(:,:,:,tomoIDX) = gather((tomoChunk ./ rmsMask).*validAreaMask); - else - % Using the non-ctf corrected stack since we limit toA all practical - % defocus (<8um) should be entirely negative, so just flip in real - % space - - tomoStack(:,:,:,tomoIDX) = gather(-1.*(tomoChunk ./ rmsMask).*validAreaMask); -% backgroundVol = backgroundVol + gather(tomoChunk.*maskStack(:,:,:,tomoIDX)); - end - - clear rmsMask - - - fullX = fullX + sum(sum(sum(tomoStack(:,:,:,tomoIDX)))); - fullX2 = fullX2 + sum(sum(sum(tomoStack(:,:,:,tomoIDX).^2))); - fullnX = fullnX + prod(sizeChunk); - - tomoCoords(tomoIDX,:) = [cutX,cutY,cutZ]; - tomoIDX = tomoIDX + 1; - - - end % end of loop over Z chunks - end % end of loop over Y chunks -end % end of loop over X chunks - -% Normalize the global variance -globalVariance = (fullX2 - fullX)/fullnX; -fprintf('After local normalization, scaling also the global variance\n'); - -for iChunk = 1:tomoIDX-1 - tomoStack(:,:,:,iChunk) = tomoStack(:,:,:,iChunk) ./ globalVariance; -end - - - -clear tomoWedgeMask averagingMask rmsMask bandpassFilter statBinary validAreaMask tomoChunk - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -kVal = 0; - -[ OUTPUT ] = BH_multi_iterator( [sizeTempBIN;kVal.*[1,1,1]], 'extrapolate' ); - - -switch wedgeType - case 1 - % make a binary wedge - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'binaryWedgeGPU',particleThickness,... - 1, 1, samplingRate); - case 2 - % make a non-CTF wedge - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 2, 1, samplingRate); - case 3 - % make a CTF without exposure weight - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 3, 1, samplingRate); - case 4 - % make a wedge with full-ctf - [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... - 'applyMask',particleThickness,... - 4, 1, samplingRate); - otherwise - error('wedgeType must be 1-4'); -end - - - - -% -% % Now just using the mask to calculate the power remaining in the template, -% % without actually applying. -% wedgeMask = gather(find(ifftshift(wedgeMask))); - - - - - - - -currentGlobalAngle = 1; -ANGLE_LIST = zeros(nAngles(1),3, 'single'); - -highThr=sqrt(2).*erfcinv(ceil(peakThreshold.*0.025).*2./(numel(tomogram).*nAngles(1))) - -nComplete = 0; -totalTime = 0; -firstLoopOverTomo = true; -for iAngle = 1:size(angleStep,1) - - theta = angleStep(iAngle,1); - - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. - if (doHelical) - phi_step = 360; - else - phiStep = angleStep(iAngle,3); - end - phiStep = angleStep(iAngle,3); - - gpuDevice(useGPU); - - %numRefIter = nAngles(1); - numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - tempImg = gpuArray(templateBIN); %%%%% NEW switch to bin -% tempWdg = gpuArray(wedgeMask); -% tempBnd = gpuArray(tempBandpass); - tempBandpass = gpuArray(tomoBandpass); - - interpolationNormFactor = sum(abs(tempImg(:)).^2); - - - - clear referenceStack tempFilter - % Calculate all references for each out of plane tilt only once - referenceStack = zeros([sizeTempBIN,numRefIter], 'single', 'gpuArray'); - - - - tomoIDX = 1; - firstLoopOverAngle = true; - % Iterate over the tomogram pulling each chunk one at a time. - for iTomo = 1:nTomograms - tic; - iCut = tomoCoords(tomoIDX,:); - % reset the angle count and value at the begining of loop - % inside, while each new outer loop changes the start values. - -% nAngle = angleIncStart; - intraLoopAngle = 1; - - % Truth value to initialize temp results matrix each new tomo - % chunk. - firstLoopOverChunk = true; - - fprintf('working on tilt(%d/%d) tomoChunk(idx%d/%d)\t' ... - ,iAngle,size(angleStep,1), tomoIDX,nTomograms); - - - % fftn(double(gpuArray))) ~ 2.5x faster than transfering a double - % complex - if strcmpi(precision, 'double') - tomoFou = fftn(double(gpuArray(tomoStack(:,:,:,tomoIDX)))); - else - tomoFou = fftn(gpuArray(tomoStack(:,:,:,tomoIDX))); - end - for iAzimuth = 0:angleStep(iAngle,2) - - if ( doHelical ) - phi = 90 ; - else - phi = phiStep * iAzimuth; - end - - for iInPlane = inPlaneSearch - psi = iInPlane; - - %calc references only on first chunk - if (firstLoopOverAngle) - - ANGLE_LIST(currentGlobalAngle,:) = [phi, theta, psi - phi]; - [phi, theta, psi - phi]; - % Rotate the reference, lowpass and wedge mask, send to gpu - % Inverse rotation(i.e. rotate particle, angles saved - % are to rotate frame to particle for extraction.) -% % % % % tempRot = BH_resample3d(tempImg, [phi, theta, psi - phi], [1,1,1], ... -% % % % % {'Bah', 1,'linear',1,interpMaskGPU},... -% % % % % 'GPU','forward'); - - - tempRot = BH_resample3d(tempImg, [phi, theta, psi - phi], [1,1,1], ... - {'Bah', 1,'linear',1},... - 'GPU','forward'); - - - - %%%%%tempFou = BH_bandLimitCenterNormalize(tempRot,tempWedgeMask,'',[tempPre;tempPost],precisionTaper); - - %%%%%tempRot = BH_padZeros3d(real(ifftn(tempFou)),-1.*tempPre,-1.*tempPost,'GPU',precision); - - %%%%%tempRot = gather(BH_reScale3d(tempRot,'',sprintf('%f',1/samplingRate),'GPU')); - - % if (firstLoopOverTomo) - % SAVE_IMG(MRCImage(tempRot), sprintf('temp_%s.mrc',convTMPNAME),pixelSize); - % end - - % First correct for any change in power due to - % rotation/interpolation - tempRot = tempRot .* (interpolationNormFactor./sum(abs(tempRot(:)).^2)); - % Then correct for any change in power due to the wedge. These can be combined -% normFT = abs(fftn(tempRot).*tempBnd).^2; -% % -% % -% normScore = sum(normFT(:)) ./ sum(normFT(:).*tempWdg(:)); -% clear normFT; -% tempRot = tempRot .* normScore; - %clear normScore - - referenceStack(:,:,:,intraLoopAngle) = tempRot; - -% % % % % tempFou = fftn(BH_padZeros3d(tempRot,padBIN(1,:),padBIN(2,:),'GPU',precision)); - - tempFou = BH_bandLimitCenterNormalize( tempRot, tempBandpass, '', ... - padBIN, 'single' ); - - - else - -% % % % % tempFou = (fftn(BH_padZeros3d( ... -% % % % % referenceStack(:,:,:,intraLoopAngle), ... -% % % % % padBIN(1,:), padBIN(2,:),'GPU', precision))); - - - tempFou = BH_bandLimitCenterNormalize( referenceStack(:,:,:,intraLoopAngle), tempBandpass, '', ... - padBIN, 'single' ); - - end - - -% % % ccfmapFull = fftshift(real(single(ifftn(tomoFou.*conj(tempFou))))); -% % % -% % % -% % % ccfmap = ccfmapFull(vA(1,1) + 1:end - vA(2,1), ... -% % % vA(1,2) + 1:end - vA(2,2), ... -% % % vA(1,3) + 1:end - vA(2,3)); - - % Even with local normalization, test with all padding and - % goodness. -% tomoNorm = ((sqrt(sum(sum(sum(abs(tomoFou).^2)))) ./ numel(tomoFou))); -% tempNorm = ((sqrt(sum(sum(sum(abs(tempFou).^2)))) ./ numel(tempFou))); - -% ./(sum(sum(sum((abs(tomoFou).*abs(tempFou)).^2)))) - ccfmap = BH_padZeros3d(fftshift(real(single(... - ifftn(tomoFou.*conj(tempFou) )))),...%./(tomoNorm.*tempNorm))))),... - trimValid(1,:),trimValid(2,:),'GPU',precision); -% -% Since the vast majority of ccf values are known to not be due to -% the target, they are noise. Try just normalizing the StdDev of -% everything assuming this is the noise. - ccfmap = ccfmap ./ std(ccfmap(:)); - - - - if ( tmpDecoy > 0 ) - tempFou = []; - if (firstLoopOverAngle) - - decoy = BH_padZeros3d(BH_reScale3d(tempRot./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - padDecoy(1,:),padDecoy(2,:),'GPU','single'); - else - % Probably just make a second decoy stack to avoid - % re-interpolating. If it works, then do this. - decoy = BH_padZeros3d(BH_reScale3d(referenceStack(:,:,:,intraLoopAngle)./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - padDecoy(1,:),padDecoy(2,:),'GPU','single'); - end - - - - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*conj(fftn(decoy)))))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU',precision); - - - elseif ( tmpDecoy < 0 ) - - % Just use the mirror image of the template, i.e. take the conj - % (of the conj) so just the padded FFT of the ref. - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*tempFou)))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU',precision); - tempFou = []; - end - clear tempRot - % If first loop over tomo, initialize the storage volumes, if - % first loop over the chunk but not over the tomo, pull storage - % chunks from storage volume. - if (firstLoopOverTomo && firstLoopOverChunk) - %store ccfmap as complex with phase = angle of reference - magTmp = ccfmap; - if ( tmpDecoy ) - decoyTmp = decoy; - end - angTmp = zeros(size(magTmp), 'single','gpuArray'); - angTmp = angTmp + 1; - - if (scale_mip) - sumTmp = ccfmap; - sumSqTmp = ccfmap.^2; - end - firstLoopOverTomo = false; - firstLoopOverChunk = false; - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - - elseif (firstLoopOverChunk) - % These double cuts are old, and don't really make sense. Make - % this more consistant with current operations when there is - % time. - magTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - if ( tmpDecoy ) - decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyTmp = gpuArray(decoyTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - - magTmp = gpuArray(magTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - - firstLoopOverChunk = false; - - replaceTmp = ( magTmp < ccfmap ); - - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - - if (scale_mip) - - sumTmp = RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sumSqTmp = RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sumTmp = gpuArray(sumTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - sumSqTmp = gpuArray(sumSqTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - end - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp - - else - % update higher values of ccfmap with new reference if applicable. - - if (scale_mip) - sumTmp = sumTmp + ccfmap; - sumSqTmp = sumSqTmp + ccfmap.^2; - end - - replaceTmp = ( magTmp < ccfmap ); - - - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - if ( tmpDecoy ) - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp - end - nComplete = nComplete + 1; - - end - end - - % After searching all angles on this chunk, but out meaningful - % portion for storage. - - magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - - - magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); - angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); - - - RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; - - clear magStoreTmp - - RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; - clear angStoreTmp - - if (scale_mip) - sumStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sumSqStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - - - sumStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sumTmp); - sumSqStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sumSqTmp); - - - RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sumStoreTmp; - - clear sumStoreTmp - - RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sumSqStoreTmp; - clear sumSqStoreTmp - end - - if ( tmpDecoy ) - decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(decoyTmp); - RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = decoyStoreTmp; - - end - tomoTime = toc; - totalTime = totalTime + toc; timeEstimate = totalTime * (nTomograms*nAngles(1)./(nComplete-1)); - fprintf('elapsed time = %f s est remain %f s\n', tomoTime, timeEstimate); - tomoIDX = tomoIDX + 1; - firstLoopOverAngle = false; - currentGlobalAngle = currentGlobalAngle - intraLoopAngle + 1; - end - - currentGlobalAngle = currentGlobalAngle + intraLoopAngle - 1; -end -%save('angle_list.txt','angle_list','-ascii'); -clear tomoStack -% Cut out the post padding used to iterate over the tomogram -RESULTS_peak = RESULTS_peak(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); -%RESULTS_peak(RESULTS_peak < 0) = 0; -RESULTS_angle = RESULTS_angle(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); - -if (scale_mip) - RESULTS_sum = RESULTS_sum(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)) ./ nComplete; - - RESULTS_sum_sq = RESULTS_sum_sq(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)) ./ nComplete; -end - - -if ( tmpDecoy ) - RESULTS_decoy = RESULTS_decoy(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); -% RESULTS_decoy = RESULTS_decoy ./ std(RESULTS_decoy(:)); - RESULTS_decoy(RESULTS_decoy < 1) = 1; - -end -gpuDevice(useGPU); - - -% scale the magnitude of the results to be 0 : 1 -szK = latticeRadius;%floor(0.8.*szM); -rmDim = max(max(eraseMaskRadius),max(szK)).*[1,1,1]; -mag = RESULTS_peak; clear RESULTS_peak -% Normalize so the difference if using a decoy makes sense. The input decoy -% should have the same power, so I'm not sure why this is needed, but it is -% an easy fix and a problem for future Ben to figure out. -% mag = mag ./ std(mag(:)); - -system(sprintf('mkdir -p %s',convTMPNAME)); -system(sprintf('mv temp_%s.mrc %s',convTMPNAME,convTMPNAME)); - -resultsOUT = sprintf('./%s/%s_convmap.mrc',convTMPNAME,mapName); -anglesOUT = sprintf('./%s/%s_angles.mrc',convTMPNAME,mapName); -angleListOUT = sprintf('./%s/%s_angles.list',convTMPNAME,mapName); -SAVE_IMG(MRCImage(mag),resultsOUT); -SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); - -if (scale_mip) - scaledMipOUT = sprintf('./%s/%s_convmap_scaled.mrc',convTMPNAME,mapName); - VarEst = RESULTS_sum_sq - RESULTS_sum.^2; - nonZero = abs(VarEst(:)) > 1e-3; - SAVE_IMG(MRCImage(RESULTS_sum),sprintf('./%s/%s_convmap_sum.mrc',convTMPNAME,mapName)); - SAVE_IMG(MRCImage(RESULTS_sum_sq),sprintf('./%s/%s_convmap_sumSq.mrc',convTMPNAME,mapName)); - SAVE_IMG(MRCImage(VarEst),sprintf('./%s/%s_convmap_varEst.mrc',convTMPNAME,mapName)); - - VarEst(nonZero) = ( mag(nonZero) - RESULTS_sum(nonZero) ) ./ VarEst(nonZero); - SAVE_IMG(MRCImage(VarEst),scaledMipOUT); -end - -if ( tmpDecoy ) - decoyOUT = sprintf('./%s/%s_decoy.mrc',convTMPNAME,mapName); - SAVE_IMG(MRCImage((RESULTS_decoy)),decoyOUT); - diffOUT = sprintf('./%s/%s_convmap-decoy.mrc',convTMPNAME,mapName); - decoyLogical = mag < RESULTS_decoy; - mag(decoyLogical) = 0; - mag(~decoyLogical) = mag(~decoyLogical) - RESULTS_decoy(~decoyLogical); clear RESULTS_decoy - SAVE_IMG(MRCImage((mag)),diffOUT); -end -angleFILE = fopen(angleListOUT,'w'); -fprintf(angleFILE,'%2.2f\t%2.2f\t%2.2f\n', ANGLE_LIST'); -fclose(angleFILE); - - -% mag = mag - min(mag(:)); mag = mag ./ max(mag(:)); - -% Zero out one lattice width from the edges to reduce edge effect (but cutting -% out and padding back in.) Also pad by size of removal mask (subtract this from -% coordinates) -mag = mag(szK(1)+1:end - szK(1), ... - szK(2)+1:end - szK(2), ... - szK(3)+1:end - szK(3)); -mag = BH_padZeros3d(mag,szK+rmDim,szK+rmDim, 'cpu', 'single'); -%dev.FreeMemory; -%%%Ang = angle(RESULTS_peak); %clear Results -% negative phase angles mapped back to 0-->pi -%Ang(sign(Ang) < 0) = Ang(sign(Ang)<0) + pi; serotonin_ali1_75_1.mod -%Ang = BH_padZeros3d(round(Ang./angleIncrement),szK,szK,'cpu','single'); -Ang = BH_padZeros3d(RESULTS_angle,rmDim,rmDim,'cpu','single'); - - -%mag = mag - min(mag(:)); mag = mag ./ max(mag(:)); - -Tmean = mean(mag(( mag ~= 0 ))); -Tstd = std(mag(( mag~=0 ))); -threshold = Tmean + peakThreshold*Tstd; -mag((Ang < 0)) = 0; - -mag = gpuArray(mag); -sizeTomo = size(mag); - - -[MAX, coord] = max(mag(:)); - -peakMat = zeros(peakThreshold,10*nPeaks); - -n = 1; - -fprintf('rmDim %f szK %f\n', rmDim,szK); -removalMask = BH_mask3d(eraseMaskType,[2,2,2].*rmDim+1,eraseMaskRadius,[0,0,0]); - -maskCutOff = 0.999; -nIncluded = gather(sum(sum(sum(removalMask > maskCutOff)))); -nTries = 0; -if strcmpi(eraseMaskType,'rectangle') - areaPreFactor = 0; -else - areaPreFactor = (4/3*pi); -end - -while nIncluded < areaPreFactor*prod(eraseMaskRadius) - maskCutOff = 0.99*maskCutOff; - nIncluded = gather(sum(sum(sum(removalMask > maskCutOff)))); - nTries = nTries + 1; - if (nTries > 1000) - error('Did not find an appropriate erase mask'); - end - -end -this_try = 0; -while n <= peakThreshold && (this_try < max_tries) -this_try = this_try + 1; - -% -% Some indicies come back as an error, even when they seem like the -% should be fine. I'm not sure why, and I should think about this -% more, but for now, just set that one index to zero (instead of a -% whole box) and move on with life. It looks like the index that is -% kicking out the error is equal to -1*numberofreferences, which -% might be an issue because that corresonds to the positive upper -% limit of the reference index. Ignoring it still seems to be okay -% but it bothers me not to know. - - -[i,j,k] = ind2sub(sizeTomo,coord); -try - c = gather([i,j,k]); -catch - print('Ran into some trouble gathering the i,j,k. Breaking out\n'); - break -end - - if Ang(gather(coord)) > 0 - - % box for removal and center of mass calc, use a larger box if multiple - % peaks are being saved. - bDist = 1+round(log(nPeaks)); - clI = c(1) - bDist; - chI = c(1) + bDist; - clJ = c(2) - bDist; - chJ = c(2) + bDist; - clK = c(3) - bDist; - chK = c(3) + bDist; - - magBox = mag(clI:chI,clJ:chJ,clK:chK); - - angBox = Ang(clI:chI,clJ:chJ,clK:chK); - - [cmX, cmY, cmZ] = ndgrid(-1*bDist:1*bDist, ... - -1*bDist:1*bDist, ... - -1*bDist:1*bDist ); - - cMass = [ sum(sum(sum(magBox.*cmX))) ; ... - sum(sum(sum(magBox.*cmY))) ; ... - sum(sum(sum(magBox.*cmZ))) ] ./ sum(magBox(:)); - - - -% cenP = [ (c(1)+cMass(1)-1) - sizeTomo(1)./2 ,... -% (c(2)+cMass(2)-1) - sizeTomo(2)./2 ,... -% (c(3)+cMass(3)-1) - sizeTomo(3)./2 ]; - - % Switching from centered to lower left coordinates and subtracting the - % padding - - cenP = c + cMass' - rmDim; - - - - % If the most frequent peak is unique use it; - [peakM, ~, peakC] = mode(angBox(:)); - if length(peakC) == 1 && peakM - % Need to ensure the mode is none zero which is possible. - peakMat(n,4:6) = ANGLE_LIST(peakM,:); - topPeak = peakM; - else - % Otherwise use the value at the max for the peak val; - peakMat(n,4:6) = ANGLE_LIST(Ang(coord),:); - topPeak = Ang(coord); - end - peakMat(n,1:3) = gather(samplingRate.*cenP); - - if nPeaks > 1 - oldPeaks = ( angBox == topPeak ); - - for iPeak = 2:nPeaks - [peakM, ~, ~] = mode(angBox(~oldPeaks)); - % There could be redundancy, as given by peakC, but just take the - % first value given by peak M. - peakMat(n,[1:3]+10*(iPeak-1)) = gather(samplingRate.*cenP); - peakMat(n,[4:6]+10*(iPeak-1)) = ANGLE_LIST(peakM,:); - - oldPeaks = ( angBox == peakM | oldPeaks ); - - end - - end - - - - - rmMask = BH_resample3d(removalMask,peakMat(n,4:6),[0,0,0],'Bah','GPU','forward'); - % Invert after resampling so that zeros introduced by not extrapolating - % the corners are swapped to ones, i.e. not removed. -% rmMask = (1-rmMask); - - mag(c(1)-rmDim:c(1)+rmDim,... - c(2)-rmDim:c(2)+rmDim,... - c(3)-rmDim:c(3)+rmDim) = ... - mag(c(1)-rmDim:c(1)+rmDim,... - c(2)-rmDim:c(2)+rmDim,... - c(3)-rmDim:c(3)+rmDim) .* (rmMask< maskCutOff); - - peakMat(n,10) = (gather(MAX) - Tmean)./Tstd; % record stds above mean - n = n + 1; - - if ~mod(n,100) - n - end - - else - Ang(gather(coord)); - mag(coord) = 0; - end - - -[MAX, coord] = max(mag(:)); - -end - -peakMat = peakMat( ( peakMat(:,1)>0 ),:); - -%save('peakMat_post.mat', 'peakMat'); - -% A temp test, not the correct output just score x y z dx dy dz e1 e2 e3 - -csv_out = sprintf('./%s/%s.csv',convTMPNAME,mapName); -pos_out = sprintf('./%s/%s.pos',convTMPNAME,mapName); -%fieldOUT = zeros(length(peakMat(:,1)),26); -fileID = fopen(csv_out,'w'); -fileID2 = fopen(pos_out,'w'); -errID = fopen(sprintf('./%s/%s.errID',convTMPNAME,mapName)); - - -if SYMMETRY > 1 - symmetry = 0:360/SYMMETRY:359; - symCell = cell(length(symmetry),1); - for iSym = 1:length(symmetry) - symCell{iSym} = BH_defineMatrix([symmetry(iSym),0,0], 'Bah', 'inv'); - end - -end -n=1 -for i = 1:length(peakMat(:,1)) - if all(peakMat(i,1:3)) - - if SYMMETRY > 1 - % Generate a uniform distribution over the in-plane - % randomizations - iSym = rem( n + SYMMETRY, SYMMETRY)+1; - r = reshape(BH_defineMatrix(peakMat(i,4:6), 'Bah', 'inv')*... - symCell{iSym},1,9); - else - r = reshape(BH_defineMatrix(peakMat(i,4:6), 'Bah', 'inv'),1,9); - end - fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... - peakMat(i,4:6),r,1); - - if nPeaks > 1 - - for iPeak = 2:nPeaks - if SYMMETRY > 1 - % Generate a uniform distribution over the in-plane - % randomizations - iSym = rem( n + SYMMETRY, SYMMETRY)+1; - r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), 'Bah', 'inv')*... - symCell{iSym},1,9); - else - r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), 'Bah', 'inv'),1,9); - end - fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,[1:3]+10*(iPeak-1)), ... - peakMat(i,[4:6]+10*(iPeak-1)),r,1); - end - - - end - - fprintf(fileID,'\n'); - - - - - fprintf(fileID2,'%f %f %f\n',peakMat(i,1:3)./samplingRate); - - - n = n +1; - end -end - -%lastIndex = find(fieldOUT(:,4),1,'last'); - -fclose(fileID); -fclose(fileID2); - -system(sprintf('point2model -number 1 -sphere 3 -scat ./%s/%s.pos ./%s/%s.mod', convTMPNAME,mapName,convTMPNAME, mapName)); - -fileID = fopen(sprintf('./%s/%s.path',convTMPNAME,mapName),'w'); -fprintf(fileID,'%s,%s,%s,%s',mapName,mapPath,mapExt,RAWTLT); -fclose(fileID); -%subTomoMeta.('cycle000').('geometry').(mapName) = fieldOUT; -% subTomoMeta.('mapPath').(mapName) = mapPath; -% subTomoMeta.('mapExt').(mapName) = mapExt; - -% if any(ismember(fieldnames(subTomoMeta), 'nSubTomoTotal')) -% subTomoMeta.('nSubTomoTotal') = subTomoMeta.('nSubTomoTotal') + lastIndex; -% else -% subTomoMeta.('nSubTomoTotal') = lastIndex; -% end - -% preFscSplit = gather(subTomoMeta); -% -% % Randomly divide the data into half sets. -% [ subTomoMeta ] = BH_fscSplit( preFscSplit ); -% subTomoMeta.('currentCycle') = 0; - -% save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); -% save(sprintf('./convmap/%s.mat~', emc.('subTomoMeta')), 'subTomoMeta'); -%save('test.pos','a','-ascii'); - - -fprintf('Total execution time : %f seconds\n', etime(clock, startTime)); - - - -end % end of templateSearch3d function - diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index d68f75cc..ba36e6a5 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -131,11 +131,7 @@ nPreviousSubTomos = 0; reconScaling = 1; -try - nPeaks = emc.('nPeaks'); -catch - nPeaks = 1; -end + ignore_threshold = false; try @@ -1197,7 +1193,7 @@ [MAX, coord] = max(mag(:)); -peakMat = zeros(peakThreshold,10*nPeaks); +peakMat = zeros(peakThreshold,10*emc.nPeaks); n = 1; @@ -1256,7 +1252,7 @@ % box for removal and center of mass calc, use a larger box if multiple % peaks are being saved. - bDist = 1+round(log(nPeaks)); + bDist = 1+round(log(emc.nPeaks)); clI = c(1) - bDist; chI = c(1) + bDist; clJ = c(2) - bDist; @@ -1299,11 +1295,11 @@ peakMat(n,10) = gather(MAX); iSNR = 0; - if nPeaks > 1 + if emc.nPeaks > 1 possible_angles = gather(magBox); possible_angles(angBox == topPeak) = 0; nRandom = 2; - for iPeak = 2:nPeaks + for iPeak = 2:emc.nPeaks useRandom = false; @@ -1407,8 +1403,8 @@ i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... peakMat(i,4:6),r,1); - if nPeaks > 1 - for iPeak = 2:nPeaks + if emc.nPeaks > 1 + for iPeak = 2:emc.nPeaks iSym = mod(nSym,symOps.nSymMats)+1; r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), rotConvention , 'inv') * symOps.symmetry_matrices{iSym},1,9); diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index 3d21efbc..8bee4d51 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -99,11 +99,6 @@ end nOrientations=1;%nOrientations = emc.('pseudoMLnumber'); nCTFgroups = 9; -try - nPeaks = emc.('nPeaks'); -catch - nPeaks = 1; -end % Resolution lower than this is not gold standard, and will also be mixed % in the references to keep orientations from diverging. Should be > 2.25 x @@ -443,10 +438,8 @@ subTomoOrigin = fix(tmpSearchGeom(iSubTomo,11:13)./dupInTheLoop); if any(subTomoOrigin < 1 + dupRadius) || any([sx,sy,sz] < subTomoOrigin + dupRadius) - tmpSearchGeom(iSubTomo,26:26:26*nPeaks) = -9999; -% if any(subTomoOrigin < 1 + dupRadius) || any([sx,sy,sz] < subTomoOrigin + dupRadius) -% tmpSearchGeom(iSubTomo,26) = -9999; -% else + tmpSearchGeom(iSubTomo,26:26:26*emc.nPeaks) = -9999; + else positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = 1; positionIDX(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... @@ -456,9 +449,6 @@ end % loop building position matrix - % Add in the points from the modified list, which will increase the value at - % any retained positions to 2. - for iSubTomo = 1:size(modGeom,1) subTomoOrigin = fix(modGeom(iSubTomo,:)); @@ -503,9 +493,8 @@ mapName = fileInfo{iTomo,2}; tmpGeom = parResults{iGPU}.(mapName); - tmpGeom(:,9:26:26*nPeaks) = repmat(ceil(tmpGeom(:,11)./ ... - subTomoMeta.('ctfGroupSize').(mapName)(2)),1,nPeaks); -% tmpGeom(:,9) = ceil(tmpGeom(:,11)./ subTomoMeta.('ctfGroupSize').(mapName)(2)); + tmpGeom(:,9:26:26*emc.nPeaks) = repmat(ceil(tmpGeom(:,11)./ ... + subTomoMeta.('ctfGroupSize').(mapName)(2)),1,emc.nPeaks); % Sort so that CTFs can be left in main mem, and only pulled when needed and only % once per round of alignment. @@ -513,9 +502,7 @@ for iSubTomo = 1:size(tmpGeom,1) - tmpGeom(iSubTomo, 4:26:26*nPeaks) = nIDX; -% tmpGeom(iSubTomo, 4) = nIDX; - + tmpGeom(iSubTomo, 4:26:26*emc.nPeaks) = nIDX; nIDX = nIDX +1; end @@ -542,7 +529,7 @@ preFscSplit = gather(subTomoMeta); % Randomly divide the data into half sets. -[ subTomoMeta ] = BH_fscSplit( preFscSplit, splitOnTomos, nPeaks); +[ subTomoMeta ] = BH_fscSplit( preFscSplit, splitOnTomos, emc.nPeaks); subTomoMeta.('currentCycle') = 0; subTomoMeta.('currentTomoCPR') = mapBackIter; subTomoMeta.('currentResForDefocusError') = lowResCut; diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 80f9f6a8..93b8aaac 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -42,9 +42,31 @@ % Now check for optional parameters +% Early development parameter, used to store more than one orientation during template matching +% and use for further refinement. if ~isfield(emc, 'nPeaks') emc.('nPeaks') = 1; end +% Used when cutting out subtomos for further processing. Adds extra padding to anticipate shifts etc. +% This has not been well tested + +% When used in average3d, this value is stored in the subTomoMeta. +if ~isfield(emc, 'CUTPADDING') + emc.('CUTPADDING') = 20; +end + +if isfield(emc, 'whitenPS') + if (numel(emc.whitenPS) == 3) + emc.('wiener_constant') = emc.whitenPS(3); + else + error('whitenPS should be a 3 element vector'); + end +else + emc.('whitenPS') = [0.0,0.0,0.0]; + emc.('wiener_constant') = 0.0; +end + + end diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index d99ff053..3a7114c2 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -41,17 +41,11 @@ error('args = PARAMETER_FILE, CYCLE') end -try - nPeaks = emc.('nPeaks'); -catch - nPeaks = 1; -end + % Backup the current geometry system(sprintf('cp %s.mat preDupRemoval_%s.mat',emc.('subTomoMeta'),emc.('subTomoMeta'))); load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); geometry = subTomoMeta.(cycleNumber).RawAlign; -% % % mapExt = subTomoMeta.mapExt; -% % % mapPath= subTomoMeta.mapPath; masterTM = subTomoMeta; clear subTomoMeta @@ -130,7 +124,7 @@ % Logical translating particle ids to postions in geometry file posList = ismember(positionList(:,4), idxList); % Replace ones in logical with CCC from previous raw Alignment - cccList = max(positionList(:,1:26:26*nPeaks),[],2).*posList; + cccList = max(positionList(:,1:26:26*emc.nPeaks),[],2).*posList; [~ , maxCCCcoord] = max(cccList); % set highest CCC to zero, so the remaining are all inferior @@ -138,8 +132,8 @@ % column 3 posList(maxCCCcoord) = 0; nRemoved = nRemoved + sum(posList); - positionList(posList,26:26:26*nPeaks) = -9999; - positionList(posList,3:26:26*nPeaks) = -9999; + positionList(posList,26:26:26*emc.nPeaks) = -9999; + positionList(posList,3:26:26*emc.nPeaks) = -9999; end catch diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index b42a16f3..b38298f6 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -62,11 +62,6 @@ flgGold = 1; end -try - nPeaks = emc.('nPeaks'); -catch - nPeaks = 1; -end nRows = length(emc.('pcaScaleSpace')); featureVector = cell(2,1); @@ -211,7 +206,7 @@ oldPca = load(coeffMatrix); coeffsUNTRIMMED = oldPca.coeffs idxList = oldPca.idxList; - if nPeaks > 1 + if emc.nPeaks > 1 peakList = oldPca.idxList; else peakList = []; @@ -438,9 +433,9 @@ % for trouble shooting try - if (nPeaks > 1) + if (emc.nPeaks > 1) for thisIDX = 1:length(lIndClass) - for iPeak = 0:nPeaks-1 + for iPeak = 0:emc.nPeaks-1 positionList(lIndPart(thisIDX), 26 + 26*iPeak) = class(lIndClass(thisIDX)+iPeak); % fprintf('iTomo %d iSubtomo %d iPeak %d Class %d\n',iTomo,lIndPart(thisIDX),iPeak+1,class(lIndClass(thisIDX)+iPeak)); end diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 03d8c72d..d695f387 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -182,11 +182,6 @@ outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); %%%flgGold = emc.('flgGoldStandard'); -try - nPeaks = emc.('nPeaks'); -catch - nPeaks = 1; -end flgNorm = 1;% emc.('flgNormalizeWMDs'); try @@ -664,14 +659,14 @@ end % Extend the random subset to each peak if needed - if (nPeaks > 1) + if (emc.nPeaks > 1) for iTomo = 1:nTomograms selectedList = geometry.(tomoList{iTomo})(:,8) > 0; - geometry.(tomoList{iTomo})(selectedList,8+26:26:nPeaks*26) = 1; + geometry.(tomoList{iTomo})(selectedList,8+26:26:emc.nPeaks*26) = 1; clear selectedList end - nTOTAL = nTOTAL*nPeaks; - nSUBSET = nSUBSET*nPeaks; + nTOTAL = nTOTAL*emc.nPeaks; + nSUBSET = nSUBSET*emc.nPeaks; end % Initialize array in main memory for pca @@ -813,7 +808,7 @@ iPeak=0; % make sure this exists if we are no including the particle if (includeParticle) make_sf3d = true; - for iPeak = 0:nPeaks-1 + for iPeak = 0:emc.nPeaks-1 % Get position and rotation info, angles stored as e1,e3,e2 as in AV3 % and PEET. This also makes inplane shifts easier to see. @@ -953,7 +948,7 @@ if (keepTomo) idxList(1, nExtracted) = particleIDX; - peakList(1,nExtracted) = iPeak+1; % This probably is not necessary - it should be 1:nPEaks,1:nPeaks,1:nPeaks... + peakList(1,nExtracted) = iPeak+1; nExtracted = nExtracted + 1; nTemp = nTemp + 1; @@ -986,7 +981,7 @@ end % end of ignore if statment if ~rem(iSubTomo,100) fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... - iSubTomo, nSubTomos,iPeak+1,nPeaks, iTomo,nTomograms); + iSubTomo, nSubTomos,iPeak+1,emc.nPeaks, iTomo,nTomograms); fprintf('Total nExtracted = %d\n', nExtracted-1); fprintf('Total nIgnored = %d\n', nIgnored); diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index e3a2eb09..5aa87eb8 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -1,94 +1,17 @@ function [ ] = BH_average3d(PARAMETER_FILE, CYCLE, STAGEofALIGNMENT) -%Extract and interpolate a subTomogram from a 3d volume. -% -% Input variables: -% -% CYCLE = 0,1,2 etc. -% -% NEW_CYCLE = 0,1 truth value, use the specified geometry to initialize a new -% alignment cycle. -% -% IMAGE = 3d volume, or a string specifing a volume to read in.exit - -% -% classVector = Extract a subset of class averages. -% 0 = Ignore classes, and average all -% -% 2, # classes = Extract specified classes with unique symmetry. -% [1, 2, 5, 6; -% 3, 1, 6, 1] -% -% CLASS_NAME = class to draw from -% -% MOTIF_SIZE = Size of the window to extract. This must be at least 14 pixels -% larger than the key features to allow for a soft apodization -% over 7 pixels in each direction. -% -% -% SAMPLING = Binning factor, assumed to be integer value. Image is first -% smoothed by an appropriate low-pass filter to reduce aliasing. -% -% GEOMETRY = A structure with tomogram names as the field names, and geometry -% information in a 26 columSn array. -% Additionally, a field called 'source_path' has a value with the -% absolute path to the location of the tomograms. -% -% The input is a string 'Geometry_templatematching.mat' for -% example, and it is expected that the structure is saved as the -% variable named geometry. -% -% -% FSC = Randomly divide the data into halves (not just even/odd) for use in -% Fourier shell correlation calculation. -% -% OUTPUT_PREFIX = String to prepend to output volumes. -% -% -% Output variables: -% -% None = files are written to disk in the current directory. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% Goals & Limitations: -% -% Cut out a subTomogram and transform to the standard basis (microscope -% reference frame) from an existing 3d volume. -% -% Assumed to run on GPU. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% TODO -% - Error checking for memory limitations -% - In testing verify "implicit" gpu arrays are actually gpu arrays -% - Check binning -% - Confirm position 7 is where I want to keep FSC value -% - Update geometry to record % sampling -% -% - Store binning & Size in class specific. geometry and use as a check for other programs -% at runtime. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + if (nargin ~= 3) error('args = PARAMETER_FILE, CYCLE, STAGEofALIGNMENT') end -try - CUTPADDING = subTomoMeta.('CUTPADDING') -catch - CUTPADDING=20 -end % FIXME: hacking in a test test_fuzz=false; test_multi_ref_diffmap= true; -% Explicit reference to location of variables in main memory, or on the GPU. -cpu = struct(); -GPU = struct(); startTime = clock; CYCLE = EMC_str2double(CYCLE); @@ -109,16 +32,6 @@ reconScaling = 1; -try - tmpVal = emc.('whitenPS'); - if (numel(tmpVal) == 3) - wiener_constant = tmpVal(3); - else - error('flgWhitenPS should be a 3 element vector'); - end -catch - wiener_constant = 0.0; -end try fscBfactor = emc.('Fsc_bfactor'); @@ -267,7 +180,7 @@ bh_global_ML_angleTolerance = 5; end -if (nPeaks > 1) +if (emc.nPeaks > 1) fprintf('For ML approach:\nUsing a compression factor %3.3f\nUsing an angulare tolerance of %3.3f degrees\n', ... bh_global_ML_compressByFactor, bh_global_ML_angleTolerance); end @@ -746,7 +659,6 @@ wgtVect = []; angVect = []; chiVect = []; -% positionList(:,1:26:26*nPeaks) if (spike_prior) @@ -776,22 +688,22 @@ % iCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); tmpTomo = []; - spike_info.(f{iTomo}).('angular_diff') = zeros(size(geometry.(f{iTomo}) , 1),nPeaks,'single'); - spike_info.(f{iTomo}).('normal_distance') = zeros(size(geometry.(f{iTomo}) , 1),nPeaks,'single'); + spike_info.(f{iTomo}).('angular_diff') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); + spike_info.(f{iTomo}).('normal_distance') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); - spike_info.(f{iTomo}).('angular_prob') = zeros(size(geometry.(f{iTomo}) , 1),nPeaks,'single'); - spike_info.(f{iTomo}).('angular_weight') = zeros(size(geometry.(f{iTomo}) , 1),nPeaks,'single'); + spike_info.(f{iTomo}).('angular_prob') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); + spike_info.(f{iTomo}).('angular_weight') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); nSubTomos = size(geometry.(f{iTomo}) , 1); - particle_coords = zeros(nSubTomos .* nPeaks,8,'single'); + particle_coords = zeros(nSubTomos .* emc.nPeaks,8,'single'); nVol = 1; for iSubTomo = 1:nSubTomos - for iPeak = 1:nPeaks + for iPeak = 1:emc.nPeaks particle_coords(nVol,1:5) = geometry.(f{iTomo})(iSubTomo,[26,4,11:13]+(iPeak-1)*26); nVol = nVol + 1; end end - % Logical size nsubtomos x nPeaks + % Logical size nsubtomos x emc.nPeaks positions_to_analyze = particle_coords(:,1) ~= -9999; display_fit = false; radial_shrink_factor = 2; @@ -806,7 +718,7 @@ particle_coords(positions_to_analyze,[6:8]) = [ normal_vect]; nVol = 1; for iSubTomo = 1:nSubTomos - for iPeak = 1:nPeaks + for iPeak = 1:emc.nPeaks if (particle_coords(nVol,1) ~= -9999) particleAxis = reshape(geometry.(f{iTomo})(iSubTomo,[17:25]+(iPeak-1)*26),3,3)*[0;0;1]; angularDiff = dot(particle_coords(nVol,6:8), particleAxis); @@ -855,14 +767,14 @@ for iParProc = 1:nParProcesses for iTomo = iterList{iParProc} if (track_stats) - geometry.(tomoList{iTomo})(:,1:26:26*nPeaks) = geometry.(tomoList{iTomo})(:,1:26:26*nPeaks)./geometry.(tomoList{iTomo})(:,2:26:26*nPeaks); + geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks) = geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks)./geometry.(tomoList{iTomo})(:,2:26:26*emc.nPeaks); end min_weight = 1e-6; if (spike_prior) for iSubTomo = 1:size(geometry.(tomoList{iTomo}) , 1) - peakList = false(nPeaks,1); - for iPeak = 1:nPeaks + peakList = false(emc.nPeaks,1); + for iPeak = 1:emc.nPeaks if (geometry.(tomoList{iTomo})(iSubTomo,26*iPeak)~=-9999) iWeight = ... spike_info.('angular_pdf')(spike_info.(tomoList{iTomo}).('angular_diff')(iSubTomo,iPeak)); @@ -880,7 +792,7 @@ spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList) = ... spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList) ./ ... sum(spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList)); - for iScoreMod = 1:nPeaks + for iScoreMod = 1:emc.nPeaks if (peakList(iScoreMod)) geometry.(tomoList{iTomo})(iSubTomo,2 + 26*(iScoreMod-1)) = ... spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,iScoreMod); @@ -893,14 +805,14 @@ end - keepVect = geometry.(tomoList{iTomo})(:,26:26:26*nPeaks)~=-9999 ; + keepVect = geometry.(tomoList{iTomo})(:,26:26:26*emc.nPeaks)~=-9999 ; - tmpVect = geometry.(tomoList{iTomo})(:,1:26:26*nPeaks); + tmpVect = geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks); cccVect = [cccVect ; reshape(tmpVect(keepVect),[],1)]; - tmpVect = geometry.(tomoList{iTomo})(:,2:26:26*nPeaks); + tmpVect = geometry.(tomoList{iTomo})(:,2:26:26*emc.nPeaks); wgtVect = [wgtVect ; reshape(tmpVect(keepVect),[],1)]; end @@ -922,7 +834,7 @@ masterTM.(cycleNumber).('score_sigma') = std(cccVect); if (spike_prior) -% spike_info.('normalization_factor') = 1;%nVolumes ./ (nPeaks * addedWeight); +% spike_info.('normalization_factor') = 1;%nVolumes ./ (emc.nPeaks * addedWeight); % fprintf('From %d possible volumes the total weight is %3.3e\n',nVolumes,addedWeight); end avgCCC = mean(cccVect); @@ -1056,7 +968,7 @@ % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); - nSubTomos = sum(any(positionList(:,26:26:26*nPeaks) ~= -9999,2)); + nSubTomos = sum(any(positionList(:,26:26:26*emc.nPeaks) ~= -9999,2)); nSubTomosTotal = nSubTomosTotal + nSubTomos; @@ -1121,12 +1033,12 @@ if ( flgEstSNR ) % When the class is for estimating SNR - includeList = ( any(abs(positionList(:,1:26:26*nPeaks)) >= cccCutOff,2) & ... + includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... positionList(:,10) == iClassIDX & ... positionList(:,7) == iGold ); else % When the class is from statistical analysis - includeList = ( any(abs(positionList(:,1:26:26*nPeaks)) >= cccCutOff,2) & ... + includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... positionList(:,26) == iClassIDX & ... positionList(:,7) == iGold ); end @@ -1134,8 +1046,8 @@ else % if class is 0, pick all non-ignored particles - includeList = ( any(abs(positionList(:,1:26:26*nPeaks)) >= cccCutOff,2) & ... - any(positionList(:,26:26:26*nPeaks) ~= -9999,2) & ... + includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... + any(positionList(:,26:26:26*emc.nPeaks) ~= -9999,2) & ... positionList(:,7) == iGold ); end @@ -1153,7 +1065,7 @@ % symmetry = classVector{iGold}(2, iClassPos); - if ( nPeaks > 1 ) + if ( emc.nPeaks > 1 ) % Calculate a relative weighting, normalize max score to one % and then raise to compressBy factor to downweight lower % scores. @@ -1161,7 +1073,7 @@ [ peakWgt, sortedList ] = BH_weightAngCheckPeaks( ... positionList(iSubTomo,:),... - nPeaks, ... + emc.nPeaks, ... masterTM.(cycleNumber).('score_sigma') ,... iSubTomo, tomoList{iTomo},... track_stats); @@ -1177,7 +1089,7 @@ make_sf3d = true; - for iPeak = 1:nPeaks + for iPeak = 1:emc.nPeaks if peakWgt(iPeak) == -9999 positionList(iSubTomo, 26*iPeak) = -9999; @@ -1402,7 +1314,7 @@ iSubTomo, tomoList{iTomo}); iParticle(:,:,:) = 0; % Flag the particle as ignored - positionList(iSubTomo, 26:26:nPeaks*26) = -9999; + positionList(iSubTomo, 26:26:emc.nPeaks*26) = -9999; else if (test_fuzz) @@ -1474,9 +1386,9 @@ iSubTomo, tomoList{iTomo}, 1-padVAL); % Flag the particle as ignored - positionList(iSubTomo, 26:26:26*nPeaks) = -9999; + positionList(iSubTomo, 26:26:26*emc.nPeaks) = -9999; nIgnored = nIgnored + 1; - peakWgt(1:nPeaks) = -9999; + peakWgt(1:emc.nPeaks) = -9999; end % end From 1846edec2182655baa451afb9102c7941e07ac4f Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 08:55:33 -0500 Subject: [PATCH 008/151] Move whitenPS to parser and remove old commented out function from ctf3d --- alignment/BH_alignRaw3d_v2.m | 12 +-- alignment/BH_templateSearch3d_2.m | 10 --- ctf/BH_ctf_Correct3d.m | 142 ++---------------------------- 3 files changed, 6 insertions(+), 158 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 6c25e09e..31b737e4 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -82,16 +82,6 @@ flgCutOutVolumes=0 end -try - tmpVal = emc.('whitenPS'); - if (numel(tmpVal) == 3) - wiener_constant = tmpVal(3); - else - error('flgWhitenPS should be a 3 element vector'); - end -catch - wiener_constant = 0.0; -end % TODO decide on a "reasonable" padding based on expected shifts. try @@ -1105,7 +1095,7 @@ make_SF3D = false; if use_v2_SF3D % For now excluding the soften weight. - [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); + [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, emc.wiener_constant); imgWdgInterpolator = ''; % The unshifted mask is kept in texture mem until no longer % needed diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index ba36e6a5..65ef526d 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -79,16 +79,6 @@ expand_lines = ''; end -try - tmpVal = emc.('whitenPS'); - if (numel(tmpVal) == 3) - wiener_constant = tmpVal(3); - else - error('flgWhitenPS should be a 3 element vector'); - end -catch - wiener_constant = 0.0; -end peakThreshold = emc.('Tmp_threshold'); diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 9db779b8..e46a10b2 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -116,9 +116,9 @@ catch end -if (bh_global_turn_on_phase_plate(1) && any(flgWhitenPS)) +if (bh_global_turn_on_phase_plate(1) && any(emc.whitenPS)) fprintf('WARNING: phakePhasePlate and whitening are conflicting preocesses. Turning off whitening.\n') - flgWhitenPS = [0,0,0]; + emc.whitenPS = [0,0,0]; end @@ -270,8 +270,8 @@ % we expect the resolution to improve beyond our current value, we % multiply by 1/2, which gives a (only loosely optimized) resTarget. resTarget = mean(masterTM.('currentResForDefocusError')*0.5); - if (flgWhitenPS(1)) - flgWhitenPS(2) = resTarget; + if (emc.whitenPS(1)) + emc.whitenPS(2) = resTarget; end else % For template search @@ -686,7 +686,7 @@ useSurfaceFit,invertDose,... bh_global_turn_on_phase_plate,... filterProjectionsForTomoCPRBackground,... - flgWhitenPS); + emc.whitenPS); end % Write out the stack to the cache directory as a tmp file @@ -904,138 +904,6 @@ end -% % % function [STACK, evalMask, deltaZ] = loadAndMaskStack(TLT, STACK_PRFX, ... -% % % mapBackIter,maxZpix,... -% % % samplingRate,... -% % % PosControl2d,... -% % % tiltWeight,flgWhitenPS,pixelSize) -% % % -% % % if (PosControl2d) -% % % prefix = 'ctf'; -% % % suffix = '_ctf' -% % % else -% % % prefix = 'ali'; -% % % suffix = ''; -% % % end -% % % -% % % if samplingRate > 1 -% % % fullStack = sprintf('%sStacks/%s_ali%d%s.fixed', ... -% % % prefix,STACK_PRFX,mapBackIter+1,suffix); -% % % inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed',... -% % % STACK_PRFX,mapBackIter+1,suffix,samplingRate); -% % % if ~exist(inputStack, 'file') -% % % % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s > /dev/null',samplingRate,fullStack,inputStack); -% % % %% binCMD = sprintf('newstack -bin %d -antialias 6 %s %s ',samplingRate,fullStack,inputStack); -% % % % -% % % % system(binCMD); -% % % BH_multi_loadOrBin(fullStack,-1.*samplingRate,2); -% % % -% % % end -% % % else -% % % inputStack = sprintf('%sStacks/%s_ali%d%s.fixed',... -% % % prefix,STACK_PRFX,mapBackIter+1,suffix) -% % % end -% % % -% % % system(sprintf('header %s',inputStack)); -% % % % iHeader = MRCImage(inputStack,0); -% % % % STACK = gpuArray(single(getVolume(iHeader))); -% % % -% % % STACK = single(getVolume(MRCImage(inputStack))); -% % % -% % % % iHeader = getHeader(iHeader); -% % % % iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... -% % % % iHeader.cellDimensionY/iHeader.nY, ... -% % % % iHeader.cellDimensionZ/iHeader.nZ]; -% % % -% % % -% % % [d1,d2,d3] = size(STACK); -% % % nPrjs = d3; -% % % -% % % useableArea = [d1-128,d2-128,maxZpix]; -% % % -% % % -% % % -% % % [evalMask, deltaZ ] = BH_multi_projectionMask( [d1,d2,d3;useableArea], TLT, 'cpu' ); -% % % -% % % -% % % -% % % -% % % % Local normalization doesn't address any large scale gradients in the -% % % % images. Do a simple high pass over the lowest 7 frequencyBinns -% % % bandNyquist = BH_bandpass3d([d1,d2,1],0,0,1,'GPU','nyquistHigh'); -% % % -% % % taperMask = gpuArray(fspecial('gaussian',[9,9],1.5)); -% % % -% % % -% % % for iPrj = 1:nPrjs -% % % -% % % -% % % iEvalMask = gpuArray(evalMask(:,:,TLT(iPrj,1))); -% % % fprintf('iPrj %d size %d, %d\n',iPrj,size(STACK,3),TLT(iPrj,1)); -% % % iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); -% % % -% % % % iMask = convn(single(iEvalMask),taperMask,'same'); -% % % -% % % -% % % iProjection = iProjection - mean(iProjection(iEvalMask)); -% % % if ( flgWhitenPS(1) ) -% % % %fprintf('confirm whitening PS\n.'); -% % % flgWhitenPS -% % % [iProjection,~] = BH_whitenNoiseSpectrum(iProjection,'',[600,14,pixelSize,160],flgWhitenPS); -% % % mean2(iProjection) -% % % else -% % % iProjection = real(ifftn(fftn(iProjection).*bandNyquist)); -% % % end -% % % % iProjection = real(ifftn(fftn(iProjection.*iMask).*bandNyquist)); -% % % -% % % -% % % inFin = ~(isfinite(iProjection)); nInf = sum(inFin(:)); -% % % if (nInf) -% % % % fprintf('Removing %d (%2.4f) inf from prj %d\n',nInf,100*nInf/numel(iProjection),TLT(iPrj,1)); -% % % iProjection(inFin) = 0; -% % % end -% % % -% % % iRms = rms(iProjection(iEvalMask)); -% % % outliers = (iProjection > 6 * iRms); nOutliers = sum(outliers(:)); -% % % tiltScale = 1- ( abs(sind(TLT(iPrj,4))).* tiltWeight(1)); -% % % if (nOutliers) -% % % -% % % % iProjection(outliers) = sign(iProjection(outliers)).*3.*iRms.*((rand(size(iProjection(outliers)))./2)+0.5); -% % % iProjection(outliers) = 6.*iRms.* (rand(size(iProjection(outliers)))-0.5); -% % % iProjection = iProjection ./ ( rms(iProjection(iEvalMask)) ./ tiltScale); -% % % else -% % % iProjection = iProjection ./ ( iRms ./ tiltScale); -% % % end -% % % -% % % % if ( flgWhitenPS ) -% % % % %fprintf('confirm whitening PS\n.'); -% % % % [iProjection,~] = BH_whitenNoiseSpectrum(iProjection,'',pixelSize,1); -% % % % end -% % % -% % % if tiltWeight(2) -% % % % I don't think this makes sense, but test keeping the power constant -% % % % after application of the exposure filter. -% % % iProjection = fftn(iProjection); -% % % iPower = sum(abs(iProjection(:))); -% % % iProjection = iProjection .* iExpFilter; -% % % STACK(:,:,TLT(iPrj,1)) = gather(single(real(ifftn(iProjection.* ... -% % % (iPower./sum(abs(iProjection(:)))))))); -% % % else -% % % STACK(:,:,TLT(iPrj,1)) = gather(iProjection);%gather(single(real(ifftn(fftn(iProjection) .* iExpFilter)))); -% % % end -% % % % STACK(:,:,TLT(iPrj,1)) = gather(single(iMask.*real(ifftn(fftn(iProjection) .* iExpFilter)))); -% % % % STACK(:,:,TLT(iPrj,1)) = gather(single(iProjection.*iMask)); -% % % clear iProjection iMask iExpFilter iEvalMask -% % % end -% % % -% % % -% % % clear bandNyquist iMask exposureFilter iProjection lowRMSMAsk -% % % -% % % -% % % end - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - function [] = preBinStacks(TLT, STACK_PRFX, mapBackIter,usableArea,... samplingRate,... PosControl2d,... From be41daddaada47dd8cac24b3183e471db575131a Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 09:10:07 -0500 Subject: [PATCH 009/151] Remove all optional refs to sf3d_v2 (always on now) --- alignment/BH_alignRaw3d_v2.m | 133 +++++---------------------------- statistics/BH_pcaPub.m | 25 ++----- transformations/BH_average3d.m | 59 +-------------- 3 files changed, 24 insertions(+), 193 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 31b737e4..a1a8f950 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -90,11 +90,7 @@ CUTPADDING=20 end -try - use_v2_SF3D = emc.('use_v2_SF3D') -catch - use_v2_SF3D = true; -end + try symmetry_op = emc.('symmetry'); @@ -740,51 +736,22 @@ system('mkdir -p alignResume'); system(sprintf('mkdir -p alignResume/%s',outputPrefix)); -softenWeight = 1/sqrt(samplingRate); -if ~(use_v2_SF3D) - for iParProc = 1:nParProcesses - - % Caclulating weights takes up a lot of memory, so do all that are necessary - % prior to the main loop -- CHANGE THE CHECK TO JUST READ THE HEADER NOT LOAD - % THE WEIGHT INTO GPU MEMORY - - for iTomo = iterList{iParProc} - - BH_multi_loadOrCalcWeight(masterTM,ctfGroupList,tomoList{iTomo},samplingRate ,... - sizeCalc,geometry,flgPrecision,1); - - - end - end - % Clear all of the GPUs prior to entering the main processing loop - for iGPU = 1:nGPUs - g = gpuDevice(iGPU); - fprintf('\n\nClear gpu %d mem prior to main loop, %3.3e available\n\n',iGPU,g.AvailableMemory); - clear g - end -end parVect = 1:nParProcesses; fprintf('Starting main loopwith N references %d\n', nReferences(1)); parfor iParProc = parVect symmetry = symmetry_op; % Why TF would this be necessary? -% for iParProc = 1:nParProcesses -%profile on + bestAngles_tmp = struct(); geometry_tmp = geometry; -% % % % Get the gpuIDX assigned to this process -% % % iGPUidx = gpuDevice(); -% % % iGPUidx = iGPUidx.Index; - gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; - iGPUidx = gpuIDXList(iParProc); - gpuDevice(iGPUidx); - fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); - for iTomo = iterList{iParProc} + gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; + iGPUidx = gpuIDXList(iParProc); + gpuDevice(iGPUidx); + fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); + for iTomo = iterList{iParProc} - - nCtfGroups = ctfGroupList.(tomoList{iTomo})(1); % Check for interupted alignment. previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); if exist(previousAlignment,'file') @@ -891,29 +858,10 @@ iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; - if ~(use_v2_SF3D) - wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); -% wgtName = sprintf('cache/%s_bin%d.wgt', tomoList{iTomo},... -% samplingRate); - maxWedgeMask = BH_unStackMontage4d(1:nCtfGroups,wgtName,... - ceil(sqrt(nCtfGroups)).*[1,1],''); - maxWedgeIfft = maxWedgeMask; - - for iWdg = 1:length(maxWedgeMask) - if ~isempty(maxWedgeMask{iWdg}) - maxWedgeMask{iWdg} = (maxWedgeMask{iWdg} - min(maxWedgeMask{iWdg}(:))) + 1e-3; - maxWedgeMask{iWdg} = maxWedgeMask{iWdg}.^softenWeight; - maxWedgeIfft{iWdg} = ifftshift(maxWedgeMask{iWdg}); - - end - end - fprintf('loaded %s.\n',wgtName); - end - - % Can't clear inside the parfor, but make sure we don't have two tomograms + % Can't clear inside the parfor, but make sure we don't have two tomograms % in memory at once. tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; @@ -960,33 +908,7 @@ make_SF3D = true; breakPeak = 0; % for try catch on cut out vols - if (wdgIDX ~= positionList(iSubTomo,9)) && ~(use_v2_SF3D) - % Geometry is sorted on this value so that tranfers are minimized, - % as these can take up a lot of mem. For 9 ctf Groups on an 80s - % ribo at 2 Ang/pix at full sampling ~ 2Gb eache. - - wdgIDX = positionList(iSubTomo,9); - fprintf('pulling the wedge %d onto the GPU\n',wdgIDX); - % Avoid temporar - - iMaxWedgeMask = []; iMaxWedgeIfft = []; - iMaxWedgeMask = gpuArray(maxWedgeMask{wdgIDX}); - iMaxWedgeIfft = gpuArray(maxWedgeIfft{wdgIDX}); - imgWdgInterpolator = ''; - [imgWdgInterpolator, ~] = interpolator(iMaxWedgeMask,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - - end - - -% [~,iw1,iw2,iw3] = BH_resample3d(iMaxWedgeMask, eye(3), [0,0,0], ... -% {'Bah',1,'linear',1,wdgBinary_tmp}, ... -% 'GPU', 'inv'); -% inputWgtVectors = {iw1,iw2,iw3}; -% iw1 = []; iw2 = []; iw3 = []; - - - for iPeak = 1:emc.nPeaks if (track_stats) @@ -994,8 +916,6 @@ mip.('x') = {}; mip.('x2') = {}; mip.('N') = 0; -% mip.('X') = zeros(1,3,'single','gpuArray'); -% mip.('X2') = zeros(3,3,'single','gpuArray'); end if (breakPeak) continue; @@ -1003,9 +923,6 @@ getInitialCCC = 1; cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); - powerOut = zeros(nAngles(1).*nReferences(1),1,'gpuArray'); - - % Used in refinment loop angCount = 1; @@ -1093,16 +1010,14 @@ if (make_SF3D) make_SF3D = false; - if use_v2_SF3D - % For now excluding the soften weight. - [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, emc.wiener_constant); - imgWdgInterpolator = ''; - % The unshifted mask is kept in texture mem until no longer - % needed - [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); + % For now excluding the soften weight. + [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, emc.wiener_constant); + imgWdgInterpolator = ''; + % The unshifted mask is kept in texture mem until no longer + % needed + [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); - end % Just use C1 to initialize, whether or not this is the final refInterpolator = ''; refWdgInterpolator= ''; @@ -1247,22 +1162,12 @@ end -% iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... -% {rotConvention ,symmetry,'linear',1,wdgBinary_tmp}, ... -% 'GPU', 'inv',inputWgtVectors); [ iWedgeMask ] = imgWdgInterpolator.interp3d(... RotMat,... [0,0,0],rotConvention ,... 'inv',symmetry); - - - - - - -% % % powerOut(angCount) = sum(abs(iTrimParticle(volBinary_tmp))).^2; end @@ -1278,13 +1183,11 @@ error('flgMultiRefAlignment is not 0,1,2') end - for iRef = refToAlign % 1:max(nReferences(:)) + for iRef = refToAlign switch alignLoop case 1 - - % use transpose of RotMat [ iRotRef ] = refInterpolator.interp3d(... @@ -1849,9 +1752,7 @@ rotParticle = []; end % end loop over possible peaks - if use_v2_SF3D - iMaxWedgeIfft = []; - end + iMaxWedgeIfft = []; end % loop over subTomos diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index d695f387..4334e699 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -174,11 +174,6 @@ scaleCalcSize = 1.5; end -try - use_v2_SF3D = emc.('use_v2_SF3D') -catch - use_v2_SF3D = true -end outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); %%%flgGold = emc.('flgGoldStandard'); @@ -817,13 +812,12 @@ angles = positionList(iSubTomo,[17:25]+26*iPeak); - if (use_v2_SF3D && make_sf3d) + if ( make_sf3d ) make_sf3d = false; radialGrid = ''; padWdg = [0,0,0;0,0,0]; [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, ... TLT, center,reconGeometry, wiener_constant); - end % If flgGold there is no change, otherwise temporarily resample the @@ -883,9 +877,7 @@ use_only_once = true; [ ~, iParticle ] = interpolator(gpuArray(iParticle),angles, shiftVAL, 'Bah', 'inv', symmetry, use_only_once); - if (use_v2_SF3D) - [ ~, iWedge ] = interpolator(gpuArray(wedgeMask),angles,[0,0,0], 'Bah', 'inv', symmetry, use_only_once); - end + [ ~, iWedge ] = interpolator(gpuArray(wedgeMask),angles,[0,0,0], 'Bah', 'inv', symmetry, use_only_once); else % Transform the particle, and then trim to motif size @@ -893,10 +885,8 @@ [ iParticle ] = BH_resample3d(iParticle, angles, shiftVAL, ... 'Bah', 'GPU', 'inv'); - if (use_v2_SF3D) - [ iWedge ] = BH_resample3d(wedgeMask, angles, [0,0,0], ... + [ iWedge ] = BH_resample3d(wedgeMask, angles, [0,0,0], ... 'Bah', 'GPU', 'inv'); - end end @@ -922,15 +912,10 @@ - if (use_v2_SF3D) - [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iPrt,ifftshift(iWedge),... + [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iPrt,ifftshift(iWedge),... flgNorm,pixelSize,radialMask, padWdg); - else - - iWmd = real(ifftn(abs(iPrt) .* exp(1i.*(angle(iPrt) - angle(avgMotif_FT{iGold, iScale}))))); - end - + if all(isfinite(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale))))) diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 5aa87eb8..2d777c66 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -166,11 +166,7 @@ scaleCalcSize = 1.5; end -try - use_v2_SF3D = emc.('use_v2_SF3D') -catch - use_v2_SF3D = true -end + global bh_global_ML_compressByFactor; global bh_global_ML_angleTolerance; if isempty(bh_global_ML_compressByFactor) @@ -593,49 +589,7 @@ padVal = padVal .* (padVal > 0); fscPAD = padCalc;%[floor(padVal./2); ceil((padVal)./2)] -if ~(use_v2_SF3D) - - delete(gcp('nocreate')); - EMC_parpool(nGPUs); - - % TODO need some way of only sending out the command for one tilt or - % something to prevent collisions that result in no complete recon. - - tiltNameList = fieldnames(masterTM.mapBackGeometry); - tiltNameList = tiltNameList(~ismember(tiltNameList,{'tomoName','viewGroups'})); - wgtList = tomoList; - - for iGPU = 1:nGPUs - nThisGPU = 0; - for iParProc = iGPU:nGPUs:nParProcesses - for iTomo = iterList{iParProc} - iTilt = masterTM.mapBackGeometry.tomoName.(wgtList{iTomo}).tiltName; - if (any(ismember(tiltNameList,iTilt))) - tiltNameList{ismember(tiltNameList,iTilt)} = 'continue'; - else - wgtList{iTomo} = 'continue'; - end - end - end - end - wgtList=wgtList(~ismember(wgtList,'continue')); - maxPerGPU = ceil(length(wgtList)/nGPUs) + 1; - - parfor iGPU = 1:nGPUs - % for iGPU = 1:nGPUs - for iParProc = iGPU:nGPUs:length(wgtList) - % Caclulating weights takes up a lot of memory, so do all that are necessary - % prior to the main loop - - BH_multi_loadOrCalcWeight(masterTM,ctfGroupList,wgtList{iParProc},samplingRate ,... - sizeCalc,geometry,'single',iGPU); - - - - end - end -end try EMC_parpool(nParProcesses+1) @@ -1003,13 +957,8 @@ iTiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); nCtfGroups = ctfGroupList.(tomoList{1})(1); - if ~(use_v2_SF3D) - wedgeMask = BH_unStackMontage4d(1:nCtfGroups,wgtName,... - ceil(sqrt(nCtfGroups)).*[1,1],''); - end - % Work on each class seperately pushing to main memory when finished. for iGold = 1:2-flgFinalAvg iGold @@ -1115,11 +1064,7 @@ TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); if (make_sf3d) - if (use_v2_SF3D) - [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); - else - iSF3D = gpuArray(wedgeMask{wdgIDX}); - end + [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); make_sf3d = false; end From 5404edec2146a7f9861271b5bc1bf08a938aad72 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 09:24:59 -0500 Subject: [PATCH 010/151] Manually formated all files, primarily to fix nesting indent inconsitencies that make editing code hard. The formater did some things I don't like, and will rever like alignment of function input/outputs. --- alignment/BH_alignRaw3d_v2.m | 2382 +++++++------- alignment/BH_fitBeads.m | 196 +- alignment/BH_refine_on_beads.m | 240 +- alignment/BH_runAutoAlign.m | 124 +- alignment/BH_templateSearch3d_2.m | 1214 +++---- coordinates/BH_decomposeIMODxf.m | 196 +- coordinates/BH_defineMatrix.m | 198 +- coordinates/BH_multi_angularSearch.m | 468 +-- coordinates/BH_multi_calcBinShift.m | 20 +- coordinates/BH_multi_calcBinShift.mrc | 10 - coordinates/BH_multi_gridCoordinates.m | 180 +- coordinates/BH_multi_gridSearchAngles.m | 46 +- coordinates/BH_multi_iterator.m | 92 +- coordinates/BH_multi_recGeom.m | 6 +- coordinates/EMC_coordGrids.m | 2 +- coordinates/EMC_coordTransform.m | 70 +- coordinates/EMC_coordVectors.m | 16 +- coordinates/eulerSearch.m | 100 +- ctf/BH_ctfCalc.m | 164 +- ctf/BH_ctfCalcError.m | 12 +- ctf/BH_ctf_Correct.m | 134 +- ctf/BH_ctf_Correct3d.m | 842 ++--- ctf/BH_ctf_Estimate.m | 1826 +++++----- ctf/BH_ctf_Refine2.m | 486 +-- ctf/BH_ctf_Updatefft.m | 848 ++--- ctf/BH_runCtfFind.m | 26 +- logicals/BH_imodWait.m | 46 +- logicals/BH_isWindowValid.m | 74 +- logicals/BH_multi_maskCheck.m | 10 +- logicals/BH_multi_parallelJobs.m | 8 +- logicals/BH_multi_parallelWorkers.m | 4 +- logicals/BH_multi_validArea.m | 16 +- logicals/BH_weightAngCheckPeaks.m | 26 +- masking/BH_bandLimitCenterNormalize.m | 12 +- masking/BH_bandLimitCenterNormalize_2.m | 8 +- masking/BH_bandLimitCenterNormalize_cpu.m | 10 +- masking/BH_bandpass3d.m | 68 +- masking/BH_eraseBeads.m | 182 +- masking/BH_fftShift.m | 36 +- masking/BH_mask3d.m | 520 +-- masking/BH_mask3d_cpu.m | 214 +- masking/BH_multi_calcTaper.m | 2 +- masking/BH_multi_gaussian2d.m | 22 +- masking/BH_multi_gaussian3d.m | 10 +- masking/BH_multi_makeHermitian.m | 10 +- masking/BH_multi_padVal.m | 34 +- masking/BH_multi_projectionMask.m | 34 +- masking/BH_padZeros3d.m | 66 +- masking/BH_weightMask3d.m | 326 +- masking/BH_weightMaskMex.m | 20 +- masking/BH_weightMask_dp.m | 538 +-- masking/BH_weightMask_dpRUN.m | 14 +- masking/EMC_applyBandpass.m | 18 +- masking/EMC_getBandpass.m | 50 +- masking/EMC_limits.m | 16 +- masking/EMC_maskIndex.m | 22 +- masking/EMC_maskReference.m | 66 +- masking/EMC_maskShape.m | 62 +- masking/EMC_resize.m | 56 +- masking/EMC_taper.m | 14 +- masking/alignmentVol.m | 38 +- metaData/BH_checkInstall.m | 12 +- metaData/BH_geometryInitialize.m | 340 +- metaData/BH_geometry_Constraints.m | 236 +- metaData/BH_mergeClassGeometry.m | 20 +- metaData/BH_parseParameterFile.m | 14 +- metaData/BH_randomSubset.m | 70 +- metaData/BH_recordAngularSampling.m | 38 +- metaData/BH_refAlignmentsApply.m | 84 +- metaData/BH_removeDuplicates.m | 90 +- metaData/BH_returnIncludedTilts.m | 24 +- metaData/BH_skipClassAlignment.m | 28 +- metaData/BH_trimImodLocal.m | 14 +- metaData/EMC_parpool.m | 6 +- metaData/EMC_str2double.m | 22 +- metaData/SAVE_IMG.m | 66 +- metaData/test_tm.py | 17 - statistics/BH_clusterPub.m | 414 +-- statistics/BH_diffMap.m | 64 +- statistics/BH_exposureFilter.m | 106 +- statistics/BH_fit_ellipsoidal_prior.m | 80 +- statistics/BH_fscGold_class.m | 1124 +++---- statistics/BH_fscSplit.m | 32 +- statistics/BH_localWiener2d.m | 134 +- statistics/BH_movingAverage.m | 2 +- statistics/BH_movingRMS.m | 4 +- statistics/BH_multi_cRef.m | 184 +- statistics/BH_multi_cRef_Vnorm.m | 418 +-- statistics/BH_multi_cRef_VnormApply.m | 18 +- statistics/BH_multi_cRef_wgtCritical.m | 36 +- statistics/BH_multi_loadAndMaskStack.m | 44 +- statistics/BH_multi_statScale.m | 24 +- statistics/BH_multi_xcf_Rotational.m | 76 +- statistics/BH_multi_xcf_Translational.m | 64 +- statistics/BH_multi_xcf_Translational_2.m | 80 +- statistics/BH_pcaPub.m | 766 ++--- statistics/BH_rocCurve.m | 74 +- synthetic/BH_synthetic_mapBack.m | 2924 ++++++++--------- synthetic/BH_to_cisTEM_mapBack.m | 918 +++--- testScripts/BH_benchmark.m | 46 +- testScripts/BH_whitenNoiseSpectrum.m | 44 +- testScripts/CTF.m | 44 +- testScripts/EMC_convn.m | 2 +- testScripts/EMC_is3d.m | 2 +- testScripts/EMC_maximizeSNR.m | 352 +- testScripts/EMC_setPrecision.m | 2 +- testScripts/ellipsoid_fit.m | 90 +- testScripts/emClarity.m | 676 ++-- testScripts/fourierCtfRec.m | 274 +- testScripts/fourierCtfRecTex.m | 458 +-- testScripts/fourierTransformer.m | 160 +- testScripts/imshow3D.m | 36 +- testScripts/interpolator.m | 466 +-- testScripts/rotm2eul.m | 14 +- transformations/BH_average3d.m | 2224 ++++++------- transformations/BH_axialSymmetry.m | 42 +- transformations/BH_montage4d.m | 36 +- transformations/BH_montage4d_halfGrid.m | 24 +- transformations/BH_multi_combineLowResInfo.m | 32 +- transformations/BH_multi_loadOrBin.m | 60 +- transformations/BH_multi_loadOrBuild.m | 92 +- transformations/BH_multi_loadOrCalcWeight.m | 124 +- transformations/BH_reScale3d.m | 26 +- transformations/BH_resample2d.m | 82 +- transformations/BH_resample3d.m | 226 +- transformations/BH_unStackMontage4d.m | 38 +- .../BH_unStackMontage4d_halfGrid.m | 10 +- 127 files changed, 13436 insertions(+), 13463 deletions(-) delete mode 100644 coordinates/BH_multi_calcBinShift.mrc delete mode 100644 metaData/test_tm.py diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index a1a8f950..ba0be5d6 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -1,5 +1,5 @@ - function [ ] = BH_alignRaw3d_v2(PARAMETER_FILE, CYCLE, varargin) - +function [ ] = BH_alignRaw3d_v2(PARAMETER_FILE, CYCLE, varargin) + %Extract and align class averages and references from 4D montages derived. % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -45,8 +45,8 @@ CYCLE = EMC_str2double(CYCLE); cycle_numerator = ''; cycle_denominator =''; - flgStartThird = 0; - flgReverseOrder = 0; +flgStartThird = 0; +flgReverseOrder = 0; if numel(CYCLE) == 3 cycle_numerator = CYCLE(2); cycle_denominator = CYCLE(3); @@ -58,7 +58,7 @@ flgReverseOrder = 1; flgStartThird = 0; CYCLE = abs(CYCLE); - + end @@ -66,11 +66,11 @@ emc = BH_parseParameterFile(PARAMETER_FILE); cycleNumber = sprintf('cycle%0.3u', CYCLE); load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); -mapBackIter = subTomoMeta.currentTomoCPR; +mapBackIter = subTomoMeta.currentTomoCPR; reconScaling = 1; -try +try track_stats = emc.('track_stats'); catch track_stats = false; @@ -98,7 +98,7 @@ error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); end -try +try use_new_grid_search = emc.('use_new_grid_search'); catch use_new_grid_search = true; @@ -127,7 +127,7 @@ flgMultiRefAlignment = 0; end -try +try updateClassByBestReferenceScore = emc.('updateClassByBestReferenceScore'); catch updateClassByBestReferenceScore = false; @@ -145,7 +145,7 @@ % FIXME: unused, fix experimental options option try flgSymmetrizeSubTomos = emc.('flgSymmetrizeSubTomos'); -catch +catch flgSymmetrizeSubTomos = 0; end flgRaw_shapeMask = 0;%= emc.('experimentalOpts')(3) @@ -160,18 +160,18 @@ angleSearch = emc.('Raw_angleSearch'); peakSearch = (emc.('particleRadius')./pixelSize); peakCOM = [1,1,1].*3; -className = emc.('Raw_className'); +className = emc.('Raw_className'); -try +try loadTomo = emc.('loadTomo') catch loadTomo = 0; end -try +try eraseMaskType = emc.('Peak_mType'); - eraseMaskRadius = emc.('Peak_mRadius')./pixelSize; + eraseMaskRadius = emc.('Peak_mRadius')./pixelSize; fprintf('Further restricting peak search to radius %f %f %f\n',... - eraseMaskRadius); + eraseMaskRadius); eraseMask = 1; catch eraseMask = 0; @@ -213,7 +213,7 @@ else refName = emc.('Raw_className'); end - + outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); @@ -270,7 +270,7 @@ nRefOut(1:2) = [length(unique(refGroup{1})) + sum(( refSym{1} < 0 )),... - length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; + length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; %%%%%%%%%%%%%%%%%%%%%%% @@ -292,13 +292,13 @@ [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', pixelSize) + BH_multi_maskCheck(emc, 'Ali', pixelSize) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) + BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) -try +try flgLimitToOneProcess = emc.('flgLimitToOneProcess'); catch flgLimitToOneProcess = 0; @@ -315,14 +315,14 @@ limitToOne = emc.('nCpuCores'); end -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); +[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); if ( flgReverseOrder ) % fprintf('nCpuCores is %d\n', limitToOne); - % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); + % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); % for iParProc = 1:nParProcesses % iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' % end - % Flip the order for reverse processing on a second machine. This will also disable saving of + % Flip the order for reverse processing on a second machine. This will also disable saving of % of the metadata so there aren't conflicts. for iParProc = 1:nParProcesses iterList{iParProc} = flip(iterList{iParProc}); @@ -330,16 +330,16 @@ elseif ( flgStartThird ) % fprintf('nCpuCores is %d\n', limitToOne); - % [ nParProcesses, iterList_full] = BH_multi_parallelJobs(nTomograms,nGPUs*cycle_denominator, sizeCalc(1),limitToOne*cycle_denominator); - + % [ nParProcesses, iterList_full] = BH_multi_parallelJobs(nTomograms,nGPUs*cycle_denominator, sizeCalc(1),limitToOne*cycle_denominator); + % for iParProc = 1:nParProcesses % iterList_full{iParProc} = sortedTomoIDX(iterList_full{iParProc})'; % end - + % % Need to scale this back down % nParProcesses = limitToOne; - - % Shift to start at one third through to process on a third machine. This will also disable saving of + + % Shift to start at one third through to process on a third machine. This will also disable saving of % of the metadata so there aren't conflicts. % iterList = {}; % for iParProc = 1:nParProcesses @@ -356,27 +356,27 @@ lIDX = min(cycle_numerator*nParts,length(iterList{iParProc})); iterList{iParProc} = iterList{iParProc}(fIDX:lIDX); end - - + + else % error('not supported run config'); % fprintf('nCpuCores is %d\n', limitToOne); - % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); + % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); % for iParProc = 1:nParProcesses % iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' % end - - + + end if any(peakSearch > maskRadius) fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') peakSearch( (peakSearch > maskRadius) ) = ... - maskRadius( (peakSearch > maskRadius) ); + maskRadius( (peakSearch > maskRadius) ); end - + % Read in the references. % Read in the references. refIMG = cell(2,1); @@ -384,39 +384,39 @@ refWgtROT = cell(2,1); imgCounts = cell(2,1); for iGold = 1:2 - + if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - - - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) - + + + imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) + weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', refName, halfSet); imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; - - + + [ refTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - + masterTM.(cycleNumber).(imgNAME){1}, ... + masterTM.(cycleNumber).(imgNAME){2},... + sizeWindow); + [ wdgTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... - sizeCalc); - + masterTM.(cycleNumber).(weightNAME){1},... + masterTM.(cycleNumber).(weightNAME){2},... + sizeCalc); + sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}(2:2:6)'; - + if (flgCenterRefCOM) -% % % % % % % [ comMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); - [ comMask ] = EMC_maskShape(maskType, sizeMask, maskRadius, 'gpu', {'shift', maskCenter}); + % % % % % % % [ comMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); + [ comMask ] = EMC_maskShape(maskType, sizeMask, maskRadius, 'gpu', {'shift', maskCenter}); end - - % get boxSize + + % get boxSize n = 1 ; tIMG = cell(numel(refVector{iGold})); tWDG = cell(numel(refVector{iGold}));tWDG_r = tWDG; for iP = 1:numel(refTMP) if ~isempty(refTMP{iP}) @@ -424,40 +424,40 @@ if (flgCenterRefCOM) % Not sure if this is always the best approach, but it may be % useful in some cases. -% % % % % % % [~,iCOM] = BH_mask3d(gpuArray(tIMG{n}).*comMask,pixelSize,'','',1); + % % % % % % % [~,iCOM] = BH_mask3d(gpuArray(tIMG{n}).*comMask,pixelSize,'','',1); [~, ~, ~,iCOM] = EMC_maskReference(gpuArray(tIMG{n}).*comMask, pixelSize, {'fsc',true; 'com', true}); fprintf('centering ref %d on COM %3.3f %3.3f %3.3f \n',n,iCOM); - + tIMG{n} = BH_resample3d(tIMG{n},[0,0,0],gather(iCOM), ... - {'Bah',1,'spline'},'cpu','inv'); - + {'Bah',1,'spline'},'cpu','inv'); + end - - tWDG{n} = wdgTMP{iP}; wdgTMP{iP} = []; - tWDG{n} = tWDG{n} - min(tWDG{n}(:)) + 1e-6; - tWDG{n} = tWDG{n} ./ max(tWDG{n}(:)); - + + tWDG{n} = wdgTMP{iP}; wdgTMP{iP} = []; + tWDG{n} = tWDG{n} - min(tWDG{n}(:)) + 1e-6; + tWDG{n} = tWDG{n} ./ max(tWDG{n}(:)); + n = n + 1; end end - - - wdgPAD = BH_multi_padVal(size(tWDG{1}), sizeCalc); - for iWdg = 1:n-1 - tWDG_r{iWdg} = BH_padZeros3d(tWDG{iWdg},wdgPAD(1,:),wdgPAD(2,:),... - 'cpu',flgPrecision); - tWDG{iWdg} = ifftshift(tWDG_r{iWdg}); - end - - refWGT{iGold} = tWDG; clear tWDG wdgTMP - refWgtROT{iGold} = tWDG_r; clear tWDG_r - + + + wdgPAD = BH_multi_padVal(size(tWDG{1}), sizeCalc); + for iWdg = 1:n-1 + tWDG_r{iWdg} = BH_padZeros3d(tWDG{iWdg},wdgPAD(1,:),wdgPAD(2,:),... + 'cpu',flgPrecision); + tWDG{iWdg} = ifftshift(tWDG_r{iWdg}); + end + + refWGT{iGold} = tWDG; clear tWDG wdgTMP + refWgtROT{iGold} = tWDG_r; clear tWDG_r + refIMG{iGold} = tIMG ; clear tIMG refTMP - - + + clear comMask end @@ -475,163 +475,163 @@ - - - stat_mask = []; - if (eraseMask) - peakMask = EMC_maskShape(eraseMaskType,sizeCalc,floor(eraseMaskRadius),'cpu',{'kernel',false}); - if track_stats - stat_mask = single(find(peakMask > 0.95)); - end - - else - if track_stats - stat_mask = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter}); - stat_mask = single(find(stat_mask > 0.95)); - end - [ peakMask ] = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter;'kernel',false}); + +stat_mask = []; +if (eraseMask) + peakMask = EMC_maskShape(eraseMaskType,sizeCalc,floor(eraseMaskRadius),'cpu',{'kernel',false}); + + if track_stats + stat_mask = single(find(peakMask > 0.95)); end - - if ( flgRaw_shapeMask ) - - [ volMask ] = gather(sqrt(volMask .* ... - EMC_maskReference(refIMG{1}{iRef}+refIMG{2}{iRef}, pixelSize, {'fsc', true}))); - - else -% % % % % % % [ volMask ] = gather(BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter)); - [ volMask ] = gather(EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter})); - - end - - - - bandpassFilt = cell(nReferences(1),1); - bandpassFiltREF = bandpassFilt; - wCCC = cell(nReferences(1),1); - for iWccc = 1:length(nReferences(1)); - wCCC{iWccc} = 0; +else + if track_stats + stat_mask = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter}); + stat_mask = single(find(stat_mask > 0.95)); end - if (flgClassify || flgMultiRefAlignment) - for iRef = 1:nReferences(1) - if (flgClassify) - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); - else - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Raw%d',iRef)); % % % % - end + [ peakMask ] = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter;'kernel',false}); +end - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - % returns a cpu array - if (flgWeightCCC) - [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1); - else - [ bandpassFilt{iRef}, ~] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1); - end - - bandpassFiltREF{iRef} = 1; +if ( flgRaw_shapeMask ) + + [ volMask ] = gather(sqrt(volMask .* ... + EMC_maskReference(refIMG{1}{iRef}+refIMG{2}{iRef}, pixelSize, {'fsc', true}))); + +else + % % % % % % % [ volMask ] = gather(BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter)); + [ volMask ] = gather(EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter})); + +end - end - else +bandpassFilt = cell(nReferences(1),1); +bandpassFiltREF = bandpassFilt; +wCCC = cell(nReferences(1),1); +for iWccc = 1:length(nReferences(1)); + wCCC{iWccc} = 0; +end +if (flgClassify || flgMultiRefAlignment) + for iRef = 1:nReferences(1) + if (flgClassify) + fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); + else + fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Raw%d',iRef)); % % % % + end - for iRef = 1 - fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); - [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... - 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); - % returns a cpu array - if (flgWeightCCC) - [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1 ); - else - [ bandpassFilt{iRef},~ ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1 ); - end - - bandpassFiltREF{iRef} = 1; - - + [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... + 'GPU', {'none'}, 1, 0, 1 ); + radialGrid = single(radialGrid./pixelSize); + % returns a cpu array + if (flgWeightCCC) + [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1); + else + [ bandpassFilt{iRef}, ~] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1); end - + + + bandpassFiltREF{iRef} = 1; + + end +else + + + + for iRef = 1 + fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); + [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... + 'GPU', {'none'}, 1, 0, 1 ); + radialGrid = single(radialGrid./pixelSize); + % returns a cpu array + if (flgWeightCCC) + [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1 ); + else + [ bandpassFilt{iRef},~ ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1 ); + end + + bandpassFiltREF{iRef} = 1; + + end +end + % if (flgWeightCCC) % for i = 1:length(wCCC{1}) % i % length(wCCC{1}{i}) % end % end - - % This is just used to limit the interpolation search so use the most - % permissive bandpass, while the appropriate bandpass (given a multi-ref - % alignment) will still be applied. - mostPermissive = zeros(1,nReferences(1)); - for iRef = 1:nReferences(1) - mostPermissive(iRef) = sum(bandpassFilt{iRef}(:)); - end - [~,mPidx] = max(mostPermissive); - - wdgBinary = single(find(fftshift(bandpassFilt{mPidx} > 10^-2))); - + +% This is just used to limit the interpolation search so use the most +% permissive bandpass, while the appropriate bandpass (given a multi-ref +% alignment) will still be applied. +mostPermissive = zeros(1,nReferences(1)); +for iRef = 1:nReferences(1) + mostPermissive(iRef) = sum(bandpassFilt{iRef}(:)); +end +[~,mPidx] = max(mostPermissive); + +wdgBinary = single(find(fftshift(bandpassFilt{mPidx} > 10^-2))); + ref_FT1 = cell(2,1); ref_FT2 = cell(2,1); - -for iGold = 1:2 +for iGold = 1:2 + if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - + nOut = 1; refOUT = cell(2.*nReferences(iGold),2); - + for iRef = 1:nReferences(iGold) refTMP_2 = refIMG{iGold}{iRef}; refIMG{iGold}{iRef} = []; refTMP = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); - - + padWindow(1,2) + 1: end - padWindow(2,2), ... + padWindow(1,3) + 1: end - padWindow(2,3)); + + % if not using a weighted average (adapted SPW filter), apply an % approximation the cRef from Rosenthal/Henderson. This is currently always set to one % and is just doing the masking and normalization. It should be okay to just apply the mask % and rely on the normalization during the CCC calc. TODO - ref_FT1{iGold}{iRef} = gather(conj(BH_bandLimitCenterNormalize(... - refTMP.*volMask, bandpassFiltREF{iRef}, (volMask>0.01), padCalc, flgPrecision))); - - - - + ref_FT1{iGold}{iRef} = gather(conj(BH_bandLimitCenterNormalize(... + refTMP.*volMask, bandpassFiltREF{iRef}, (volMask>0.01), padCalc, flgPrecision))); + + + + ref_FT2{iGold}{iRef} = gather(refTMP_2); % Trim for output reference - refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); + refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... + padWindow(1,2) + 1: end - padWindow(2,2), ... + padWindow(1,3) + 1: end - padWindow(2,3)); % Overwrite a copy of the filtered, bandpassed ref for output refOUT{nOut} = real(ifftn(conj(ref_FT1{iGold}{iRef}))); refOUT{nOut} = gather(refOUT{nOut}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); - - + padCalc(1,2) + 1: end - padCalc(2,2), ... + padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); + + refOUT{nOut} = refOUT{nOut}.*volMask; refOUT{nOut+1} = real(ifftn(BH_bandLimitCenterNormalize(... - refTMP_2, '', '', padCalc, 'single'))); + refTMP_2, '', '', padCalc, 'single'))); refOUT{nOut+1} = gather(refOUT{nOut+1}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) ); + padCalc(1,2) + 1: end - padCalc(2,2), ... + padCalc(1,3) + 1: end - padCalc(2,3)) ); nOut = nOut + 2; refOUT{nOut} = refOUT{nOut} - mean(refOUT{nOut}(:)); @@ -640,15 +640,15 @@ refOUT{nOut+1} = refOUT{nOut+1} - mean(refOUT{nOut+1}(:)); refOUT{nOut+1} = refOUT{nOut+1} ./ rms(refOUT{nOut+1}(:)); end - - + + % Save a montage of the masked reference & shape masks if requested. - -% maskedOUTFILE = sprintf('%s_maskedRef-mont_%s.mrc',outputPrefix,halfSet); -% [ maskedReferences, ~ ] = BH_montage4d(refOUT, ''); -% SAVE_IMG(MRCImage(single(maskedReferences)), maskedOUTFILE); - + % maskedOUTFILE = sprintf('%s_maskedRef-mont_%s.mrc',outputPrefix,halfSet); + % [ maskedReferences, ~ ] = BH_montage4d(refOUT, ''); + % SAVE_IMG(MRCImage(single(maskedReferences)), maskedOUTFILE); + + end clear refIMG refWDG refOUT iRef @@ -656,20 +656,20 @@ %%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't %%%%%%%%%%%%%%%%%%%%% search at all in that dimension. -updateWeights = false; -gridSearch = ''; +updateWeights = false; +gridSearch = ''; if (use_new_grid_search) gridSearch = eulerSearch(symmetry_op, angleSearch(1),... - angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, true); + angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, true); nAngles = sum(gridSearch.number_of_angles_at_each_theta); inPlaneSearch = gridSearch.parameter_map.psi - -try - symmetry_constrained_search = emc.('symmetry_constrained_search'); - fprintf('Using symmetry constrained search\n'); -catch - symmetry_constrained_search = false; -end + + try + symmetry_constrained_search = emc.('symmetry_constrained_search'); + fprintf('Using symmetry constrained search\n'); + catch + symmetry_constrained_search = false; + end if (symmetry_constrained_search) % symmetry expansion on in-plane search only @@ -682,26 +682,26 @@ end end end - + flgRefine=false; - + for i = 1:length(gridSearch.parameter_map.phi) if gridSearch.parameter_map.phi{i} > 0 flgRefine=true; break; end end - - - angleStep = []; + + + angleStep = []; else [ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch); + = BH_multi_gridSearchAngles(angleSearch); if any(angleStep(:,1)) flgRefine = true; else flgRefine = false; - end + end if sum(angleStep(:,2) > 0) updateWeights = true; @@ -710,8 +710,8 @@ end -% [masterTM] = BH_recordAngularSampling( masterTM, cycleNumber, angleStep, inPlaneSearch); - +% [masterTM] = BH_recordAngularSampling( masterTM, cycleNumber, angleStep, inPlaneSearch); + nCount = 1; @@ -741,1065 +741,1065 @@ fprintf('Starting main loopwith N references %d\n', nReferences(1)); parfor iParProc = parVect symmetry = symmetry_op; % Why TF would this be necessary? - + bestAngles_tmp = struct(); geometry_tmp = geometry; - + gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; iGPUidx = gpuIDXList(iParProc); gpuDevice(iGPUidx); fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); - + for iTomo = iterList{iParProc} - - % Check for interupted alignment. - previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); - if exist(previousAlignment,'file') - % Sometimes when multiple nodes are used, an extra line is added. - % TODO fix this workaround - system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s_clean; mv %s_clean %s',... - previousAlignment,previousAlignment,previousAlignment,previousAlignment)); - bestAngles_tmp.(tomoList{iTomo}) = load(previousAlignment); - fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); - else - % There is some memory leak somewhere that I haven't been able to figure - % out. I am clearing all vars but output in the children functions ... this - % isn't ideal, but for now is an acceptable stop gap. - %D = gpuDevice(gpuList(iGPU)); - - % shake up the random number generator for phi and theta - rng('shuffle'); - bandpassFilt_tmp = cell(nReferences(1),1); - bandpassFiltREF_tmp = cell(nReferences(1),1); - for iRef = 1:nReferences(1) - if flgMultiRefAlignment <= 2 - bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); - else - bandpassFilt_tmp{iRef} = (bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = (bandpassFiltREF{iRef}); - end - end - - - - ref_FT1_tmp = cell(2,1); - ref_FT2_tmp = cell(2,1); - ref_WGT_tmp = cell(2,1); - ref_WGT_rot = cell(2,1); - - - volMask_tmp = gpuArray(volMask); - volBinary_tmp = single(find( volMask_tmp > 0.01 )); - peakMaskInterpolator = ''; - peakMaskInterpolator = interpolator(gpuArray(peakMask),[0,0,0],[0,0,0], rotConvention , 'forward', 'C1', false); - - if (track_stats) - mip = struct(); - mip.('mask') = gpuArray(stat_mask); - end - - - wCCC_tmp = cell(length(wCCC)); - - - - for iRef = 1:nReferences(1) - for iWccc = 1:length(wCCC{iRef}) - if (flgWeightCCC) - wCCC_tmp{iRef}{iWccc} = gpuArray(wCCC{iRef}{iWccc}); + % Check for interupted alignment. + previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); + if exist(previousAlignment,'file') + % Sometimes when multiple nodes are used, an extra line is added. + % TODO fix this workaround + system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s_clean; mv %s_clean %s',... + previousAlignment,previousAlignment,previousAlignment,previousAlignment)); + bestAngles_tmp.(tomoList{iTomo}) = load(previousAlignment); + fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); + else + % There is some memory leak somewhere that I haven't been able to figure + % out. I am clearing all vars but output in the children functions ... this + % isn't ideal, but for now is an acceptable stop gap. + %D = gpuDevice(gpuList(iGPU)); + + % shake up the random number generator for phi and theta + rng('shuffle'); + + bandpassFilt_tmp = cell(nReferences(1),1); + bandpassFiltREF_tmp = cell(nReferences(1),1); + for iRef = 1:nReferences(1) + if flgMultiRefAlignment <= 2 + bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); + bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); else - % The check in xcf_rotational looks for a cell - wCCC_tmp{iRef} = 0; + bandpassFilt_tmp{iRef} = (bandpassFilt{iRef}); + bandpassFiltREF_tmp{iRef} = (bandpassFiltREF{iRef}); end end - end - - - for iGold = 1:2 - for iRef = 1:nReferences(iGold) - if flgMultiRefAlignment <= 2 - ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = gpuArray(refWgtROT{iGold}{iRef}); - else - % Temp workaround, six big ribo refs crashing - ref_FT1_tmp{iGold}{iRef} = (ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = (ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = (refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = (refWgtROT{iGold}{iRef}); + + + ref_FT1_tmp = cell(2,1); + ref_FT2_tmp = cell(2,1); + ref_WGT_tmp = cell(2,1); + ref_WGT_rot = cell(2,1); + + + volMask_tmp = gpuArray(volMask); + volBinary_tmp = single(find( volMask_tmp > 0.01 )); + peakMaskInterpolator = ''; + peakMaskInterpolator = interpolator(gpuArray(peakMask),[0,0,0],[0,0,0], rotConvention , 'forward', 'C1', false); + + if (track_stats) + mip = struct(); + mip.('mask') = gpuArray(stat_mask); + end + + + wCCC_tmp = cell(length(wCCC)); + + + + for iRef = 1:nReferences(1) + for iWccc = 1:length(wCCC{iRef}) + if (flgWeightCCC) + wCCC_tmp{iRef}{iWccc} = gpuArray(wCCC{iRef}{iWccc}); + else + % The check in xcf_rotational looks for a cell + wCCC_tmp{iRef} = 0; + end end end - end - - - sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) - tic; - - % Load the tomo into gpu - tomoName = tomoList{iTomo}; + + + + for iGold = 1:2 + for iRef = 1:nReferences(iGold) + if flgMultiRefAlignment <= 2 + ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); + ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); + ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); + ref_WGT_rot{iGold}{iRef} = gpuArray(refWgtROT{iGold}{iRef}); + else + % Temp workaround, six big ribo refs crashing + ref_FT1_tmp{iGold}{iRef} = (ref_FT1{iGold}{iRef}); + ref_FT2_tmp{iGold}{iRef} = (ref_FT2{iGold}{iRef}); + ref_WGT_tmp{iGold}{iRef} = (refWGT{iGold}{iRef}); + ref_WGT_rot{iGold}{iRef} = (refWgtROT{iGold}{iRef}); + end + end + end + + + sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) + tic; + + % Load the tomo into gpu + tomoName = tomoList{iTomo}; %fprintf('gpu %d working on tomoName %s\n', iGPU, tomoName); - + tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); - % Load in the geometry for the tomogram, and get number of subTomos. - positionList = geometry_tmp.(tomoList{iTomo}); - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); - -% [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); - binShift = [0,0,0]; - nSubTomos = size(positionList,1); - - - + % Load in the geometry for the tomogram, and get number of subTomos. + positionList = geometry_tmp.(tomoList{iTomo}); + + tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + coords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); + + % [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); + binShift = [0,0,0]; + nSubTomos = size(positionList,1); + + + iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; - - - + + + % Can't clear inside the parfor, but make sure we don't have two tomograms % in memory at once. - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); - - if (flgCutOutVolumes) - volumeData = []; - else - [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,loadTomo); - if ( loadTomo ) - volHeader = struct(); - volHeader.('nX') = size(volumeData,1); - volHeader.('nY') = size(volumeData,2); - volHeader.('nZ') = size(volumeData,3); - else - volHeader = getHeader(volumeData); - end - end - - - % For now, set up for full grid-search only, as I intend to just do - % translational and in-plane searches for now anyhow. - - [~,iv1,iv2,iv3] = BH_resample3d(volMask_tmp,eye(3),[0,0,0],... - {'Bah',1,'linear',1,volBinary_tmp}, ... - 'GPU', 'inv'); - inputVectors = {iv1,iv2,iv3}; - iv1 = []; iv2 = []; iv3 = []; - cccStorageBest = cell(emc.nPeaks,1); - cccStorageRefine = cell(emc.nPeaks,1); - for iPeak = 1:emc.nPeaks - cccStorageBest{iPeak} = zeros(nSubTomos,10); - cccStorageRefine{iPeak}= zeros(nSubTomos,10); - end - % reset for each tomogram - wdgIDX = 0; - - for iSubTomo = 1:nSubTomos - - make_SF3D = true; - breakPeak = 0; % for try catch on cut out vols - - for iPeak = 1:emc.nPeaks - - if (track_stats) - measure_noise = true; - mip.('x') = {}; - mip.('x2') = {}; - mip.('N') = 0; - end - if (breakPeak) - continue; + tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); + TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); + + if (flgCutOutVolumes) + volumeData = []; + else + [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... + reconCoords, mapBackIter, ... + samplingRate,iGPUidx,reconScaling,loadTomo); + if ( loadTomo ) + volHeader = struct(); + volHeader.('nX') = size(volumeData,1); + volHeader.('nY') = size(volumeData,2); + volHeader.('nZ') = size(volumeData,3); + else + volHeader = getHeader(volumeData); end - getInitialCCC = 1; - cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); - cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); - - % Used in refinment loop - angCount = 1; + end - % Check that the given subTomo is not to be ignored - classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); - particleIDX = positionList(iSubTomo, 4); - half_set = positionList(iSubTomo, 7); - - - % if classVector{half_set}(1,:) == 0 - % classPosition = 1; - % flgAllClasses = true; - % else - % classPosition = find(classVector{half_set}(1,:) == classIDX); - % flgAllClasses = false; - % end - % Align all valid subtomos, even if the do not belong to the classes we've selected as references. - % To ignore particles, remove them with geometry RemoveClases.m - flgAllClasses = true; - - - - if (classIDX ~= -9999) && ... % All previously ignored particles - ( flgAllClasses || ismember(classIDX, classVector{half_set}(1,:)) ) - - - center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; - angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); + + % For now, set up for full grid-search only, as I intend to just do + % translational and in-plane searches for now anyhow. + + [~,iv1,iv2,iv3] = BH_resample3d(volMask_tmp,eye(3),[0,0,0],... + {'Bah',1,'linear',1,volBinary_tmp}, ... + 'GPU', 'inv'); + inputVectors = {iv1,iv2,iv3}; + iv1 = []; iv2 = []; iv3 = []; + cccStorageBest = cell(emc.nPeaks,1); + cccStorageRefine = cell(emc.nPeaks,1); + for iPeak = 1:emc.nPeaks + cccStorageBest{iPeak} = zeros(nSubTomos,10); + cccStorageRefine{iPeak}= zeros(nSubTomos,10); + end + % reset for each tomogram + wdgIDX = 0; + + for iSubTomo = 1:nSubTomos - % Find range to extract, and check for domain error. - if (flgCutOutVolumes) - % Need some check that the windowsize has not changed! TODO TODO - - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(2*CUTPADDING+sizeWindow, ... - sizeWindow,maskRadius, center); - else - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... - sizeWindow,maskRadius, center); - end - - - - - if ischar(indVAL) - fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) - nIgnored = nIgnored + 1; - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - else + make_SF3D = true; + breakPeak = 0; % for try catch on cut out vols - if (flgCutOutVolumes) - % Test with some generic padding , only to be used on bin 1 at - % first!!! TODO add a flag to check this. - try - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iparticle = gpuArray(getVolume(MRCImage(particleOUT_name),[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - catch - fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - breakPeak = 1; - continue; + for iPeak = 1:emc.nPeaks + + if (track_stats) + measure_noise = true; + mip.('x') = {}; + mip.('x2') = {}; + mip.('N') = 0; end - else - - if ( loadTomo ) - iparticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3))); - - else - iparticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); + if (breakPeak) + continue; end + getInitialCCC = 1; + cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); + cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); - end - [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... - padVAL(2,1:3), 'GPU', 'singleTaper'); - - - if (make_SF3D) - make_SF3D = false; - % For now excluding the soften weight. - [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, emc.wiener_constant); - imgWdgInterpolator = ''; - % The unshifted mask is kept in texture mem until no longer - % needed - [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); - - % Just use C1 to initialize, whether or not this is the final - refInterpolator = ''; - refWdgInterpolator= ''; - particleInterpolator= ''; - - [refInterpolator, ~] = interpolator(gpuArray(ref_FT2_tmp{1}{1}),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator = interpolator(gpuArray(ref_WGT_rot{half_set}{iRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); - particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); - end - - if (use_new_grid_search) - theta_search = 1:gridSearch.number_of_out_of_plane_angles; - else - theta_search = 1:size(angleStep,1); - end - - for iAngle = theta_search - - if (use_new_grid_search) - theta = gridSearch.parameter_map.theta(iAngle); - if length(gridSearch.parameter_map.phi{iAngle}) > 1 - phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); - else - phiInc = 0; - end - thetaInc = gridSearch.theta_step; - numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); - else - theta = angleStep(iAngle,1); - phiInc = angleStep(iAngle,3); - thetaInc = angleStep(iAngle,4); - numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; + % Used in refinment loop + angCount = 1; + + % Check that the given subTomo is not to be ignored + classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); + particleIDX = positionList(iSubTomo, 4); + half_set = positionList(iSubTomo, 7); + + + % if classVector{half_set}(1,:) == 0 + % classPosition = 1; + % flgAllClasses = true; + % else + % classPosition = find(classVector{half_set}(1,:) == classIDX); + % flgAllClasses = false; + % end + % Align all valid subtomos, even if the do not belong to the classes we've selected as references. + % To ignore particles, remove them with geometry RemoveClases.m + flgAllClasses = true; + + + + if (classIDX ~= -9999) && ... % All previously ignored particles + ( flgAllClasses || ismember(classIDX, classVector{half_set}(1,:)) ) + + + center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; + angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); + % Find range to extract, and check for domain error. + if (flgCutOutVolumes) + % Need some check that the windowsize has not changed! TODO TODO + + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid(2*CUTPADDING+sizeWindow, ... + sizeWindow,maskRadius, center); + else + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... + sizeWindow,maskRadius, center); end - % To prevent only searching the same increments each time in a limited - % grid search, radomly offset the azimuthal angle by a random number - % between 0 and 1/2 the azimuthal increment. - azimuthalRandomizer = (rand(1)-0.5)*phiInc; - - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. - - - + + if ischar(indVAL) + fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) + nIgnored = nIgnored + 1; + geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; + else + + + if (flgCutOutVolumes) + % Test with some generic padding , only to be used on bin 1 at + % first!!! TODO add a flag to check this. + try + particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); + iparticle = gpuArray(getVolume(MRCImage(particleOUT_name),[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + catch + fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); + geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; + breakPeak = 1; + continue; + end + else + + if ( loadTomo ) + iparticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... + indVAL(1,2):indVAL(2,2), ... + indVAL(1,3):indVAL(2,3))); + + else + iparticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + end + + end + [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... + padVAL(2,1:3), 'GPU', 'singleTaper'); + + + if (make_SF3D) + make_SF3D = false; + % For now excluding the soften weight. + [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, emc.wiener_constant); + imgWdgInterpolator = ''; + % The unshifted mask is kept in texture mem until no longer + % needed + [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); + + % Just use C1 to initialize, whether or not this is the final + refInterpolator = ''; + refWdgInterpolator= ''; + particleInterpolator= ''; + + [refInterpolator, ~] = interpolator(gpuArray(ref_FT2_tmp{1}{1}),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator = interpolator(gpuArray(ref_WGT_rot{half_set}{iRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); + particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); + end + if (use_new_grid_search) - % FIXME randomizer passed as bool to eulerSearch - phi_search = gridSearch.parameter_map.phi{iAngle}; + theta_search = 1:gridSearch.number_of_out_of_plane_angles; else - phi_search = 0:angleStep(iAngle,2); + theta_search = 1:size(angleStep,1); end - - for iAzimuth = phi_search - - if (use_new_grid_search) - phi = rem(iAzimuth + azimuthalRandomizer,360); - psiInc = gridSearch.psi_step; - else - phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); - psiInc = angleStep(iAngle,5); - - end - - - for iInPlane = inPlaneSearch - psi = iInPlane; - %[phi,theta,psi-phi]; - - - RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - cccStorageTrans= zeros(1.*nReferences(1),10,'gpuArray'); - - for alignLoop = 1:2 - - - switch alignLoop + for iAngle = theta_search - case 1 - % This takes care of non-inter shift in the origin that is - % ignored during the windowing of the particle. - estPeakCoord = shiftVAL; - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - - case 2 - + if (use_new_grid_search) + theta = gridSearch.parameter_map.theta(iAngle); + if length(gridSearch.parameter_map.phi{iAngle}) > 1 + phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); + else + phiInc = 0; + end + thetaInc = gridSearch.theta_step; + numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); + else + theta = angleStep(iAngle,1); + phiInc = angleStep(iAngle,3); + thetaInc = angleStep(iAngle,4); + numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - bestOfRefs = sortrows(gather(cccStorageTrans), -6); - %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); - - estPeakCoord = bestOfRefs(1,8:10); - - - + end + + % To prevent only searching the same increments each time in a limited + % grid search, radomly offset the azimuthal angle by a random number + % between 0 and 1/2 the azimuthal increment. - -% fprintf('Symmetry confirmation %d\n',symmetry); -% [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... -% estPeakCoord,... -% {'Bah',symmetry,'linear',1,volBinary_tmp}, ... -% 'GPU', 'inv',inputVectors); - [ iTrimParticle ] = particleInterpolator.interp3d(... - RotMat,... - estPeakCoord,rotConvention ,... - 'inv',symmetry); - + azimuthalRandomizer = (rand(1)-0.5)*phiInc; + + + % Calculate the increment in phi so that the azimuthal sampling is + % consistent and equal to the out of plane increment. + + + + if (use_new_grid_search) + % FIXME randomizer passed as bool to eulerSearch + phi_search = gridSearch.parameter_map.phi{iAngle}; + else + phi_search = 0:angleStep(iAngle,2); + end + + + for iAzimuth = phi_search + + if (use_new_grid_search) + phi = rem(iAzimuth + azimuthalRandomizer,360); + psiInc = gridSearch.psi_step; + else + phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); + psiInc = angleStep(iAngle,5); - - - if (getInitialCCC) -% [ iTrimInitial ] = BH_resample3d(iparticle, ... -% reshape(angles,3,3),... -% shiftVAL,... -% {rotConvention ,symmetry,'linear',1,volBinary_tmp}, ... -% 'GPU', 'inv',inputVectors); - [ iTrimInitial ] = particleInterpolator.interp3d(... - reshape(angles,3,3),... - shiftVAL,rotConvention ,... - 'inv',symmetry); + end + + + for iInPlane = inPlaneSearch + psi = iInPlane; + %[phi,theta,psi-phi]; - -% % % powerInitial = sum(abs(iTrimInitial(volBinary_tmp))).^2; -% % % - -% iWedgeInitial = BH_resample3d(iMaxWedgeMask, reshape(angles,3,3), [0,0,0], ... -% {rotConvention ,symmetry,'linear',1,wdgBinary_tmp}, ... -% 'GPU', 'inv',inputWgtVectors); - [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... - reshape(angles,3,3),... - [0,0,0],rotConvention ,... - 'inv',symmetry); + + RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; + + cccStorageTrans= zeros(1.*nReferences(1),10,'gpuArray'); + + for alignLoop = 1:2 - - end - + + switch alignLoop + + case 1 + % This takes care of non-inter shift in the origin that is + % ignored during the windowing of the particle. + estPeakCoord = shiftVAL; + % Estimate the peakshift by rotating the ref not the particle. + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + + case 2 + + + bestOfRefs = sortrows(gather(cccStorageTrans), -6); + %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); + + estPeakCoord = bestOfRefs(1,8:10); + + + + + + % fprintf('Symmetry confirmation %d\n',symmetry); + % [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... + % estPeakCoord,... + % {'Bah',symmetry,'linear',1,volBinary_tmp}, ... + % 'GPU', 'inv',inputVectors); + [ iTrimParticle ] = particleInterpolator.interp3d(... + RotMat,... + estPeakCoord,rotConvention ,... + 'inv',symmetry); + + + + + if (getInitialCCC) + % [ iTrimInitial ] = BH_resample3d(iparticle, ... + % reshape(angles,3,3),... + % shiftVAL,... + % {rotConvention ,symmetry,'linear',1,volBinary_tmp}, ... + % 'GPU', 'inv',inputVectors); + [ iTrimInitial ] = particleInterpolator.interp3d(... + reshape(angles,3,3),... + shiftVAL,rotConvention ,... + 'inv',symmetry); + + + % % % powerInitial = sum(abs(iTrimInitial(volBinary_tmp))).^2; + % % % + + % iWedgeInitial = BH_resample3d(iMaxWedgeMask, reshape(angles,3,3), [0,0,0], ... + % {rotConvention ,symmetry,'linear',1,wdgBinary_tmp}, ... + % 'GPU', 'inv',inputWgtVectors); + [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... + reshape(angles,3,3),... + [0,0,0],rotConvention ,... + 'inv',symmetry); + + + end + + + [ iWedgeMask ] = imgWdgInterpolator.interp3d(... + RotMat,... + [0,0,0],rotConvention ,... + 'inv',symmetry); + + + end + + + switch flgMultiRefAlignment + case 0 + refToAlign = 1; + case 1 + refToAlign = 1:max(nReferences(:)); + case 2 + refToAlign = classIDX; + otherwise + error('flgMultiRefAlignment is not 0,1,2') + end + + for iRef = refToAlign + + switch alignLoop + + case 1 + % use transpose of RotMat + + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + estPeakCoord,rotConvention ,... + 'forward','C1'); + + + + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + + + + % maybe I should be rotating peak mask here in case it has + % an odd shape, since we are leaving the proper frame + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{iRef} ,'',... + padCalc,flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{iRef} ,'',padCalc,flgPrecision); + + if (track_stats && measure_noise) + + + [ ~, mip ] = BH_multi_xcf_Translational_2( ... + rotPart_FT, ... + conj(iRotRef),... + ifftshift(iRotWdg),... + iMaxWedgeIfft,... + iRotMask, peakCOM,... + mip); + + + end + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + cccStorageTrans(iRef,:) = [iRef, particleIDX, ... + phi, theta, psi - phi, ... + 0, 0, ... + peakCoord + estPeakCoord]; + case 2 + + % get starting point + if (getInitialCCC) + + initialRotPart_FT = BH_bandLimitCenterNormalize(... + iTrimInitial.*volMask_tmp,... + bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); + + + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( initialRotPart_FT, ... + ref_FT1_tmp{half_set}{iRef}, ... + ifftshift(iWedgeInitial),... + ref_WGT_tmp{half_set}{iRef}, ... + wCCC_tmp{iRef}); + + + + + cccInitial(iRef,:) = [iRef, particleIDX, ... + 0,0,0, ... + iCCC, 1, ... + shiftVAL]; + + + initialRotPart_FT = []; + + + end + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); + + + + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1_tmp{half_set}{iRef},... + ifftshift(iWedgeMask),... + ref_WGT_tmp{half_set}{iRef}, ... + wCCC_tmp{iRef}); + + + + + + + % Note that no new translational estimate is made, so no + % need to multiply by RotMat + cccStorage2(angCount,:) = ... + [iRef, particleIDX, ... + phi, theta, psi , ... + iCCC, 1, ... + estPeakCoord]; + + + angCount = angCount + 1; + end + + + end % loop over references. + + + + end + % This volume won't be needed until the next subTomo is considered, + % which is also where getInitialCCC Boolean is set to True again. + iTrimInitial = []; + getInitialCCC = 0; - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... - RotMat,... - [0,0,0],rotConvention ,... - 'inv',symmetry); - - - end - + end % in plane angles + end % azimuth + end % polar + + % % % fprintf('Power ratio is %3.3f\n',powerOut./powerInitial); + + cccPreRefineSort = sortrows(gather(cccStorage2),-6); + + if (length(refToAlign) > 1) + cccInitial = sortrows(gather(cccInitial), -6); + cccInitial = cccInitial(1,:); + else + cccInitial = gather(cccInitial(refToAlign,:)); - switch flgMultiRefAlignment - case 0 - refToAlign = 1; - case 1 - refToAlign = 1:max(nReferences(:)); - case 2 - refToAlign = classIDX; - otherwise - error('flgMultiRefAlignment is not 0,1,2') end - - for iRef = refToAlign - - switch alignLoop - - case 1 - % use transpose of RotMat - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - estPeakCoord,rotConvention ,... - 'forward','C1'); - - - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - - - - % maybe I should be rotating peak mask here in case it has - % an odd shape, since we are leaving the proper frame - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{iRef} ,'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{iRef} ,'',padCalc,flgPrecision); + + if cccInitial(1,6 ) > cccPreRefineSort(1,6) + cccPreRefineSort(1,:) = cccInitial(1,:); + end + + + + % This only seems to be a problem with cut out volumes. + % Normalization maybe? + if ~any(cccPreRefineSort(1,:)) + cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); + continue + end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + if (flgRefine) + + + + % Get the results from just this subTomo and sort on CCC + + rRef = cccPreRefineSort(1,1); + rPart = cccPreRefineSort(1,2); + rPhi = cccPreRefineSort(1,3); + rPhiInc = phiInc / 4; + rTheta= cccPreRefineSort(1,4); + rTheInc = thetaInc /2; + rPsi = cccPreRefineSort(1,5); + rPsiInc = psiInc /2; + % Confirm shiftVAL is doing what it should be + rXYZest = cccPreRefineSort(1,8:10); + + if (rTheInc) + % For a larger out of plane step, search a larger range in plane + psiRefineStep = floor(sqrt(rTheInc)); + else + psiRefineStep = 1; + end + + thetaRefineStep =1; + phiRefineStep=2; + totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; + totalRefineStep = prod((2.*totalRefineStep)+1); + + cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); + + if (rPsiInc == 0) + inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; + else + inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; + end + polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; + azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; + + searchList = zeros(totalRefineStep,3); + nSearch = 1; + for iPhi = azimuthalRefine + for iTheta = polarRefine + for iPsi = inPlaneRefine + % best iPsi is origin Psi - Phi, no need to subtract here. - if (track_stats && measure_noise) - - - [ ~, mip ] = BH_multi_xcf_Translational_2( ... - rotPart_FT, ... - conj(iRotRef),... - ifftshift(iRotWdg),... - iMaxWedgeIfft,... - iRotMask, peakCOM,... - mip); - - + searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; + + nSearch = nSearch + 1; end - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); + end + end % end of building angle list + + for iRefine = 1:nSearch-1 + for alignLoop = 1:2 + if alignLoop == 1 + rXYZ = rXYZest; + elseif alignLoop == 2 + rXYZ = cccStorage3(iRefine,8:10); + end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - - cccStorageTrans(iRef,:) = [iRef, particleIDX, ... - phi, theta, psi - phi, ... - 0, 0, ... - peakCoord + estPeakCoord]; - case 2 + RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; - % get starting point - if (getInitialCCC) - - initialRotPart_FT = BH_bandLimitCenterNormalize(... - iTrimInitial.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( initialRotPart_FT, ... - ref_FT1_tmp{half_set}{iRef}, ... - ifftshift(iWedgeInitial),... - ref_WGT_tmp{half_set}{iRef}, ... - wCCC_tmp{iRef}); - - - - - cccInitial(iRef,:) = [iRef, particleIDX, ... - 0,0,0, ... - iCCC, 1, ... - shiftVAL]; - - - initialRotPart_FT = []; - + switch alignLoop + % This keeps seperate shifts due to windowing and binning from + % shifts found in CCC + case 1 + + % Estimate the peakshift by rotating the ref not the particle. + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + case 2 + + % Assuming if class specific symmetry, then some not just 1 + if (force_no_symmetry) + symmetry = 'C1'; + end + + % [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... + % rXYZ,... + % {rotConvention ,symmetry,'linear',1,volBinary_tmp}, ... + % 'GPU', 'inv',inputVectors); + [ iTrimParticle ] = particleInterpolator.interp3d(... + RotMat,... + rXYZ,rotConvention ,... + 'inv',symmetry); + + + % iTrimParticle = iTrimParticle(... + % padWindow(1,1) + 1:end - padWindow(2,1) , ... + % padWindow(1,2) + 1:end - padWindow(2,2) , ... + % padWindow(1,3) + 1:end - padWindow(2,3) ); + % + % iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... + % {rotConvention ,symmetry,'linear',1,wdgBinary_tmp},... + % 'GPU', 'inv',inputWgtVectors); + + [ iWedgeMask ] = imgWdgInterpolator.interp3d(... + RotMat,... + [0,0,0],rotConvention ,... + 'inv',symmetry); + + + + + + + + end - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - - + + + if alignLoop == 1 + + % use transpose of RotMat + % try + % iRotRef = BH_resample3d(ref_FT2_tmp{half_set}{rRef}, RotMat', ... + % rXYZ, {rotConvention ,1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); + % catch + % cccPreRefineSort(1,1) + % end + % iRotWdg = BH_resample3d(ref_WGT_rot{half_set}{rRef}, RotMat', ... + % [0,0,0], {rotConvention ,1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); + % + + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + rXYZ,rotConvention ,... + 'forward','C1'); + + + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{rRef},'',... + padCalc,flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{rRef} ,'',padCalc,flgPrecision); + + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + % 2016-11-11 also took out (+ rXYZ) + cccStorage3(iRefine,:) = [rRef, rPart, ... + searchList(iRefine,:), ... + 1, 1, ... + peakCoord+rXYZ]; + else + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{rRef},volBinary_tmp,... + padCalc,flgPrecision); [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{half_set}{iRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{half_set}{iRef}, ... - wCCC_tmp{iRef}); - - - - - - - % Note that no new translational estimate is made, so no - % need to multiply by RotMat - cccStorage2(angCount,:) = ... - [iRef, particleIDX, ... - phi, theta, psi , ... - iCCC, 1, ... - estPeakCoord]; - - - angCount = angCount + 1; + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1_tmp{half_set}{rRef},... + ifftshift(iWedgeMask),... + ref_WGT_tmp{half_set}{rRef}, ... + wCCC_tmp{iRef}); + + + cccStorage3(iRefine,:) = [rRef, rPart, ... + searchList(iRefine,:), ... + iCCC, 1, ... + rXYZ] ; + end + end + end - - - end % loop over references. + + sortRef = sortrows(gather(cccStorage3),-6); + cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); + + end % end of refinement loop - - - end - % This volume won't be needed until the next subTomo is considered, - % which is also where getInitialCCC Boolean is set to True again. - iTrimInitial = []; - getInitialCCC = 0; - - end % in plane angles - end % azimuth - end % polar - -% % % fprintf('Power ratio is %3.3f\n',powerOut./powerInitial); - - cccPreRefineSort = sortrows(gather(cccStorage2),-6); - - if (length(refToAlign) > 1) - cccInitial = sortrows(gather(cccInitial), -6); - cccInitial = cccInitial(1,:); - else - cccInitial = gather(cccInitial(refToAlign,:)); - - end - - if cccInitial(1,6 ) > cccPreRefineSort(1,6) - cccPreRefineSort(1,:) = cccInitial(1,:); - end - - - - % This only seems to be a problem with cut out volumes. - % Normalization maybe? - if ~any(cccPreRefineSort(1,:)) - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); - continue - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - if (flgRefine) - - - - % Get the results from just this subTomo and sort on CCC - - rRef = cccPreRefineSort(1,1); - rPart = cccPreRefineSort(1,2); - rPhi = cccPreRefineSort(1,3); - rPhiInc = phiInc / 4; - rTheta= cccPreRefineSort(1,4); - rTheInc = thetaInc /2; - rPsi = cccPreRefineSort(1,5); - rPsiInc = psiInc /2; - % Confirm shiftVAL is doing what it should be - rXYZest = cccPreRefineSort(1,8:10); - - if (rTheInc) - % For a larger out of plane step, search a larger range in plane - psiRefineStep = floor(sqrt(rTheInc)); - else - psiRefineStep = 1; - end - - thetaRefineStep =1; - phiRefineStep=2; - totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; - totalRefineStep = prod((2.*totalRefineStep)+1); - - cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); - - if (rPsiInc == 0) - inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; - else - inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; - end - polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; - azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; - - searchList = zeros(totalRefineStep,3); - nSearch = 1; - for iPhi = azimuthalRefine - for iTheta = polarRefine - for iPsi = inPlaneRefine - % best iPsi is origin Psi - Phi, no need to subtract here. - - searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; - - nSearch = nSearch + 1; - end - end - end % end of building angle list - - for iRefine = 1:nSearch-1 - for alignLoop = 1:2 - if alignLoop == 1 - rXYZ = rXYZest; - elseif alignLoop == 2 - rXYZ = cccStorage3(iRefine,8:10); + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Get the final translational shift for the best scoring angular + % match. + try + if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) + bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); + + + else + bestRotPeak = cccPreRefineSort(1,:); + bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); + + + end + catch + fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); + cccStorageRefine{iPeak}(iSubTomo,:) + cccPreRefineSort(1,:) + end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); + finalRef = bestRotPeak(1,1); + finalPart = bestRotPeak(1,2); + finalPhi = bestRotPeak(1,3); + finalTheta= bestRotPeak(1,4); + finalPsi = bestRotPeak(1,5); + % Confirm shiftVAL is doing what it should be + finalrXYZest = bestRotPeak(1,8:10); + + RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); RotMat = reshape(angles,3,3) * RotMat; - - - - - switch alignLoop - % This keeps seperate shifts due to windowing and binning from - % shifts found in CCC - case 1 - - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - case 2 - - % Assuming if class specific symmetry, then some not just 1 - if (force_no_symmetry) - symmetry = 'C1'; - end - -% [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... -% rXYZ,... -% {rotConvention ,symmetry,'linear',1,volBinary_tmp}, ... -% 'GPU', 'inv',inputVectors); - [ iTrimParticle ] = particleInterpolator.interp3d(... - RotMat,... - rXYZ,rotConvention ,... - 'inv',symmetry); - - -% iTrimParticle = iTrimParticle(... -% padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); -% -% iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... -% {rotConvention ,symmetry,'linear',1,wdgBinary_tmp},... -% 'GPU', 'inv',inputWgtVectors); - - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... - RotMat,... - [0,0,0],rotConvention ,... - 'inv',symmetry); - - - - - - - - end - - - - if alignLoop == 1 - - % use transpose of RotMat -% try -% iRotRef = BH_resample3d(ref_FT2_tmp{half_set}{rRef}, RotMat', ... -% rXYZ, {rotConvention ,1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); -% catch -% cccPreRefineSort(1,1) -% end -% iRotWdg = BH_resample3d(ref_WGT_rot{half_set}{rRef}, RotMat', ... -% [0,0,0], {rotConvention ,1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); -% - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - rXYZ,rotConvention ,... - 'forward','C1'); - - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{rRef},'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{rRef} ,'',padCalc,flgPrecision); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - - % 2016-11-11 also took out (+ rXYZ) - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - 1, 1, ... - peakCoord+rXYZ]; - else - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{rRef},volBinary_tmp,... - padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{half_set}{rRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{half_set}{rRef}, ... - wCCC_tmp{iRef}); - - - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - iCCC, 1, ... - rXYZ] ; - end - end - - end - - sortRef = sortrows(gather(cccStorage3),-6); - cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); - - end % end of refinement loop - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % Get the final translational shift for the best scoring angular - % match. - try - if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) - bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); - - - else - bestRotPeak = cccPreRefineSort(1,:); - bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); - - - end - catch - fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); - cccStorageRefine{iPeak}(iSubTomo,:) - cccPreRefineSort(1,:) - - end - - finalRef = bestRotPeak(1,1); - finalPart = bestRotPeak(1,2); - finalPhi = bestRotPeak(1,3); - finalTheta= bestRotPeak(1,4); - finalPsi = bestRotPeak(1,5); - % Confirm shiftVAL is doing what it should be - finalrXYZest = bestRotPeak(1,8:10); - - RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - - - - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - + + + + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + % use transpose of RotMat %%% 2016-11-11 estPeakCoord should have been finalrXYZest in %%% the last writing, but now switching to zeros try -% iRotRef = BH_resample3d(ref_FT2_tmp{half_set}{finalRef}, RotMat', ... -% finalrXYZest, {rotConvention ,1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); -% iRotWdg = BH_resample3d(ref_WGT_rot{half_set}{finalRef}, RotMat', ... -% [0,0,0], {rotConvention ,1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - finalrXYZest,rotConvention ,... - 'forward','C1'); - - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); + % iRotRef = BH_resample3d(ref_FT2_tmp{half_set}{finalRef}, RotMat', ... + % finalrXYZest, {rotConvention ,1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); + % iRotWdg = BH_resample3d(ref_WGT_rot{half_set}{finalRef}, RotMat', ... + % [0,0,0], {rotConvention ,1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); + + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + finalrXYZest,rotConvention ,... + 'forward','C1'); + + + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); catch fprintf('\n\nFinal ref,part,phi,theta,psi %f %f %f %f %f\n\n',... bestRotPeak(:,1:5)); - bestRotPeak(1,1:5) - fprintf('BreakPeak %d\n',breakPeak); + bestRotPeak(1,1:5) + fprintf('BreakPeak %d\n',breakPeak); error('errrorsoedfsdf') end - - -% iRotRef = ... -% iRotRef(padWindow(1,1) + 1:end - padWindow(2,1) , ... -% padWindow(1,2) + 1:end - padWindow(2,2) , ... -% padWindow(1,3) + 1:end - padWindow(2,3) ); - - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{finalRef} ,'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{finalRef} ,'',padCalc,flgPrecision ); - - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - - - -% % % end - - - - - % Subtract shiftVAL since this is due to windowing, not the actual - % position. - cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... - peakCoord + finalrXYZest - shiftVAL]) ; - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - % It is probably more useful see the shifts in the particle - % reference frame vs. the avg which was the original - if (bh_global_print_shifts_in_particle_basis) - printShifts = zeros(3,3); - printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); - printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); - printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); - else - printShifts = [cccInitial(1,end-2:end); ... - cccPreRefineSort(1,end-2:end);... - cccStorageBest{iPeak}(iSubTomo,end-2:end)]; - end - - % Print out in Angstrom - printShifts = printShifts .* pixelSize; - - - deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); - if (deltaCCC < 0) && (abs(deltaCCC) > 0.15*cccInitial(1,6)) - fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); - fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + + + % iRotRef = ... + % iRotRef(padWindow(1,1) + 1:end - padWindow(2,1) , ... + % padWindow(1,2) + 1:end - padWindow(2,2) , ... + % padWindow(1,3) + 1:end - padWindow(2,3) ); + + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{finalRef} ,'',... + padCalc,flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{finalRef} ,'',padCalc,flgPrecision ); + + + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + + + % % % end + + + + + % Subtract shiftVAL since this is due to windowing, not the actual + % position. + cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... + peakCoord + finalrXYZest - shiftVAL]) ; + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + % It is probably more useful see the shifts in the particle + % reference frame vs. the avg which was the original + if (bh_global_print_shifts_in_particle_basis) + printShifts = zeros(3,3); + printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); + printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); + printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); + else + printShifts = [cccInitial(1,end-2:end); ... + cccPreRefineSort(1,end-2:end);... + cccStorageBest{iPeak}(iSubTomo,end-2:end)]; + end + + % Print out in Angstrom + printShifts = printShifts .* pixelSize; + + + deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); + if (deltaCCC < 0) && (abs(deltaCCC) > 0.15*cccInitial(1,6)) + fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); + fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + + end + + if (track_stats) + + if thetaInc > 0 + cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); + else + cccStorageBest{iPeak}(iSubTomo,end-3) = 0; + end + + % % I'm not sold on what do do with this. The distribution over the + % % shift parameters doesn't really seem to make sense to me. There + % % are too many factors that can lead to large shifts (e.g. + % % tomoCPR) If we were searching the full angular space each + % % iteration, then this would make sense. + % mip_mean = mip.X./mip.N; + % mip_covar = mip.X2./mip.N - transpose(mip_mean)*(mip_mean); + % mip_covar_inv = mip_covar\eye(3); + % gauss_norm = ((2.*pi).^(3/2).*abs(mip_covar)).^-1; + % gauss_exp = exp(-0.5.*(printShifts(2,:)-mip_mean)*mip_covar_inv*transpose(printShifts(2,:)-mip_mean)); + + end + + + cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); + cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); + if (flgRefine) + cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:), ... + 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... + cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... + 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + + else + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + + end + + + end % if condition on newly ignored particles + + end - end - - if (track_stats) - - if thetaInc > 0 - cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); - else - cccStorageBest{iPeak}(iSubTomo,end-3) = 0; + if ~(rem(iSubTomo,100)) + timeClass = toc; + fprintf('\nworking on %d/%d subTomo from %s...%fs\n',... + iSubTomo,nSubTomos,tomoName,timeClass); + tic; end -% % I'm not sold on what do do with this. The distribution over the -% % shift parameters doesn't really seem to make sense to me. There -% % are too many factors that can lead to large shifts (e.g. -% % tomoCPR) If we were searching the full angular space each -% % iteration, then this would make sense. -% mip_mean = mip.X./mip.N; -% mip_covar = mip.X2./mip.N - transpose(mip_mean)*(mip_mean); -% mip_covar_inv = mip_covar\eye(3); -% gauss_norm = ((2.*pi).^(3/2).*abs(mip_covar)).^-1; -% gauss_exp = exp(-0.5.*(printShifts(2,:)-mip_mean)*mip_covar_inv*transpose(printShifts(2,:)-mip_mean)); - - end + + iParticle = []; + iSymParti = []; + iTrimParticle = []; + iAsym = []; + iTrimAsym = []; + iWedgeMask = []; + rotPart_FT = []; + rotParticle = []; + end % end loop over possible peaks + iMaxWedgeIfft = []; + end % loop over subTomos + + + for iPeak = 1:emc.nPeaks - cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); - cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); - if (flgRefine) - cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:), ... - 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... - cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... - 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - + % Get rid of any zero entries left over from pre-initialization + if iPeak == 1 + nonZeroInits = ( cccStorageBest{iPeak}(:,2) ~= 0 ); + cccStorageBest{1}=cccStorageBest{1}(nonZeroInits,:); + sortCCC = zeros(size(cccStorageBest{1},1),10*emc.nPeaks); else - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - + cccStorageBest{iPeak}=cccStorageBest{iPeak}(nonZeroInits,:); end - - - end % if condition on newly ignored particles - - end - - if ~(rem(iSubTomo,100)) - timeClass = toc; - fprintf('\nworking on %d/%d subTomo from %s...%fs\n',... - iSubTomo,nSubTomos,tomoName,timeClass); - tic; - end - - - iParticle = []; - iSymParti = []; - iTrimParticle = []; - iAsym = []; - iTrimAsym = []; - iWedgeMask = []; - rotPart_FT = []; - rotParticle = []; - end % end loop over possible peaks - iMaxWedgeIfft = []; - end % loop over subTomos - - - for iPeak = 1:emc.nPeaks - - % Get rid of any zero entries left over from pre-initialization - if iPeak == 1 - nonZeroInits = ( cccStorageBest{iPeak}(:,2) ~= 0 ); - cccStorageBest{1}=cccStorageBest{1}(nonZeroInits,:); - sortCCC = zeros(size(cccStorageBest{1},1),10*emc.nPeaks); - else - cccStorageBest{iPeak}=cccStorageBest{iPeak}(nonZeroInits,:); + sortCCC(:,1+10*(iPeak-1):10+10*(iPeak-1)) = cccStorageBest{iPeak}; end - - sortCCC(:,1+10*(iPeak-1):10+10*(iPeak-1)) = cccStorageBest{iPeak}; - end - -% % % % I think this is redundant now, but leaving until I double check. -% % % save('sortCCC.mat','sortCCC'); - [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); - - cccSortedandUnique = sortCCC(a,:); -% % % save('cccSortedandUnique.mat','cccSortedandUnique'); -% % % g = gather(geometry); -% % % save('TBL_geom.mat','g'); - - bestAngles_tmp.(tomoList{iTomo}) = gather(cccSortedandUnique); - - % save doesn't work in a parfor, so write out the results for each tomogram so that a - % run may be resumed if cancelled. - angOut = fopen(sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}),'w'); - - for iRow = 1:size( bestAngles_tmp.(tomoList{iTomo}),1) - for iPeak = 1:emc.nPeaks - fprintf(angOut,'%d %d %6.3f %6.3f %6.3f %6.6f %6.6f %6.3f %6.3f %6.3f ', ... - bestAngles_tmp.(tomoList{iTomo})(iRow,1+10*(iPeak-1):10+10*(iPeak-1))); - end - fprintf(angOut,'\n'); - end - fclose(angOut); - - end % if clause to check for previous alignment + + % % % % I think this is redundant now, but leaving until I double check. + % % % save('sortCCC.mat','sortCCC'); + [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); + + cccSortedandUnique = sortCCC(a,:); + % % % save('cccSortedandUnique.mat','cccSortedandUnique'); + % % % g = gather(geometry); + % % % save('TBL_geom.mat','g'); + + bestAngles_tmp.(tomoList{iTomo}) = gather(cccSortedandUnique); + + % save doesn't work in a parfor, so write out the results for each tomogram so that a + % run may be resumed if cancelled. + angOut = fopen(sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}),'w'); + + for iRow = 1:size( bestAngles_tmp.(tomoList{iTomo}),1) + for iPeak = 1:emc.nPeaks + fprintf(angOut,'%d %d %6.3f %6.3f %6.3f %6.6f %6.6f %6.3f %6.3f %6.3f ', ... + bestAngles_tmp.(tomoList{iTomo})(iRow,1+10*(iPeak-1):10+10*(iPeak-1))); + end + fprintf(angOut,'\n'); + end + fclose(angOut); + + end % if clause to check for previous alignment end % loop over tomos bestAnglesResults{iParProc} = bestAngles_tmp; geometryResults{iParProc} = geometry_tmp; -%profile off -%profsave + %profile off + %profsave end % parfor @@ -1811,26 +1811,26 @@ save('bestAnglesResults.mat', 'bestAnglesResults'); bestAngles = struct(); - for iParProc = 1:nParProcesses + for iParProc = 1:nParProcesses for iTomo = iterList{iParProc} geometry.(tomoList{iTomo}) = geometryResults{iParProc}.(tomoList{iTomo}); bestAngles.(tomoList{iTomo}) = bestAnglesResults{iParProc}.(tomoList{iTomo}); end end -% save('bestAnglesTemp.mat', 'bestAngles'); - save('bestAngles.mat', 'bestAngles'); - + % save('bestAnglesTemp.mat', 'bestAngles'); + save('bestAngles.mat', 'bestAngles'); + [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, emc.nPeaks, rotConvention, updateWeights, updateClassByBestReferenceScore); masterTM.(cycleNumber).('RawAlign') = rawAlign; masterTM.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); masterTM.('updatedWeights') = true; - + clear bestAngles rawAlign subTomoMeta = masterTM; save(emc.('subTomoMeta'), 'subTomoMeta'); - - + + end delete(gcp('nocreate')) diff --git a/alignment/BH_fitBeads.m b/alignment/BH_fitBeads.m index 4d7e3057..95083208 100644 --- a/alignment/BH_fitBeads.m +++ b/alignment/BH_fitBeads.m @@ -10,7 +10,7 @@ g1 = 2.5; % number of stdDev the central gaussian should fall by the bead radius defined below %s1 = pixelRadius. Set in loop so fractions of pixel radius can be searched b/c beads are not uniform diameter (but right now only looking at 1) s2 = 1.25; % radius for the edge gaussian, I suspect this should be closer to 1 or even 0.5? -g2 = s2./2; % center the edge gaussian at pixel_radius + g2 +g2 = s2./2; % center the edge gaussian at pixel_radius + g2 g3 = 6; % give the central gaussian a more flat top to match what a bead looks like. Abs is taken so odd values okay show_ref_profile = false; @@ -20,7 +20,7 @@ bgVal = 0; edgeVal = 4; beadVal = -4; - + input_ts = gpuArray(getVolume(MRCImage(img_name))); system(sprintf('model2point -contour %s %s.txt',input_name,input_name)); input_pos= gpuArray(load(sprintf('%s.txt',input_name))); @@ -32,22 +32,22 @@ firstLoop = false; KERNEL = EMC_gaussianKernel([1,3], 0.5, 'gpu', {}); - + for iPrj = 1:nPrjs img = input_ts(:,:,iPrj); - + lCoord = input_pos(:,4) == iPrj -1; - + [w] = BH_multi_gridCoordinates([d1,d2],'Cartesian','GPU',{'none'},1,0,1); img_derivative = real(ifftn(fftn(img).*w)); -% img_derivative = EMC_convn(single(img_derivative), KERNEL); - + % img_derivative = EMC_convn(single(img_derivative), KERNEL); + img_derivative = img_derivative - mean(img_derivative(:)); img_derivative = img_derivative ./ rms(img_derivative(:)); - + r = ceil(pixel_radius.*3); avg_size = 2.*r+1 .*[1,1]; x = input_pos(lCoord,2) - 0.5; % in imod, model point 7.5 is in the "middle" of pixel 7 in emClarity, a pixel is 6.5 - 7.5 with 7 in the middle @@ -56,8 +56,8 @@ yi = floor(y); xf = x - xi; yf = y - yi; - - + + % Note if you put this into parallel, this will not work. if (firstLoop) % Get an avgerage bead for this projection to determine the intensities in @@ -66,9 +66,9 @@ avg_bead = zeros(avg_size,'single','gpuArray'); n = 0; for i = 1:length(x) - % add edge checking - avg_bead = avg_bead + img_derivative(xi(i)-r:xi(i)+r,yi(i)-r:yi(i)+r); - n = n + 1; + % add edge checking + avg_bead = avg_bead + img_derivative(xi(i)-r:xi(i)+r,yi(i)-r:yi(i)+r); + n = n + 1; end avg_bead = avg_bead ./ n ; % Get the mean values for the three main pixel values @@ -76,157 +76,157 @@ meanVect = gFit.mu; [beadVal,beadCoord] = min(meanVect); [edgeVal,edgeCoord] = max(meanVect); - + switch (beadCoord+edgeCoord) case 3 bgVal = meanVect(3); case 4 bgVal = meanVect(2); - case 5 + case 5 bgVal = meanVect(1); otherwise error('failed to find the correct index for the background mean'); end firstLoop = false; end - - - + + + % The edge radius should be around 1 pix unless you blur the derivative % first. if (show_ref_profile) v = -6:0.01:6; figure, plot(v, beadVal.* exp(-abs((g1.*v./pixel_radius).^g3)) + .... - edgeVal.*(exp(-(g1.*(v-g2-pixel_radius)./s2).^2) + exp(-(g1.*(v+g2 + pixel_radius)./s2).^2)) + ... - bgVal); + edgeVal.*(exp(-(g1.*(v-g2-pixel_radius)./s2).^2) + exp(-(g1.*(v+g2 + pixel_radius)./s2).^2)) + ... + bgVal); end - - + + [t] = BH_multi_gridCoordinates(avg_size,'Cartesian','GPU',{'none'},0,1,1); - - - - - + + + + + nBeads = length(x); - nSkipped = 0; + nSkipped = 0; s1 = pixel_radius; ref = beadVal.* exp(-abs((g1.*t./s1).^g3)) + .... - edgeVal.*(exp(-(g1.*(t-g2-s1)./s2).^2)) + ... - bgVal; + edgeVal.*(exp(-(g1.*(t-g2-s1)./s2).^2)) + ... + bgVal; ref = gpuArray(ref); imgFT = fftn(img_derivative); global_ref = zeros([d1,d2],'single','gpuArray'); - - for i = 1:nBeads - xl = xi(i)-r; - xh = xi(i)+r; - yl = yi(i)-r; - yh = yi(i)+r; - - if (xl < 1 || yl < 1 || xh > d1 || yh > d2) + + for i = 1:nBeads + xl = xi(i)-r; + xh = xi(i)+r; + yl = yi(i)-r; + yh = yi(i)+r; + + if (xl < 1 || yl < 1 || xh > d1 || yh > d2) continue; - else - global_ref(xl:xh,yl:yh,1) = ref; - end + else + global_ref(xl:xh,yl:yh,1) = ref; + end end - + global_ref = fftshift(real(ifftn(conj(fftn(global_ref)).*imgFT))); - max_global_shift = 14; - global_tile = max_global_shift.*[2,2]+1; - pad_GF = BH_multi_padVal([d1,d2],global_tile); - global_ref = BH_padZeros3d(global_ref,'fwd',pad_GF,'GPU','single'); - - [~,maxCoord] = max(global_ref(:)); - [mi,mj] = ind2sub(global_tile,maxCoord); - global_shifts = [mi,mj] - (max_global_shift+1); - xi = xi + global_shifts(1); - yi = yi + global_shifts(2); - x = x + global_shifts(1); - y = y + global_shifts(2); + max_global_shift = 14; + global_tile = max_global_shift.*[2,2]+1; + pad_GF = BH_multi_padVal([d1,d2],global_tile); + global_ref = BH_padZeros3d(global_ref,'fwd',pad_GF,'GPU','single'); + + [~,maxCoord] = max(global_ref(:)); + [mi,mj] = ind2sub(global_tile,maxCoord); + global_shifts = [mi,mj] - (max_global_shift+1); + xi = xi + global_shifts(1); + yi = yi + global_shifts(2); + x = x + global_shifts(1); + y = y + global_shifts(2); mip = zeros([d1,d2],'single','gpuArray'); padVal = BH_multi_padVal(size(t),[d1,d2]); - + % Loop over references of different fractions of the particle radius. For % now, just using 1. for iRef = [0.80:0.05:1.2] s1 = pixel_radius * iRef; - + ref = beadVal.* exp(-abs((g1.*t./s1).^g3)) + .... - edgeVal.*(exp(-(g1.*(t-g2-s1)./s2).^2)) + ... - bgVal; + edgeVal.*(exp(-(g1.*(t-g2-s1)./s2).^2)) + ... + bgVal; ref = BH_padZeros3d(ref,'fwd',padVal,'GPU','single'); - + ccf = real(fftshift(ifftn(conj(fftn(ref)).*imgFT))); lMip = ccf > mip; mip(lMip) = ccf(lMip); end - + if (show_mip) figure, imshow3D(gather(real(mip))); end % Get the updated peak positions, with optional over-sampling of the bead % an option to pad zeros in half transforms would be nice here. Just use % native matlab FFT for now. - + r = ceil(pixel_radius.*1.0); rp = padBy.*(2.*r+1).*[1,1]; padVal = BH_multi_padVal( (2.*r+1).*[1,1] , rp ); - maskRadius = 0.5.*(1- 2.5./sampling).*r.*[1,1]; + maskRadius = 0.5.*(1- 2.5./sampling).*r.*[1,1]; peakMask = BH_mask3d('sphere',padBy.*(2.*r+1).*[1,1],maskRadius,[0,0],'2d'); xo = x; yo = y; [bx,by] = ndgrid(gpuArray(-r:r),gpuArray(-r:r)); - - ro = floor(rp/2) + 1; + + ro = floor(rp/2) + 1; for i = 1:nBeads - % add edge checking - - xl = xi(i)-r; - xh = xi(i)+r; - yl = yi(i)-r; - yh = yi(i)+r; - - if (xl < 1 || yl < 1 || xh > d1 || yh > d2) - fprintf('skipping bead %d/%d\n', i,nBeads); - nSkipped = nSkipped + 1; - xo(i) = x(i); - yo(i) = y(i); - else - ccf = mip(xl:xh,yl:yh); - ccf = fftshift(fftn(ccf)); - ccf = BH_padZeros3d(ccf,'fwd',padVal,'GPU','single'); - ccf = real(ifftn(ifftshift(ccf))).*peakMask; - - [~,maxCoord] = max(ccf(:)); - [mi,mj] = ind2sub(rp,maxCoord); - mi = (mi- ro(1))./ padBy; - mj = (mj- ro(2))./ padBy; -% ccf = log(ccf+1); -% % % % % comX = sum(sum(bx.*ccf))./sum(ccf(:)); -% % % % % comY = sum(sum(by.*ccf))./sum(ccf(:)); -% % % % % xo(i) = (comX + x(i)); -% % % % % yo(i) = (comY + y(i)); - - xo(i) = (mi - xf(i) + x(i)); - yo(i) = (mj - yf(i) + y(i)); - end + % add edge checking + + xl = xi(i)-r; + xh = xi(i)+r; + yl = yi(i)-r; + yh = yi(i)+r; + + if (xl < 1 || yl < 1 || xh > d1 || yh > d2) + fprintf('skipping bead %d/%d\n', i,nBeads); + nSkipped = nSkipped + 1; + xo(i) = x(i); + yo(i) = y(i); + else + ccf = mip(xl:xh,yl:yh); + ccf = fftshift(fftn(ccf)); + ccf = BH_padZeros3d(ccf,'fwd',padVal,'GPU','single'); + ccf = real(ifftn(ifftshift(ccf))).*peakMask; + + [~,maxCoord] = max(ccf(:)); + [mi,mj] = ind2sub(rp,maxCoord); + mi = (mi- ro(1))./ padBy; + mj = (mj- ro(2))./ padBy; + % ccf = log(ccf+1); + % % % % % comX = sum(sum(bx.*ccf))./sum(ccf(:)); + % % % % % comY = sum(sum(by.*ccf))./sum(ccf(:)); + % % % % % xo(i) = (comX + x(i)); + % % % % % yo(i) = (comY + y(i)); + + xo(i) = (mi - xf(i) + x(i)); + yo(i) = (mj - yf(i) + y(i)); + end end - + fprintf('Updated the fit for %d/%d beads\n', nBeads - nSkipped, nBeads); if (show_results) figure, imshow3D(gather(img_derivative)); hold on - plot(y,x,'ro','MarkerSize',7); + plot(y,x,'ro','MarkerSize',7); plot(yo,xo, 'b+','MarkerSize', 7); end % We need to add back the 0.5 for imod model coords output_pos(lCoord,2) = gather(xo + 0.5); output_pos(lCoord,3) = gather(yo + 0.5); output_pos(lCoord,[1,4]) = gather(input_pos(lCoord,[1,4])); - + end f = fopen(sprintf('%s.txt',output_name),'w'); diff --git a/alignment/BH_refine_on_beads.m b/alignment/BH_refine_on_beads.m index 4993deb4..ddbe70f0 100644 --- a/alignment/BH_refine_on_beads.m +++ b/alignment/BH_refine_on_beads.m @@ -34,51 +34,51 @@ % Step down in samping rate sampling_step=2; -for imageBinning = [15:-sampling_step:min_sampling_rate] +for imageBinning = [15:-sampling_step:min_sampling_rate] - - if first_run + + if first_run input_name=baseName; - output_name=sprintf('%s_%d',baseName,imageBinning); + output_name=sprintf('%s_%d',baseName,imageBinning); last_binning=imageBinning; first_run=false; xTiltOption = '-XAXISTILT 0.0 '; z_factor_file = ''; local_file = sprintf('-LOCALFILE %s.local ',input_name); else - input_name=sprintf('%s_%d',baseName,last_binning); - output_name=sprintf('%s_%d',baseName,imageBinning); + input_name=sprintf('%s_%d',baseName,last_binning); + output_name=sprintf('%s_%d',baseName,imageBinning); last_binning=imageBinning; xTiltOption = sprintf('-XTILTFILE %s.Xtlt ',input_name); z_factor_file = sprintf('-ZFACTORFILE %s.Zfactor ', input_name); if isempty(doLocal) local_file = ''; else - local_file = sprintf(' -LOCALFILE %s.local ' , input_name); + local_file = sprintf(' -LOCALFILE %s.local ' , input_name); end end - + % Create a new stack at each binning [ fail ] = system(sprintf( ... - ['newstack ', ... + ['newstack ', ... '-InputFile %s.fixed ', ... % Always the same input (the preprocessed stack) - '-OutputFile %s_3dfind.ali ', ... + '-OutputFile %s_3dfind.ali ', ... '-TransformFile %s.xf ', ... % Coming from the last iteration '-ImagesAreBinned 1.0 ', ... % Input image binning is always 1 - '-BinByFactor %d'],baseName,output_name,input_name,imageBinning)); - - if (fail) - error('failed in newstack') + '-BinByFactor %d'],baseName,output_name,input_name,imageBinning)); + + if (fail) + error('failed in newstack') end - + if ( n_findBeads3d > 0) n_findBeads3d = n_findBeads3d - 1; - + % The defaults are taken from etomo on whatever tilt I had developed % this on - may not be optimal. Log option can produce problems if not % using preprocessed stack. [ fail ] = system(sprintf([... - 'tilt ' ... + 'tilt ' ... '-InputProjections %s_3dfind.ali ' ... '-OutputFile %s_3dfind.rec ' ... '-TILTFILE %s.tlt ' ... @@ -94,15 +94,15 @@ '-SHIFT 0.0 0.0 ' ... '-PERPENDICULAR ' ... '-FULLIMAGE %d,%d %s %s'],output_name,output_name,... - input_name,imageBinning,thickness,... - xTiltOption,NX,NY,local_file,z_factor_file)); - - if (fail) - error('failed in creating a 3d to look for beads') + input_name,imageBinning,thickness,... + xTiltOption,NX,NY,local_file,z_factor_file)); + + if (fail) + error('failed in creating a 3d to look for beads') end - + [ fail ] = system(sprintf([... - 'findbeads3d ' ... + 'findbeads3d ' ... '-InputFile %s_3dfind.rec ' ... '-OutputFile %s_3dfind.mod ' ... '-BeadSize %f ' ... @@ -114,123 +114,123 @@ '-BinningOfVolume %d'], ... output_name,output_name,bead_size,imageBinning ... )); - - if (fail) - error('failed in findbeads3d') - end - % The output is a 3d model, with YZ flipped. Project it perpendicular - % into natural orientation. - [ fail ] = system(sprintf([... - 'tilt ' ... - '-InputProjections %s_3dfind.ali ' ... - '-OutputFile %s.erase ' ... - '-IMAGEBINNED %d ' ... - '-TILTFILE %s.tlt ' ... - '-THICKNESS %d ' ...ls - '-RADIAL 0.35,0.035 ' ... - '-FalloffIsTrueSigma 1 ' ... - '%s ' ... - '-UseGPU 0 ' ... - '-ActionIfGPUFails 1,2 ' ... - '-OFFSET 0.0 ' ... - '-SHIFT 0.0,0.0 ' ... - '-ProjectModel %s_3dfind.mod ' ... - '-FULLIMAGE %d,%d ' ... - '-PERPENDICULAR ' ... - '-MODE 2 %s %s'], output_name,output_name,imageBinning,... - input_name,thickness,... - xTiltOption,output_name,NX,NY,local_file,z_factor_file)); - - if (fail) - error('failed in tilt') - end - - % Fix anything missed in findbeads3d - [ fail ] = system(sprintf([... + if (fail) + error('failed in findbeads3d') + end + + % The output is a 3d model, with YZ flipped. Project it perpendicular + % into natural orientation. + [ fail ] = system(sprintf([... + 'tilt ' ... + '-InputProjections %s_3dfind.ali ' ... + '-OutputFile %s.erase ' ... + '-IMAGEBINNED %d ' ... + '-TILTFILE %s.tlt ' ... + '-THICKNESS %d ' ...ls + '-RADIAL 0.35,0.035 ' ... + '-FalloffIsTrueSigma 1 ' ... + '%s ' ... + '-UseGPU 0 ' ... + '-ActionIfGPUFails 1,2 ' ... + '-OFFSET 0.0 ' ... + '-SHIFT 0.0,0.0 ' ... + '-ProjectModel %s_3dfind.mod ' ... + '-FULLIMAGE %d,%d ' ... + '-PERPENDICULAR ' ... + '-MODE 2 %s %s'], output_name,output_name,imageBinning,... + input_name,thickness,... + xTiltOption,output_name,NX,NY,local_file,z_factor_file)); + + if (fail) + error('failed in tilt') + end + + % Fix anything missed in findbeads3d + [ fail ] = system(sprintf([... 'beadtrack ' ... - '-InputSeedModel %s.erase ' ... - '-OutputModel %s_beadtrack.fid ' ... - '-ImageFile %s_3dfind.ali ' ... - '-ImagesAreBinned %d ' ... - '-PixelSize %f ' ... - '-BeadDiameter %f ' ... - '-RoundsOfTracking 3 ' ... - '-BoxSizeXandY %d,%d ' ... - '-MinBeadsInArea 3 ' ... - '-MinOverlapBeads 1 ' ... - '-UnsplitFirstRound ' ... - '-LocalAreaTracking ' ... - '-LocalAreaTargetSize %d ' ... - '-RotationAngle 0.0 ' ... - '-TiltFile %s.tlt ' ... - '-TiltDefaultGrouping 5 ' ... - '-MagDefaultGrouping 1 ' ... - '-RotDefaultGrouping 1 ' ... - '-LowPassCutoffInverseNm 0.71 '], output_name,output_name,output_name,imageBinning,... - pixelSizeInNanometers,bead_size,ptSizeX,ptSizeY,floor(512/imageBinning),... - input_name ... - )); - - if (fail) - error('Failed in beadtrack') - end + '-InputSeedModel %s.erase ' ... + '-OutputModel %s_beadtrack.fid ' ... + '-ImageFile %s_3dfind.ali ' ... + '-ImagesAreBinned %d ' ... + '-PixelSize %f ' ... + '-BeadDiameter %f ' ... + '-RoundsOfTracking 3 ' ... + '-BoxSizeXandY %d,%d ' ... + '-MinBeadsInArea 3 ' ... + '-MinOverlapBeads 1 ' ... + '-UnsplitFirstRound ' ... + '-LocalAreaTracking ' ... + '-LocalAreaTargetSize %d ' ... + '-RotationAngle 0.0 ' ... + '-TiltFile %s.tlt ' ... + '-TiltDefaultGrouping 5 ' ... + '-MagDefaultGrouping 1 ' ... + '-RotDefaultGrouping 1 ' ... + '-LowPassCutoffInverseNm 0.71 '], output_name,output_name,output_name,imageBinning,... + pixelSizeInNanometers,bead_size,ptSizeX,ptSizeY,floor(512/imageBinning),... + input_name ... + )); + + if (fail) + error('Failed in beadtrack') + end else [fail] = system(sprintf('imodtrans -i %s_3dfind.ali -2 %s.tltxf_Scaled %s_fitbyResid_%d.fid %s_beadtrack.fid', ... - output_name, input_name, input_name, n_ali_loops, output_name)); - - if (fail) - error('failed in projected model from previous tiltalign') + output_name, input_name, input_name, n_ali_loops, output_name)); + + if (fail) + error('failed in projected model from previous tiltalign') end - - end - + end + + for aliLoop = 1:n_ali_loops if aliLoop > 1 sprintf('MOVE BY RESIDUAL\n'); % use the residual from the previous, which has a header line -% cmd = sprintf('awk ''FNR==NR{if(FNR>1) {a[FNR-1]=$1-$4 FS $2-$5};next}{ print $1, a[FNR+1],$4}'' %s.resid_%d %s_beadtrack.fid.txt > %s_byResid_%d.txt',output_name,aliLoop-1,output_name,output_name,aliLoop); + % cmd = sprintf('awk ''FNR==NR{if(FNR>1) {a[FNR-1]=$1-$4 FS $2-$5};next}{ print $1, a[FNR+1],$4}'' %s.resid_%d %s_beadtrack.fid.txt > %s_byResid_%d.txt',output_name,aliLoop-1,output_name,output_name,aliLoop); cmd = sprintf('awk ''FNR==NR{a[FNR]=$1;next}{ if(FNR>1) print a[FNR-1],$1+$4 FS $2+$5 FS $3}'' %s_fit.fid.txt %s.resid_%d > %s_byResid_%d.txt',output_name,output_name,aliLoop-1,output_name,aliLoop); - + system(cmd); system(sprintf('point2model -circle %d %s_byResid_%d.txt %s_byResid_%d.fid',floor(bead_size/imageBinning),output_name,aliLoop,output_name,aliLoop)); model_to_align = sprintf('%s_fitbyResid_%d.fid',output_name,aliLoop); [~, nBeads] = BH_fitBeads(pixelSize,bead_diameter,imageBinning,... - sprintf('%s_3dfind.ali',output_name),... - sprintf('%s_byResid_%d.fid',output_name,aliLoop),... - model_to_align... - ); + sprintf('%s_3dfind.ali',output_name),... + sprintf('%s_byResid_%d.fid',output_name,aliLoop),... + model_to_align... + ); else model_to_align = sprintf('%s_fit.fid',output_name); [~, nBeads] = BH_fitBeads(pixelSize,bead_diameter,imageBinning,... - sprintf('%s_3dfind.ali',output_name),... - sprintf('%s_beadtrack.fid',output_name),... - model_to_align... - ); + sprintf('%s_3dfind.ali',output_name),... + sprintf('%s_beadtrack.fid',output_name),... + model_to_align... + ); end - + if (nBeads < 5) to_few_beads = true; return; end - + if (nBeads < 11) doLocal = ''; else doLocal = '-LocalAlignments '; end - - + + % No need to run tiltalign on the last iteration, b/c we use the 2d % model file for the input to the next iter. if (aliLoop < n_ali_loops) - + [ fail ] = system(sprintf([... - 'tiltalign ' ... + 'tiltalign ' ... '-ModelFile %s ' ... '-ImageFile %s_3dfind.ali ' ... '-ImagesAreBinned %d ' ... @@ -281,32 +281,32 @@ output_name,output_name,aliLoop ,output_name,output_name,output_name,output_name,... output_name,input_name,x_tilt_option(1:2),x_stretch_option(1:2),output_name,ptSizeX,ptSizeY,... x_tilt_option(3:4), x_stretch_option(3:4),doLocal,output_name)); - - if (fail) - error('failed in tiltalign') + + if (fail) + error('failed in tiltalign') end end % If cond on tiltAlign - + end % Transforms to full sampling (bin 1) [ fail ] = system(sprintf('xfproduct -scale 1,%d %s.xf %s.tltxf_nonScaled %s.xf', imageBinning,input_name,output_name,output_name)); - if (fail) - error('failed in xfproduct to full sampling') + if (fail) + error('failed in xfproduct to full sampling') end % imodtrans first scales by relative image size, then applies a % transformation as is, so we need to have the incremental transform at % the sampling rate of the next iteration as well. system(sprintf('awk ''{print 1.0, 0.0, 0.0, 1.0, 0.0, 0.0}'' %s.tltxf_nonScaled > dummy.xf',output_name)); [ fail ] = system(sprintf('xfproduct -scale 1,%f dummy.xf %s.tltxf_nonScaled %s.tltxf_Scaled', imageBinning/(imageBinning - sampling_step),output_name,output_name)); - - if (fail) - error('failed in xfproduct to incremental sampling') - end - - - + if (fail) + error('failed in xfproduct to incremental sampling') + end + - end + + + +end end diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index 78a75adc..af7fbfb1 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -7,13 +7,13 @@ skip_tilts = 0; if nargin > 6 - skip_tilts = EMC_str2double(varargin{1}); + skip_tilts = EMC_str2double(varargin{1}); end pixelSize = emc.('PIXEL_SIZE').*10^10; imgRotation = EMC_str2double(imgRotation); -try +try RESOLUTION_CUTOFF = emc.('autoAli_max_resolution'); catch RESOLUTION_CUTOFF=18; @@ -72,7 +72,7 @@ % FIXME this should probably be specified in Ang try - FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(emc.('autoAli_max_shift_in_angstroms')./pixelSize); + FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(emc.('autoAli_max_shift_in_angstroms')./pixelSize); catch FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(40 ./ pixelSize); end @@ -83,7 +83,7 @@ DIVIDE_SHIFT_LIMIT_BY = 1; % int(max_shift / (iter^DIVI...)) + 1 end - + % Now get the bead diameter, if it is zeros override the default to refine % on beads after patch tracking. beadDiameter = emc.('beadDiameter') * 10^10; @@ -93,11 +93,11 @@ - LOW_RES_CUTOFF=800; - CLEAN_UP_RESULTS=false; - MAG_OPTION=5; - tiltAngleOffset=0.0; - TILT_OPTION = 0; +LOW_RES_CUTOFF=800; +CLEAN_UP_RESULTS=false; +MAG_OPTION=5; +tiltAngleOffset=0.0; +TILT_OPTION = 0; inputMRC = MRCImage(stackIN,0); inputStack = single(getVolume(inputMRC)); @@ -126,18 +126,18 @@ % the rotate to avoid information loss branch iHeader = getHeader(inputMRC); iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... - iHeader.cellDimensionY/iHeader.nY, ... - iHeader.cellDimensionZ/iHeader.nZ]; + iHeader.cellDimensionY/iHeader.nY, ... + iHeader.cellDimensionZ/iHeader.nZ]; iOriginHeader= [iHeader.xOrigin , ... - iHeader.yOrigin , ... - iHeader.zOrigin ]; - + iHeader.yOrigin , ... + iHeader.zOrigin ]; + f = load(sprintf('../%s',tiltAngles)); f = f(skip_tilts_logical); fout = fopen(sprintf('%s.rawtlt',baseName),'w'); fprintf(fout,'%f\n',f'); -fclose(fout); +fclose(fout); clear f cd('../'); @@ -184,13 +184,13 @@ if (switch_axes) - -% if ( abs(abs(imgRotation) - 180) > maxAngle ) + + % if ( abs(abs(imgRotation) - 180) > maxAngle ) fprintf('Your image rotation will result in a loss of data. Switching X/Y axes\n') -% switch_axes = true; + % switch_axes = true; rotStack = zeros(nY,nX,nZ,'single'); -% + % ny = nY; nY = nX; nX = ny; @@ -200,11 +200,11 @@ % came off the scope. system(sprintf('newstack -fromone -secs %d -rotate 90 fixedStacks/%s.fixed %s >/dev/null',iPrj,baseName,tmpFile)); rotStack(:,:,iPrj) = getVolume(MRCImage(sprintf('%s',tmpFile))); - + system(sprintf('rm %s',tmpFile)); end inputStack = rotStack; clear rotStack - imgRotation = imgRotation + 90; + imgRotation = imgRotation + 90; SAVE_IMG(inputStack,sprintf('fixedStacks/%s.fixed',baseName),iPixelHeader,iOriginHeader); elseif ( skip_tilts) % Originally saved in the skip_tilts block, but that is redundant if we @@ -223,20 +223,20 @@ fprintf('Preprocessing tilt-series\n'); %gradientAliasFilter = BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',pixelSize); - gradientAliasFilter = {BH_bandpass3d(1.*[nX,nY,1],0,0,0,'GPU','nyquistHigh'),... - BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',pixelSize)}; -if pixelSize < 2 +gradientAliasFilter = {BH_bandpass3d(1.*[nX,nY,1],0,0,0,'GPU','nyquistHigh'),... + BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',pixelSize)}; +if pixelSize < 2 medianFilter = 5; else medianFilter = 3; end for iPrj = 1:nZ -% tmpPrj = BH_preProcessStack(gpuArray(inputStack(:,:,iPrj)),gradientAliasFilter,medianFilter); + % tmpPrj = BH_preProcessStack(gpuArray(inputStack(:,:,iPrj)),gradientAliasFilter,medianFilter); tmpPrj = real(ifftn(fftn(gpuArray(inputStack(:,:,iPrj))).*gradientAliasFilter{1})); tmpPrj = medfilt2(tmpPrj,medianFilter.*[1,1]); tmpPrj = real(ifftn(fftn(tmpPrj).*gradientAliasFilter{2})); -% tmpPrj = BH_resample2d(tmpPrj,rotMat,[0,0],'Bah','GPU','inv',1,size(tmpPrj)); + % tmpPrj = BH_resample2d(tmpPrj,rotMat,[0,0],'Bah','GPU','inv',1,size(tmpPrj)); inputStack(:,:,iPrj) = gather(tmpPrj); end @@ -253,30 +253,30 @@ fclose(rotFile); - - + + system('pwd') fprintf('Running %s\n',runPath); system(sprintf('%s %s %f %f %d %d %d %d %d %d %s %d %d %d %f %f %f %d %d %d > ./emC_autoAliLog_%s.txt',... - runPath, ... - baseName, ... - pixelSize, ... - imgRotation, ... - binHigh, ... - binLow, ... - binInc,... - nX,nY,nZ,ext,... - PATCH_SIZE_FACTOR,... - N_ITERS_NO_ROT,... - BORDER_SIZE_PIXELS,... - PATCH_OVERLAP,... - RESOLUTION_CUTOFF,... - LOW_RES_CUTOFF,... - ITERATIONS_PER_BIN,... - FIRST_ITER_SHIFT_LIMIT_PIXELS,... - DIVIDE_SHIFT_LIMIT_BY,... - baseName)); - + runPath, ... + baseName, ... + pixelSize, ... + imgRotation, ... + binHigh, ... + binLow, ... + binInc,... + nX,nY,nZ,ext,... + PATCH_SIZE_FACTOR,... + N_ITERS_NO_ROT,... + BORDER_SIZE_PIXELS,... + PATCH_OVERLAP,... + RESOLUTION_CUTOFF,... + LOW_RES_CUTOFF,... + ITERATIONS_PER_BIN,... + FIRST_ITER_SHIFT_LIMIT_PIXELS,... + DIVIDE_SHIFT_LIMIT_BY,... + baseName)); + cd(sprintf('%s',startDir)); % 2021-May-08 BAH, not needed b/c fixed/name.fixed should remain rotated by @@ -290,7 +290,7 @@ if strcmpi(TILT_OPTION,'0') % If not fitting tilt angles we need a copy of them with .tlt - system(sprintf('cp fixedStacks/%s.rawtlt fixedStacks/%s.tlt',baseName,baseName)); + system(sprintf('cp fixedStacks/%s.rawtlt fixedStacks/%s.tlt',baseName,baseName)); end to_few_beads = false; @@ -300,12 +300,12 @@ extList = {'tlt','xf','local'}; % stack is skipped in second round. leave as number 1 for iExt = 1:length(extList) system(sprintf('ln -sf ../fixedStacks/%s.%s %s.%s', ... - baseName,extList{iExt},baseName,extList{iExt})); - end - + baseName,extList{iExt},baseName,extList{iExt})); + end + system(sprintf('ln -sf ../fixedStacks/%s.%s.preprocessed %s.%s', ... - baseName,'fixed',baseName,'fixed')); - + baseName,'fixed',baseName,'fixed')); + % Stopping for now at a bin5, this should be dynamic along with a handful % of other options. @@ -317,15 +317,15 @@ else for iExt = 1:length(extList) system(sprintf('mv ../fixedStacks/%s.%s ../fixedStacks/%s.%s_patchTracking', ... - baseName,extList{iExt},baseName,extList{iExt})); + baseName,extList{iExt},baseName,extList{iExt})); system(sprintf('cp %s_%d.%s ../fixedStacks/%s.%s', ... - baseName,min_sampling_rate,extList{iExt},baseName,extList{iExt})); - end - - + baseName,min_sampling_rate,extList{iExt},baseName,extList{iExt})); + end + + system(sprintf('imodtrans -i ../fixedStacks/%s.fixed %s_%d_fit.fid ../fixedStacks/%s.erase',... - baseName,baseName,min_sampling_rate,baseName)); - + baseName,baseName,min_sampling_rate,baseName)); + system(sprintf('newstack -xf ../fixedStacks/%s.xf -bin 12 ../fixedStacks/%s.fixed ../fixedStacks/%s_bin12.ali',baseName,baseName,baseName)); end @@ -335,8 +335,8 @@ if (to_few_beads || ~REFINE_ON_BEADS) cd fixedStacks system(sprintf('%s %s %d %d %d %d', findBeadsPath, baseName,... - nX,nY,3000,... - ceil(1.05*100/pixelSize))); + nX,nY,3000,... + ceil(1.05*100/pixelSize))); cd .. end diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 65ef526d..da037146 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -1,8 +1,8 @@ function [] = BH_templateSearch3d_2( PARAMETER_FILE,... - tomoName,tomoNumber,TEMPLATE, ... - SYMMETRY, wedgeType, varargin) - - + tomoName,tomoNumber,TEMPLATE, ... + SYMMETRY, wedgeType, varargin) + + %3d template matching %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -41,7 +41,7 @@ % using emClarity ctf3d paramN.m templateSearch use_ctf3d_templateSearch=true; if ctf3dNoSubTomoMeta - mapBackIter = 0; + mapBackIter = 0; else try load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); @@ -52,7 +52,7 @@ shouldBeCTF = false; end end - samplingRate = emc.('Tmp_samplingRate'); +samplingRate = emc.('Tmp_samplingRate'); try tmpDecoy = emc.('templateDecoy') @@ -80,7 +80,7 @@ end - peakThreshold = emc.('Tmp_threshold'); +peakThreshold = emc.('Tmp_threshold'); latticeRadius = emc.('particleRadius'); @@ -94,7 +94,7 @@ convTMPNAME = sprintf('convmap_wedgeType_%d_bin%d',wedgeType,samplingRate) -try +try use_new_grid_search = emc.('use_new_grid_search'); catch use_new_grid_search = true; @@ -106,7 +106,7 @@ error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); end -try +try eraseMaskType = emc.('Peak_mType'); catch eraseMaskType = 'sphere'; @@ -146,41 +146,41 @@ % For testing print_warning=false; -try +try wantedCut = emc.('lowResCut'); fprintf('lowResCut is deprecated and will be removed in future versions.\n') fprintf('please switch to Tmp_bandpass\n\n'); bp_vals = [1e-3,600,wantedCut]; - print_warning = true; + print_warning = true; catch bp_vals = [1e-3,600,28]; end try - bp_vals = emc.('Tmp_bandpass'); - if numel(bp_vals) ~= 3 - error('Tmp_bandpass is [filter at zero freq, res high-pass cutoff, res low-pass cutoff]'); - end - if print_warning - fprintf('WARNING, you specified lowResCut (deprecated) and Tmp_bandpass!\n'); - end - fprintf('You specified a bandpass with values [%2.2e,%3.2f,%3.2f]\n',bp_vals); + bp_vals = emc.('Tmp_bandpass'); + if numel(bp_vals) ~= 3 + error('Tmp_bandpass is [filter at zero freq, res high-pass cutoff, res low-pass cutoff]'); + end + if print_warning + fprintf('WARNING, you specified lowResCut (deprecated) and Tmp_bandpass!\n'); + end + fprintf('You specified a bandpass with values [%2.2e,%3.2f,%3.2f]\n',bp_vals); catch - bp_vals = [1e-3,600,28]; - fprintf('Using default bandpass with values [%2.2e,%3.2f,%3.2f]\n',bp_vals); - + bp_vals = [1e-3,600,28]; + fprintf('Using default bandpass with values [%2.2e,%3.2f,%3.2f]\n',bp_vals); + end try - stats_diameter_fraction = emc.('diameter_fraction_for_local_stats') + stats_diameter_fraction = emc.('diameter_fraction_for_local_stats') catch - stats_diameter_fraction = 1 + stats_diameter_fraction = 1 end mean_r2 = 0; mean_r_mask = 0; reference_mask = []; -try +try scale_mip = emc.('scale_mip'); catch scale_mip = false; @@ -188,10 +188,10 @@ % Limit to the first zero if we are NOT using the CTF rec if ~( use_ctf3d_templateSearch ) -TLT = load(sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tomoName,mapBackIter+1)); + TLT = load(sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tomoName,mapBackIter+1)); def = mean(-1.*TLT(:,15))*10^6; %TODO if you switch to POSITIVEDEFOCUS this will be wrong firstZero = -0.2*def^2 +5.2*def +11; - % FIXME: if you have a ctf 3d tomo why limit to the firsts zero? + % FIXME: if you have a ctf 3d tomo why limit to the firsts zero? % Take the lower of firstZero lowResCut or Nyquist bp_vals(3) = max(bp_vals(3), firstZero); fprintf('\nUsing max (%f) of specified resolution cutoff of %f and first ctf zero %f Angstrom\n',bp_vals(3), wantedCut, firstZero); @@ -231,31 +231,31 @@ fprintf('\nlatticeRadius = %dx%dx%d pixels\n\n', latticeRadius); fprintf('\neraseMaskType %s, eraseMaskRadius %dx%dx%d pixels\n',eraseMaskType,eraseMaskRadius); - % For wedgeMask +% For wedgeMask particleThickness = latticeRadius(3); [ tomogram, reconGeometry ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... - reconCoords, mapBackIter, samplingRate,... - gpuIDX, reconScaling,1,'',super_sample); - + reconCoords, mapBackIter, samplingRate,... + gpuIDX, reconScaling,1,'',super_sample); + % We'll handle image statistics locally, but first place the global environment % into a predictible range - + [template, tempPath, tempName, tempExt] = ... - BH_multi_loadOrBin( TEMPLATE, 1, 3 ); - + BH_multi_loadOrBin( TEMPLATE, 1, 3 ); + % Bandpass the template so it is properly normalized bp_vals temp_bp = BH_bandpass3d(size(template),bp_vals(1),0.3.*bp_vals(2),bp_vals(3),'GPU',pixelSizeFULL); template = real(ifftn(fftn(gpuArray(template)).*temp_bp.^2)); clear temp_bp - + % The template will be padded later, trim for now to minimum so excess % iterations can be avoided. fprintf('size of provided template %d %d %d\n',size(template)); @@ -264,7 +264,7 @@ % SAVE_IMG(MRCImage(template),'template_trimmed.mrc'); clear trimTemp fprintf('size after trim to sqrt(2)*max(lattice radius) %d %d %d\n',size(template)); - + if isempty(mapPath) ; mapPath = '.' ; end if isempty(tempPath) ; tempPath = '.' ; end % Check to see if only tilt angles are supplied, implying a y-axis tilt scheme, @@ -319,32 +319,32 @@ rotConvention if (use_new_grid_search) - + gridSearch = eulerSearch(symmetry, angleSearch(1),... - angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, false); + angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, false); nAngles = sum(gridSearch.number_of_angles_at_each_theta); inPlaneSearch = gridSearch.parameter_map.psi; else - + [ nInPlane, inPlaneSearch, angleStep, nAngles] ... - = BH_multi_gridSearchAngles(angleSearch) + = BH_multi_gridSearchAngles(angleSearch) end - + highThr=sqrt(2).*erfcinv(ceil(peakThreshold.*0.10).*2./(prod(size(tomogram)).*nAngles(1))) [ OUTPUT ] = BH_multi_iterator( [targetSize; ... - size(tomogram);... - sizeTempBIN; ... - 2.*latticeRadius], 'convolution' ); + size(tomogram);... + sizeTempBIN; ... + 2.*latticeRadius], 'convolution' ); + - tomoPre = OUTPUT(1,:); tomoPost = OUTPUT(2,:); sizeChunk = OUTPUT(3,:); @@ -367,19 +367,19 @@ % the -1 searches for the next smallest fast fourier size templateBIN = gpuArray(templateBIN); - - + + decoyTest = BH_reScale3d(templateBIN,'',tmpDecoy,'GPU'); decoyTrim = BH_multi_padVal(size(decoyTest),size(templateBIN)); decoyTest = fftn(BH_padZeros3d(decoyTest,decoyTrim(1,:),decoyTrim(2,:),'GPU','single')); - decoyShift = -1.*gather(BH_multi_xcf_Translational(decoyTest,conj(fftn(templateBIN)),'',[3,3,3])); + decoyShift = -1.*gather(BH_multi_xcf_Translational(decoyTest,conj(fftn(templateBIN)),'',[3,3,3])); decoyNorm = gather(sum(abs(decoyTest(:)))./sum(abs(fftn(templateBIN(:))))); padDecoy = BH_multi_padVal(size(decoyTest),sizeChunk) + decoyTrim; clear decoyTest templateBIN = gather(templateBIN); fprintf('tmpDecoy %f normFactor %f and shift by %2.2f %2.2f %2.2f\n',tmpDecoy,decoyNorm,decoyShift); - + end @@ -411,7 +411,7 @@ % % % fftMask = BH_fftShift(validArea,sizeChunk,0); % Array for storing chunk results -RESULTS_peak = zeros(sizeTomo, 'single'); +RESULTS_peak = zeros(sizeTomo, 'single'); RESULTS_angle= zeros(sizeTomo, 'single'); if ( tmpDecoy ) RESULTS_decoy = RESULTS_peak; @@ -443,7 +443,7 @@ [ OUTPUT ] = BH_multi_iterator( [sizeChunk;kVal.*[1,1,1]], 'extrapolate' ); -% +% % switch wedgeType % case 1 % % make a binary wedge @@ -454,23 +454,23 @@ % % make a non-CTF wedge % [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... % 'applyMask',particleThickness,... -% 2, 1, samplingRate); +% 2, 1, samplingRate); % case 3 % % make a CTF without exposure weight % [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... % 'applyMask',particleThickness,... -% 3, 1, samplingRate); +% 3, 1, samplingRate); % case 4 % % make a wedge with full-ctf % [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... % 'applyMask',particleThickness,... -% 4, 1, samplingRate); +% 4, 1, samplingRate); % otherwise % error('wedgeType must be 1-4'); % end -% +% % wedgeMask = (ifftshift(wedgeMask)); -% +% % % Now just using the mask to calculate the power remaining in the template, % % without actually applying. % wedgeMask = gather(find(ifftshift(wedgeMask))); @@ -534,108 +534,108 @@ cutY = 1 + (iY-1).*validArea(2); for iZ = 1:nIters(3) cutZ = 1 + (iZ-1).*validArea(3); - - fprintf('preprocessing tomo_chunk %d/%d col %d/%d row %d/%d plane idx%d\n' , ... - iY,nIters(2),iX,nIters(1),iZ,nIters(3),tomoIDX) - - - - tomoChunk = gpuArray(tomogram(cutX:cutX+sizeChunk(1)-1,... - cutY:cutY+sizeChunk(2)-1,... - cutZ:cutZ+sizeChunk(3)-1)); - - % Make a list of the padded regions of the tomogram to exclude from - % statistical calculations - - tomoChunk = tomoChunk - mean(tomoChunk(:)); - tomoChunk = tomoChunk ./ rms(tomoChunk(:)); - -% tomoChunk = real(ifftn(fftn(tomoChunk).*tomoBandpass)); - - tomoChunk = bhF.invFFT(bhF.fwdFFT(tomoChunk,0,0,[bp_vals, pixelSize]),2); - - - if doMedFilt - if ( flgOOM ) - tomoChunk = (medfilt3(tomoChunk,doMedFilt.*[1,1,1])); + + fprintf('preprocessing tomo_chunk %d/%d col %d/%d row %d/%d plane idx%d\n' , ... + iY,nIters(2),iX,nIters(1),iZ,nIters(3),tomoIDX) + + + + tomoChunk = gpuArray(tomogram(cutX:cutX+sizeChunk(1)-1,... + cutY:cutY+sizeChunk(2)-1,... + cutZ:cutZ+sizeChunk(3)-1)); + + % Make a list of the padded regions of the tomogram to exclude from + % statistical calculations + + tomoChunk = tomoChunk - mean(tomoChunk(:)); + tomoChunk = tomoChunk ./ rms(tomoChunk(:)); + + % tomoChunk = real(ifftn(fftn(tomoChunk).*tomoBandpass)); + + tomoChunk = bhF.invFFT(bhF.fwdFFT(tomoChunk,0,0,[bp_vals, pixelSize]),2); + + + if doMedFilt + if ( flgOOM ) + tomoChunk = (medfilt3(tomoChunk,doMedFilt.*[1,1,1])); + else + tomoChunk = gpuArray(medfilt3(tomoChunk,doMedFilt.*[1,1,1])); + end else - tomoChunk = gpuArray(medfilt3(tomoChunk,doMedFilt.*[1,1,1])); - end - else - if ( flgOOM ) - % Leave on CPU - else - tomoChunk = gpuArray(tomoChunk); - statsRadius = gather(statsRadius); + if ( flgOOM ) + % Leave on CPU + else + tomoChunk = gpuArray(tomoChunk); + statsRadius = gather(statsRadius); + end end - end - - [ averageMask, flgOOM ] = BH_movingAverage_2(tomoChunk, statsRadius(1)); - rmsMask = BH_movingAverage_2(tomoChunk.^2, statsRadius(1)); - rmsMask = sqrt(rmsMask - averageMask.^2); - - % SAVE_IMG(BH_padZeros3d(real(single(... - % (rmsMask))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'),'rmsMask.mrc'); - % SAVE_IMG(templateMask,'templateMask.mrc'); - % tempPADMask = zeros(size(tomoChunk),'single','gpuArray'); - % tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... - % padBIN(1,2)+1: end - padBIN(2,2), ... - % padBIN(1,3)+1: end - padBIN(2,3)) = templateMask; - % tempPADMask = tempPADMask ./ sum(tempPADMask(:)); - % SAVE_IMG(tempPADMask,'tempPADMaskPre.mrc'); - % tempPADMask = (conj(bhF.fwdFFT(bhF.normalization_factor.^-1 .* tempPADMask )));% ./ (sum(tempPADMask > 0.01)./sum(tempPADMask(:))))); - % SAVE_IMG(bhF.invFFT(conj(tempPADMask)),'tempPADMask.mrc'); - % ms = real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor.^3 .*tomoChunk.^2.*validCalcMask),'fwd').*tempPADMask)); - % ma = real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor.^3 .*tomoChunk.*validCalcMask),'fwd').*tempPADMask)).^2; - % md = BH_padZeros3d(real(single(... - % (ms-ma))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'); - % SAVE_IMG(md,'md.mrc'); - % SAVE_IMG(sqrt(md), 'smd.mrc') - % SAVE_IMG(BH_padZeros3d(real(single(... - % ms)),... - % trimValid(1,:),trimValid(2,:),'GPU','single'),'ms.mrc'); - % SAVE_IMG(BH_padZeros3d(real(single(... - % (ma))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'),'ma.mrc'); - % error('asdf') - % correctedRMS = (ms - ma); - % correctedRMS = sqrt(BH_padZeros3d(real(single(... - % correctedRMS)),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single')); - % mean(correctedRMS(:)) - % 1/mean(correctedRMS(:)) - % SAVE_IMG(correctedRMS,'correctedRMS.mrc'); - % error('asdf') - if (test_local) - localStack(:,:,:,tomoIDX) = gather(rmsMask); - else - if ~(scale_mip) - tomoChunk = (tomoChunk - averageMask) ./ rmsMask; + + [ averageMask, flgOOM ] = BH_movingAverage_2(tomoChunk, statsRadius(1)); + rmsMask = BH_movingAverage_2(tomoChunk.^2, statsRadius(1)); + rmsMask = sqrt(rmsMask - averageMask.^2); + + % SAVE_IMG(BH_padZeros3d(real(single(... + % (rmsMask))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'),'rmsMask.mrc'); + % SAVE_IMG(templateMask,'templateMask.mrc'); + % tempPADMask = zeros(size(tomoChunk),'single','gpuArray'); + % tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... + % padBIN(1,2)+1: end - padBIN(2,2), ... + % padBIN(1,3)+1: end - padBIN(2,3)) = templateMask; + % tempPADMask = tempPADMask ./ sum(tempPADMask(:)); + % SAVE_IMG(tempPADMask,'tempPADMaskPre.mrc'); + % tempPADMask = (conj(bhF.fwdFFT(bhF.normalization_factor.^-1 .* tempPADMask )));% ./ (sum(tempPADMask > 0.01)./sum(tempPADMask(:))))); + % SAVE_IMG(bhF.invFFT(conj(tempPADMask)),'tempPADMask.mrc'); + % ms = real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor.^3 .*tomoChunk.^2.*validCalcMask),'fwd').*tempPADMask)); + % ma = real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor.^3 .*tomoChunk.*validCalcMask),'fwd').*tempPADMask)).^2; + % md = BH_padZeros3d(real(single(... + % (ms-ma))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'); + % SAVE_IMG(md,'md.mrc'); + % SAVE_IMG(sqrt(md), 'smd.mrc') + % SAVE_IMG(BH_padZeros3d(real(single(... + % ms)),... + % trimValid(1,:),trimValid(2,:),'GPU','single'),'ms.mrc'); + % SAVE_IMG(BH_padZeros3d(real(single(... + % (ma))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'),'ma.mrc'); + % error('asdf') + % correctedRMS = (ms - ma); + % correctedRMS = sqrt(BH_padZeros3d(real(single(... + % correctedRMS)),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single')); + % mean(correctedRMS(:)) + % 1/mean(correctedRMS(:)) + % SAVE_IMG(correctedRMS,'correctedRMS.mrc'); + % error('asdf') + if (test_local) + localStack(:,:,:,tomoIDX) = gather(rmsMask); + else + if ~(scale_mip) + tomoChunk = (tomoChunk - averageMask) ./ rmsMask; + end end - end - clear rmsMask averageMask - - - - tomoChunk = gather(tomoChunk .*validCalcMask); - - - - tmp_sum = sum(tomoChunk(validCalcMask > 0.1)); % REVERT - % tmp_sum = sum(tomoChunk(:)); - - fullX = fullX + gather(tmp_sum); - fullX2 = fullX2 + gather(tmp_sum.^2); - fullnX = fullnX + gather(prod(sizeChunk)); - - - tomoStack(:,:,:,tomoIDX) = tomoChunk; - - tomoCoords(tomoIDX,:) = [cutX,cutY,cutZ]; - tomoIDX = tomoIDX + 1; - + clear rmsMask averageMask + + + + tomoChunk = gather(tomoChunk .*validCalcMask); + + + + tmp_sum = sum(tomoChunk(validCalcMask > 0.1)); % REVERT + % tmp_sum = sum(tomoChunk(:)); + + fullX = fullX + gather(tmp_sum); + fullX2 = fullX2 + gather(tmp_sum.^2); + fullnX = fullnX + gather(prod(sizeChunk)); + + + tomoStack(:,:,:,tomoIDX) = tomoChunk; + + tomoCoords(tomoIDX,:) = [cutX,cutY,cutZ]; + tomoIDX = tomoIDX + 1; + end % end of loop over Z chunks end % end of loop over Y chunks @@ -657,7 +657,7 @@ kVal = 0; - + currentGlobalAngle = 1; ANGLE_LIST = zeros(nAngles(1),3, 'single'); @@ -665,19 +665,19 @@ totalTime = 0; firstLoopOverTomo = true; - % Center the spectrum by multiplication not swapping (this should just - % be in the fourierTransformer class if it isn't already) - % % swapPhase(obj, inputVol, direction) with fwd should do it - % [dU,dV,dW] = BH_multi_gridCoordinates(size(tomoStack(:,:,:,1)),... - % 'Cartesian','GPU', ... - % {'none'},1,1,0); - - % swapQuadrants = exp((-2i*pi).*(dU.*(floor(size(dU,1)/2)+1) + ... - % (dV.*(floor(size(dV,2)/2)+1) + ... - % (dW.*(floor(size(dW,3)/2)+1))))); - % clear dU dV dW - - % swapQuadrants = swapQuadrants(1:floor(size(swapQuadrants,1)/2)+1,:,:); +% Center the spectrum by multiplication not swapping (this should just +% be in the fourierTransformer class if it isn't already) +% % swapPhase(obj, inputVol, direction) with fwd should do it +% [dU,dV,dW] = BH_multi_gridCoordinates(size(tomoStack(:,:,:,1)),... +% 'Cartesian','GPU', ... +% {'none'},1,1,0); + +% swapQuadrants = exp((-2i*pi).*(dU.*(floor(size(dU,1)/2)+1) + ... +% (dV.*(floor(size(dV,2)/2)+1) + ... +% (dW.*(floor(size(dW,3)/2)+1))))); +% clear dU dV dW + +% swapQuadrants = swapQuadrants(1:floor(size(swapQuadrants,1)/2)+1,:,:); if (use_new_grid_search) theta_search = 1:gridSearch.number_of_out_of_plane_angles; @@ -687,25 +687,25 @@ for iAngle = theta_search - + if (use_new_grid_search) theta = gridSearch.parameter_map.theta(iAngle); numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); else - theta = angleStep(iAngle,1); + theta = angleStep(iAngle,1); phiStep = angleStep(iAngle,3); numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; end - - + + tempImg = gpuArray(templateBIN); %%%%% NEW switch to bin - + % interpolationNormFactor = sum(abs(tempImg(:)).^2); - - - + + + tomoIDX = 1; firstLoopOverAngle = true; @@ -718,9 +718,9 @@ templateMask_interpolator = ''; [templateMask_interpolator, ~] = interpolator(gpuArray(templateMask),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - - - + + + % Iterate over the tomogram pulling each chunk one at a time. for iTomo = 1:nTomograms @@ -728,47 +728,47 @@ iCut = tomoCoords(tomoIDX,:); % reset the angle count and value at the begining of loop % inside, while each new outer loop changes the start values. - -% nAngle = angleIncStart; + + % nAngle = angleIncStart; intraLoopAngle = 1; - + % Truth value to initialize temp results matrix each new tomo % chunk. firstLoopOverChunk = true; - + if (use_new_grid_search) fprintf('Working on tilt(%d/%d) tomoChunk(%d/%d)\t' ... - ,iAngle,gridSearch.number_of_out_of_plane_angles, tomoIDX,nTomograms); + ,iAngle,gridSearch.number_of_out_of_plane_angles, tomoIDX,nTomograms); else fprintf('working on tilt(%d/%d) tomoChunk(%d/%d)\t' ... - ,iAngle,size(angleStep,1), tomoIDX,nTomograms); + ,iAngle,size(angleStep,1), tomoIDX,nTomograms); end - - - tomoFou = gpuArray(tomoStack(:,:,:,tomoIDX)); - if test_local - localFou = BH_padZeros3d(localStack(:,:,:,tomoIDX),trimValid(1,:),trimValid(2,:),'GPU','single'); -% localStack(:,:,:,tomoIDX)); - end - % % profile on - if (scale_mip) - tomoFou_2 = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou.^2)), 'fwd'); - end - - - tomoFou = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)), 'fwd'); - - - % profile on - % if (scale_mip) - % tomoFou_2 = swapQuadrants.*bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou.^2)); - % end - - - % tomoFou = swapQuadrants.*bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)); - - - + + + tomoFou = gpuArray(tomoStack(:,:,:,tomoIDX)); + if test_local + localFou = BH_padZeros3d(localStack(:,:,:,tomoIDX),trimValid(1,:),trimValid(2,:),'GPU','single'); + % localStack(:,:,:,tomoIDX)); + end + % % profile on + if (scale_mip) + tomoFou_2 = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou.^2)), 'fwd'); + end + + + tomoFou = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)), 'fwd'); + + + % profile on + % if (scale_mip) + % tomoFou_2 = swapQuadrants.*bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou.^2)); + % end + + + % tomoFou = swapQuadrants.*bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)); + + + if (use_new_grid_search) phi_search = gridSearch.parameter_map.phi{iAngle}; else @@ -776,344 +776,344 @@ end for iAzimuth = phi_search - - + + if (use_new_grid_search) phi = iAzimuth; else - phi = phiStep * iAzimuth; + phi = phiStep * iAzimuth; end - + for iInPlane = inPlaneSearch - + psi = iInPlane; - + %calc references only on first chunk if (firstLoopOverAngle) - + ANGLE_LIST(currentGlobalAngle,:) = [phi, theta, psi - phi]; - + end - + [ tempRot ] = template_interpolator.interp3d(... - [phi, theta, psi - phi],... - [0,0,0],rotConvention,... - 'forward','C1'); - + [phi, theta, psi - phi],... + [0,0,0],rotConvention,... + 'forward','C1'); + + + + tempPAD = tempPAD .* 0; + tempPAD(padBIN(1,1)+1: end - padBIN(2,1), ... + padBIN(1,2)+1: end - padBIN(2,2), ... + padBIN(1,3)+1: end - padBIN(2,3)) = tempRot; + + tempPAD = tempPAD - mean(tempPAD(:)); + + if (scale_mip) + % I should probaly switch to using the SF3D masked reference, but that also changes the baseline implementation + % so I'll leave it for now. - - tempPAD = tempPAD .* 0; - tempPAD(padBIN(1,1)+1: end - padBIN(2,1), ... - padBIN(1,2)+1: end - padBIN(2,2), ... - padBIN(1,3)+1: end - padBIN(2,3)) = tempRot; - - tempPAD = tempPAD - mean(tempPAD(:)); - - if (scale_mip) - % I should probaly switch to using the SF3D masked reference, but that also changes the baseline implementation - % so I'll leave it for now. - - - - tempPADMask = tempPADMask .* 0; - - tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... - padBIN(1,2)+1: end - padBIN(2,2), ... - padBIN(1,3)+1: end - padBIN(2,3)) = templateMask_interpolator.interp3d(... - [phi, theta, psi - phi],... - [0,0,0],rotConvention,... - 'forward','C1'); - - tempPADMask = tempPADMask ./ sum(tempPADMask(:)); - tempMaskFou = (conj(bhF.fwdFFT(bhF.normalization_factor.^-1 .* tempPADMask )));% ./ (sum(tempPADMask > 0.01)./sum(tempPADMask(:))))); - md = BH_padZeros3d(... - real(bhF.invFFT(tomoFou_2.*tempMaskFou)) - real(bhF.invFFT(tomoFou.*tempMaskFou)).^2,... - trimValid(1,:),trimValid(2,:),'GPU','single'); - - mip_scaling = sqrt(md); - - + + + tempPADMask = tempPADMask .* 0; + + tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... + padBIN(1,2)+1: end - padBIN(2,2), ... + padBIN(1,3)+1: end - padBIN(2,3)) = templateMask_interpolator.interp3d(... + [phi, theta, psi - phi],... + [0,0,0],rotConvention,... + 'forward','C1'); + + tempPADMask = tempPADMask ./ sum(tempPADMask(:)); + tempMaskFou = (conj(bhF.fwdFFT(bhF.normalization_factor.^-1 .* tempPADMask )));% ./ (sum(tempPADMask > 0.01)./sum(tempPADMask(:))))); + md = BH_padZeros3d(... + real(bhF.invFFT(tomoFou_2.*tempMaskFou)) - real(bhF.invFFT(tomoFou.*tempMaskFou)).^2,... + trimValid(1,:),trimValid(2,:),'GPU','single'); + + mip_scaling = sqrt(md); + + + + end + + + tempFou = conj(bhF.fwdFFT(tempPAD)); + + + + ccfmap = BH_padZeros3d(real(single(... + bhF.invFFT(tomoFou.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + trimValid(1,:),trimValid(2,:),'GPU','single'); + % + + if ~(scale_mip) + ccfmap = ccfmap ./ std(ccfmap(:)); + end + + if ( tmpDecoy > 0 ) + + if (firstLoopOverAngle) + decoy = BH_padZeros3d(BH_reScale3d(tempRot./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... + padDecoy(1,:),padDecoy(2,:),'GPU','single'); + else + % Probably just make a second decoy stack to avoid + % re-interpolating. If it works, then do this. + error('This is temp broken with new interpolator'); + % decoy = BH_padZeros3d(BH_reScale3d(referenceStack(:,:,:,intraLoopAngle)./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... + % padDecoy(1,:),padDecoy(2,:),'GPU','single'); end - - - tempFou = conj(bhF.fwdFFT(tempPAD)); - - - - ccfmap = BH_padZeros3d(real(single(... - bhF.invFFT(tomoFou.*tempFou))),...%./(tomoNorm.*tempNorm))))),... - trimValid(1,:),trimValid(2,:),'GPU','single'); -% - - if ~(scale_mip) - ccfmap = ccfmap ./ std(ccfmap(:)); - end - - if ( tmpDecoy > 0 ) - - if (firstLoopOverAngle) - - decoy = BH_padZeros3d(BH_reScale3d(tempRot./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - padDecoy(1,:),padDecoy(2,:),'GPU','single'); - else - % Probably just make a second decoy stack to avoid - % re-interpolating. If it works, then do this. - error('This is temp broken with new interpolator'); - % decoy = BH_padZeros3d(BH_reScale3d(referenceStack(:,:,:,intraLoopAngle)./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - % padDecoy(1,:),padDecoy(2,:),'GPU','single'); - end - - - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*conj(fftn(decoy)))))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU','single'); - - - elseif ( tmpDecoy < 0 ) + + + decoy = BH_padZeros3d(fftshift(real(single( ... + ifftn(tomoFou.*conj(fftn(decoy)))))),..../(decoyNorm.*tomoNorm))))), + trimValid(1,:), ... + trimValid(2,:),'GPU','single'); + + + elseif ( tmpDecoy < 0 ) % Just use the mirror image of the template, i.e. take the conj % (of the conj) so just the padded FFT of the ref. - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*tempFou)))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU','single'); - - end - - if (scale_mip) - - % tempFou = conj(bhF.fwdFFT(tempPADMask.*bhF.normalization_factor^2)); - % tempFou = conj(bhF.fwdFFT(tempPADMask.*bhF.normalization_factor^0)); - - % mip_scaling = (1.0/mean_r_mask) .* ... - % BH_padZeros3d(real(single(... - % bhF.invFFT(tomoFou.*tempFou))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single').^2; - - % mip_scaling = BH_padZeros3d(real(single(... - % bhF.invFFT(tomoFou_2.*tempFou))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single') ... - % - ... - % mip_scaling; - - - - - % try - - % mip_scaling = sqrt(mean_r2) .*sqrt(mip_scaling); % FIXME add check on zero - % catch - - % lowval = mip_scaling < 0; - % numel(mip_scaling) - % sum(lowval,'all') - % mean(mip_scaling(lowval),'all') - % mean(mip_scaling(~lowval),'all') - % fprintf('\nmean_r_mask %3.3e mean_r2 %3.3e\n',mean_r_mask,mean_r2); - % fprintf('norm factor %f\n', bhF.normalization_factor); - % a = conj(bhF.fwdFFT(tempPAD.^2)); - % b = conj(bhF.fwdFFT(tempPADMask)); - % a(2:end) = 0; - % b(2:end) = 0; - % a = bhF.invFFT(a); - % b = bhF.invFFT(b); - % fprintf('mean_r_mask %3.3e mean_r2 %3.3e\n',1.0/b(1),a(1)); - % fprintf('mean_r_mask %3.3e mean_r2 %3.3e\n',1.0/b(5),a(5)); - - - % fprintf('iAngle %d idx %d iAzimuth %d iInPlane %d\n',iAngle,tomoIDX,iAzimuth,iInPlane); - % SAVE_IMG(tempPADMask,sprintf('tempPADMask_%d.mrc',tomoIDX)); - % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX)); - % SAVE_IMG( BH_padZeros3d(real(single(... - % bhF.invFFT(tomoFou))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('tomoFou_%d.mrc',tomoIDX)); - % SAVE_IMG( BH_padZeros3d((real(single(... - % bhF.invFFT(tomoFou_2)))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('tomoFou_2_%d.mrc',tomoIDX)); - % SAVE_IMG(BH_padZeros3d(real(single(... - % bhF.invFFT(tomoFou_2.*tempFou))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('mip_scaling_2_%d.mrc',tomoIDX)); - % SAVE_IMG(... - % BH_padZeros3d((real(single(... - % bhF.invFFT(tomoFou.*tempFou)))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single').^2, sprintf('mip_scaling_%d.mrc',tomoIDX)); - - % error('Faild on mip scaling caclulation'); - % end - - % mip_scaling(abs(mip_scaling) < 1e-6) = 1e-6; - - % print the min,max and mean of the mip_scaling - % fprintf('\nmip_scaling min %3.3e max %3.3e mean %3.3e\n',min(mip_scaling(:)),max(mip_scaling(:)),mean(mip_scaling(:))); - ccfmap = ccfmap ./ mip_scaling; - - % Now scale the CCF to be an SNR by using the global variance which should be mostly noise peaks. - ccfmap = ccfmap ./ std(ccfmap(:)); - + decoy = BH_padZeros3d(fftshift(real(single( ... + ifftn(tomoFou.*tempFou)))),..../(decoyNorm.*tomoNorm))))), + trimValid(1,:), ... + trimValid(2,:),'GPU','single'); + + end + + if (scale_mip) + + % tempFou = conj(bhF.fwdFFT(tempPADMask.*bhF.normalization_factor^2)); + % tempFou = conj(bhF.fwdFFT(tempPADMask.*bhF.normalization_factor^0)); + + % mip_scaling = (1.0/mean_r_mask) .* ... + % BH_padZeros3d(real(single(... + % bhF.invFFT(tomoFou.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single').^2; + + % mip_scaling = BH_padZeros3d(real(single(... + % bhF.invFFT(tomoFou_2.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single') ... + % - ... + % mip_scaling; + + + + + % try + + % mip_scaling = sqrt(mean_r2) .*sqrt(mip_scaling); % FIXME add check on zero + % catch + + % lowval = mip_scaling < 0; + % numel(mip_scaling) + % sum(lowval,'all') + % mean(mip_scaling(lowval),'all') + % mean(mip_scaling(~lowval),'all') + % fprintf('\nmean_r_mask %3.3e mean_r2 %3.3e\n',mean_r_mask,mean_r2); + % fprintf('norm factor %f\n', bhF.normalization_factor); + % a = conj(bhF.fwdFFT(tempPAD.^2)); + % b = conj(bhF.fwdFFT(tempPADMask)); + % a(2:end) = 0; + % b(2:end) = 0; + % a = bhF.invFFT(a); + % b = bhF.invFFT(b); + % fprintf('mean_r_mask %3.3e mean_r2 %3.3e\n',1.0/b(1),a(1)); + % fprintf('mean_r_mask %3.3e mean_r2 %3.3e\n',1.0/b(5),a(5)); + + + % fprintf('iAngle %d idx %d iAzimuth %d iInPlane %d\n',iAngle,tomoIDX,iAzimuth,iInPlane); + % SAVE_IMG(tempPADMask,sprintf('tempPADMask_%d.mrc',tomoIDX)); + % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX)); + % SAVE_IMG( BH_padZeros3d(real(single(... + % bhF.invFFT(tomoFou))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('tomoFou_%d.mrc',tomoIDX)); + % SAVE_IMG( BH_padZeros3d((real(single(... + % bhF.invFFT(tomoFou_2)))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('tomoFou_2_%d.mrc',tomoIDX)); + % SAVE_IMG(BH_padZeros3d(real(single(... + % bhF.invFFT(tomoFou_2.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('mip_scaling_2_%d.mrc',tomoIDX)); + % SAVE_IMG(... + % BH_padZeros3d((real(single(... + % bhF.invFFT(tomoFou.*tempFou)))),...%./(tomoNorm.*tempNorm))))),... + % trimValid(1,:),trimValid(2,:),'GPU','single').^2, sprintf('mip_scaling_%d.mrc',tomoIDX)); + + % error('Faild on mip scaling caclulation'); + % end + + % mip_scaling(abs(mip_scaling) < 1e-6) = 1e-6; + + % print the min,max and mean of the mip_scaling + % fprintf('\nmip_scaling min %3.3e max %3.3e mean %3.3e\n',min(mip_scaling(:)),max(mip_scaling(:)),mean(mip_scaling(:))); + ccfmap = ccfmap ./ mip_scaling; + + % Now scale the CCF to be an SNR by using the global variance which should be mostly noise peaks. + ccfmap = ccfmap ./ std(ccfmap(:)); + + end + + if test_local + % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX)); + ccfmap = ccfmap ./ localFou; + ccfmap = ccfmap ./ std(ccfmap(:)); + % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX+1)); + % error('asdf') + end + clear tempRot + + % If first loop over tomo, initialize the storage volumes, if + % first loop over the chunk but not over the tomo, pull storage + % chunks from storage volume. + if (firstLoopOverTomo && firstLoopOverChunk) + %store ccfmap as complex with phase = angle of reference + magTmp = ccfmap; + if ( tmpDecoy ) + decoyTmp = decoy; end - - if test_local - % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX)); - ccfmap = ccfmap ./ localFou; - ccfmap = ccfmap ./ std(ccfmap(:)); - % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX+1)); - % error('asdf') + angTmp = ones(size(magTmp), 'single','gpuArray'); + + firstLoopOverTomo = false; + firstLoopOverChunk = false; + + intraLoopAngle = intraLoopAngle + 1; + currentGlobalAngle = currentGlobalAngle + 1; + + elseif (firstLoopOverChunk) + % These double cuts are old, and don't really make sense. Make + % this more consistant with current operations when there is + % time. + magTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + if ( tmpDecoy ) + decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + decoyTmp = gpuArray(decoyTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3))); + decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); end - clear tempRot - - % If first loop over tomo, initialize the storage volumes, if - % first loop over the chunk but not over the tomo, pull storage - % chunks from storage volume. - if (firstLoopOverTomo && firstLoopOverChunk) - %store ccfmap as complex with phase = angle of reference - magTmp = ccfmap; - if ( tmpDecoy ) - decoyTmp = decoy; - end - angTmp = ones(size(magTmp), 'single','gpuArray'); - - firstLoopOverTomo = false; - firstLoopOverChunk = false; - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - - elseif (firstLoopOverChunk) - % These double cuts are old, and don't really make sense. Make - % this more consistant with current operations when there is - % time. - magTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - if ( tmpDecoy ) - decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyTmp = gpuArray(decoyTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - magTmp = gpuArray(magTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - - firstLoopOverChunk = false; - - replaceTmp = ( magTmp < ccfmap ); - - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - - - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp - - else - % update higher values of ccfmap with new reference if applicable. - replaceTmp = ( magTmp < ccfmap ); - - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - if ( tmpDecoy ) - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp + magTmp = gpuArray(magTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3))); + angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3))); + + firstLoopOverChunk = false; + + replaceTmp = ( magTmp < ccfmap ); + + magTmp(replaceTmp) = ccfmap(replaceTmp); + angTmp(replaceTmp) = currentGlobalAngle; + + + + intraLoopAngle = intraLoopAngle + 1; + currentGlobalAngle = currentGlobalAngle + 1; + clear replaceTmp + + else + % update higher values of ccfmap with new reference if applicable. + replaceTmp = ( magTmp < ccfmap ); + + magTmp(replaceTmp) = ccfmap(replaceTmp); + angTmp(replaceTmp) = currentGlobalAngle; + if ( tmpDecoy ) + decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); end - nComplete = nComplete + 1; + + + intraLoopAngle = intraLoopAngle + 1; + currentGlobalAngle = currentGlobalAngle + 1; + clear replaceTmp end + nComplete = nComplete + 1; end - % profile viewer - % return - % After searching all angles on this chunk, but out meaningful + end + % profile viewer + % return + % After searching all angles on this chunk, but out meaningful % portion for storage. - - % FIXME this double cutting and temporary allocation is ridiculous. - magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - - - magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); - angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); - - - RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; - - clear magStoreTmp - - RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; - clear angStoreTmp - if ( tmpDecoy ) - decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(decoyTmp); - RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = decoyStoreTmp; - - end + % FIXME this double cutting and temporary allocation is ridiculous. + magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); - tomoTime = toc; - totalTime = totalTime + toc; timeEstimate = totalTime * (nTomograms*nAngles(1)./(nComplete-1)); - fprintf('elapsed time = %f s est remain %f s\n', tomoTime, timeEstimate); - tomoIDX = tomoIDX + 1; - firstLoopOverAngle = false; - currentGlobalAngle = currentGlobalAngle - intraLoopAngle + 1; + magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); + angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); + + + RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; + + clear magStoreTmp + + RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; + clear angStoreTmp + + if ( tmpDecoy ) + decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + decoyStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(decoyTmp); + RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = decoyStoreTmp; + end + + + tomoTime = toc; + totalTime = totalTime + toc; timeEstimate = totalTime * (nTomograms*nAngles(1)./(nComplete-1)); + fprintf('elapsed time = %f s est remain %f s\n', tomoTime, timeEstimate); + tomoIDX = tomoIDX + 1; + firstLoopOverAngle = false; + currentGlobalAngle = currentGlobalAngle - intraLoopAngle + 1; + end currentGlobalAngle = currentGlobalAngle + intraLoopAngle - 1; - + end %save('angle_list.txt','angle_list','-ascii'); clear tomoStack % Cut out the post padding used to iterate over the tomogram RESULTS_peak = RESULTS_peak(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); -%RESULTS_peak(RESULTS_peak < 0) = 0; + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); +%RESULTS_peak(RESULTS_peak < 0) = 0; RESULTS_angle = RESULTS_angle(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); if ( tmpDecoy ) RESULTS_decoy = RESULTS_decoy(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); -% RESULTS_decoy = RESULTS_decoy ./ std(RESULTS_decoy(:)); - RESULTS_decoy(RESULTS_decoy < 1) = 1; + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); + % RESULTS_decoy = RESULTS_decoy ./ std(RESULTS_decoy(:)); + RESULTS_decoy(RESULTS_decoy < 1) = 1; end @@ -1159,8 +1159,8 @@ % out and padding back in.) Also pad by size of removal mask (subtract this from % coordinates) mag = mag(szK(1)+1:end - szK(1), ... - szK(2)+1:end - szK(2), ... - szK(3)+1:end - szK(3)); + szK(2)+1:end - szK(2), ... + szK(3)+1:end - szK(3)); mag = BH_padZeros3d(mag,szK+rmDim,szK+rmDim, 'cpu', 'single'); %dev.FreeMemory; %%%Ang = angle(RESULTS_peak); %clear Results @@ -1208,7 +1208,7 @@ if (nTries > 1000) error('Did not find an appropriate erase mask'); end - + end if ignore_threshold @@ -1217,29 +1217,29 @@ this_try = 0; while n <= 2.*peakThreshold && (this_try < max_tries) && MAX > highThr -this_try = this_try + 1; - -% -% Some indicies come back as an error, even when they seem like the -% should be fine. I'm not sure why, and I should think about this -% more, but for now, just set that one index to zero (instead of a -% whole box) and move on with life. It looks like the index that is -% kicking out the error is equal to -1*numberofreferences, which -% might be an issue because that corresonds to the positive upper -% limit of the reference index. Ignoring it still seems to be okay -% but it bothers me not to know. - - -[i,j,k] = ind2sub(sizeTomo,coord); -try - c = gather([i,j,k]); -catch - print('Ran into some trouble gathering the i,j,k. Breaking out\n'); - break -end - + this_try = this_try + 1; + + % + % Some indicies come back as an error, even when they seem like the + % should be fine. I'm not sure why, and I should think about this + % more, but for now, just set that one index to zero (instead of a + % whole box) and move on with life. It looks like the index that is + % kicking out the error is equal to -1*numberofreferences, which + % might be an issue because that corresonds to the positive upper + % limit of the reference index. Ignoring it still seems to be okay + % but it bothers me not to know. + + + [i,j,k] = ind2sub(sizeTomo,coord); + try + c = gather([i,j,k]); + catch + print('Ran into some trouble gathering the i,j,k. Breaking out\n'); + break + end + if Ang(gather(coord)) > 0 - + % box for removal and center of mass calc, use a larger box if multiple % peaks are being saved. bDist = 1+round(log(emc.nPeaks)); @@ -1249,45 +1249,45 @@ chJ = c(2) + bDist; clK = c(3) - bDist; chK = c(3) + bDist; - + magBox = mag(clI:chI,clJ:chJ,clK:chK); angBox = Ang(clI:chI,clJ:chJ,clK:chK); - + [cmX, cmY, cmZ] = ndgrid(-1*bDist:1*bDist, ... - -1*bDist:1*bDist, ... - -1*bDist:1*bDist ); - - cMass = [ sum(sum(sum(magBox.*cmX))) ; ... - sum(sum(sum(magBox.*cmY))) ; ... - sum(sum(sum(magBox.*cmZ))) ] ./ sum(magBox(:)); - - + -1*bDist:1*bDist, ... + -1*bDist:1*bDist ); + + cMass = [ sum(sum(sum(magBox.*cmX))) ; ... + sum(sum(sum(magBox.*cmY))) ; ... + sum(sum(sum(magBox.*cmZ))) ] ./ sum(magBox(:)); + + % Switching from centered to lower left coordinates and subtracting the - % padding + % padding cenP = c + cMass' - rmDim; - - -% % % % If the most frequent peak is unique use it; -% % % [peakM, ~, peakC] = mode(angBox(:)); -% % % if length(peakC) == 1 && peakM -% % % % Need to ensure the mode is none zero which is possible. -% % % peakMat(n,4:6) = ANGLE_LIST(peakM,:); -% % % topPeak = peakM; -% % % else - % Otherwise use the value at the max for the peak val; - peakMat(n,4:6) = ANGLE_LIST(Ang(coord),:); - topPeak = Ang(coord); -% % % end + + + % % % % If the most frequent peak is unique use it; + % % % [peakM, ~, peakC] = mode(angBox(:)); + % % % if length(peakC) == 1 && peakM + % % % % Need to ensure the mode is none zero which is possible. + % % % peakMat(n,4:6) = ANGLE_LIST(peakM,:); + % % % topPeak = peakM; + % % % else + % Otherwise use the value at the max for the peak val; + peakMat(n,4:6) = ANGLE_LIST(Ang(coord),:); + topPeak = Ang(coord); + % % % end peakMat(n,1:3) = gather(samplingRate.*cenP); peakMat(n,10) = gather(MAX); - + iSNR = 0; if emc.nPeaks > 1 possible_angles = gather(magBox); - possible_angles(angBox == topPeak) = 0; + possible_angles(angBox == topPeak) = 0; nRandom = 2; for iPeak = 2:emc.nPeaks @@ -1297,7 +1297,7 @@ [~, cAng] = max(possible_angles(:)); topPeak = angBox(cAng); iSNR = gather(mean( possible_angles(angBox == topPeak))); - possible_angles(angBox == topPeak) = 0; + possible_angles(angBox == topPeak) = 0; Ang(cAng) if topPeak <= 0 || Ang(cAng) <= 0 useRandom = true; @@ -1315,52 +1315,52 @@ % If we've used up all the possible peaks, just insert a random % Incrementally far from the original iAngles = [ randn(1) .* (nRandom.^2) + peakMat(n,1), ... - randn(1) .* (nRandom.^2) + peakMat(n,2), ... - randn(1) .* (nRandom.^2) + peakMat(n,3)]; + randn(1) .* (nRandom.^2) + peakMat(n,2), ... + randn(1) .* (nRandom.^2) + peakMat(n,3)]; if nRandom < 10 - nRandom = nRandom + 1; + nRandom = nRandom + 1; end end peakMat(n,[1:3]+10*(iPeak-1)) = gather(samplingRate.*cenP); - peakMat(n,[4:6]+10*(iPeak-1)) = iAngles; + peakMat(n,[4:6]+10*(iPeak-1)) = iAngles; peakMat(n,10+10*(iPeak-1)) = iSNR; -% % % oldPeaks = ( angBox == peakM | oldPeaks ); + % % % oldPeaks = ( angBox == peakM | oldPeaks ); end end - - - -% rmMask = BH_resample3d(removalMask,peakMat(n,4:6),[0,0,0],rotConvention ,'GPU','forward'); + + + + % rmMask = BH_resample3d(removalMask,peakMat(n,4:6),[0,0,0],rotConvention ,'GPU','forward'); rmMask = rmInt.interp3d(gather(peakMat(n,4:6)),[0,0,0],rotConvention,'forward','C1'); - + % Invert after resampling so that zeros introduced by not extrapolating % the corners are swapped to ones, i.e. not removed. -% rmMask = (1-rmMask); + % rmMask = (1-rmMask); mag(c(1)-rmDim:c(1)+rmDim,... - c(2)-rmDim:c(2)+rmDim,... - c(3)-rmDim:c(3)+rmDim) = ... - mag(c(1)-rmDim:c(1)+rmDim,... - c(2)-rmDim:c(2)+rmDim,... - c(3)-rmDim:c(3)+rmDim) .* (rmMask< maskCutOff); - -% % % peakMat(n,10) = (gather(MAX) - Tmean)./Tstd; % record stds above mean + c(2)-rmDim:c(2)+rmDim,... + c(3)-rmDim:c(3)+rmDim) = ... + mag(c(1)-rmDim:c(1)+rmDim,... + c(2)-rmDim:c(2)+rmDim,... + c(3)-rmDim:c(3)+rmDim) .* (rmMask< maskCutOff); + + % % % peakMat(n,10) = (gather(MAX) - Tmean)./Tstd; % record stds above mean n = n + 1; if ~mod(n,100) n end - + else - Ang(gather(coord)); - mag(coord) = 0; + Ang(gather(coord)); + mag(coord) = 0; end - - -[MAX, coord] = max(mag(:)); - + + + [MAX, coord] = max(mag(:)); + end peakMat = peakMat( ( peakMat(:,1)>0 ),:); @@ -1381,39 +1381,39 @@ nSym=1; for i = 1:length(peakMat(:,1)) if all(peakMat(i,1:3)) - + iSym = mod(nSym,symOps.nSymMats)+1; % Generate a uniform distribution over the in-plane % randomizations - - r = reshape(BH_defineMatrix(peakMat(i,4:6), rotConvention , 'inv') * symOps.symmetry_matrices{iSym},1,9); + + r = reshape(BH_defineMatrix(peakMat(i,4:6), rotConvention , 'inv') * symOps.symmetry_matrices{iSym},1,9); nSym = nSym + 1; fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... - peakMat(i,4:6),r,1); - + '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... + i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,1:3), ... + peakMat(i,4:6),r,1); + if emc.nPeaks > 1 for iPeak = 2:emc.nPeaks - + iSym = mod(nSym,symOps.nSymMats)+1; r = reshape(BH_defineMatrix(peakMat(i,[4:6]+10*(iPeak-1)), rotConvention , 'inv') * symOps.symmetry_matrices{iSym},1,9); nSym = nSym + 1; fprintf(fileID,['%1.2f %d %d %d %d %d %d %d %d %d %f %f %f %d %d %d ',... - '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... - i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,[1:3]+10*(iPeak-1)), ... - peakMat(i,[4:6]+10*(iPeak-1)),r,1); - + '%f %f %f %f %f %f %f %f %f %d '],peakMat(i,10),samplingRate,0, ... + i+nPreviousSubTomos,1,1,1,1,1,0,peakMat(i,[1:3]+10*(iPeak-1)), ... + peakMat(i,[4:6]+10*(iPeak-1)),r,1); + - end + end end - - fprintf(fileID,'\n'); - fprintf(fileID2,'%f %f %f\n',peakMat(i,1:3)./samplingRate); - - + + fprintf(fileID,'\n'); + fprintf(fileID2,'%f %f %f\n',peakMat(i,1:3)./samplingRate); + + n = n + 1; - end + end end %lastIndex = find(fieldOUT(:,4),1,'last'); @@ -1432,7 +1432,7 @@ fprintf('Total execution time : %f seconds\n', etime(clock, startTime)); - + end % end of templateSearch3d function diff --git a/coordinates/BH_decomposeIMODxf.m b/coordinates/BH_decomposeIMODxf.m index 5fe5e65c..2ac687bd 100755 --- a/coordinates/BH_decomposeIMODxf.m +++ b/coordinates/BH_decomposeIMODxf.m @@ -7,21 +7,21 @@ a21 = IMOD_XF(3); a22 = IMOD_XF(4); -% * Converts a 2 by 2 transformation matrix into four "natural" parameters of +% * Converts a 2 by 2 transformation matrix into four "natural" parameters of % * image transformation. The transformation is specified by [a11], [a12], % * [a21], and [a22], where % * ^ x' = a11 * x + a12 * y % * ^ y' = a21 * x + a22 * y ^ -% * In the converted transformation, [theta] is overall rotation, [smag] is +% * In the converted transformation, [theta] is overall rotation, [smag] is % * overall magnification, [str] is a unidirectional stretch, and [phi] is the % * angle of the stretch axis. Two equivalent solutions are possible, with the -% * stretch axis in the first or fourth quadrant. The function returns the +% * stretch axis in the first or fourth quadrant. The function returns the % * solution that makes the magnification [smag] nearer to 1.0. - + % Just calc in degrees directly % ator = 0.0174532925; -% -% /* +% +% /* % To solve for the variables, the first step is to solve for THETA by % taking the arctangent of a function of the AMAT values. It is then % possible to compute F1, F2 and F3, intermediate factors whose @@ -34,118 +34,118 @@ % one of two different formulas, depending on whether PHI is near 45 % degrees or not, then SMAG is computed. % */ - + % /* first determine if there is an axis inversion: find angle from % transformed X axis to transformed Y axis and reduce to -180 to 180 % If difference is negative then invert Y components of matrix */ - - dtheta = atan2d(a22, a12) - atan2d(a21, a11); - - if (dtheta > 180.); dtheta = dtheta - 360; end - if (dtheta <= -180.); dtheta = dtheta + 360; end - if (dtheta < 0.) - a12 = -a12; - a22 = -a22; - end + +dtheta = atan2d(a22, a12) - atan2d(a21, a11); + +if (dtheta > 180.); dtheta = dtheta - 360; end +if (dtheta <= -180.); dtheta = dtheta + 360; end +if (dtheta < 0.) + a12 = -a12; + a22 = -a22; +end % /* next find the rotation angle theta that gives the same solution for % f2 when derived from a11 and a21 as when derived from a12 and a22 */ - theta = 0; - if (a21 ~= a12 || a22 ~= -1*a11) - theta = atan2d((a21-a12), (a22+a11)); - end - costh = cosd(theta); - sinth = sind(theta); - - f1 = a11*costh+a21*sinth; - f2 = a21*costh-a11*sinth; - f3 = a22*costh-a12*sinth; - +theta = 0; +if (a21 ~= a12 || a22 ~= -1*a11) + theta = atan2d((a21-a12), (a22+a11)); +end +costh = cosd(theta); +sinth = sind(theta); + +f1 = a11*costh+a21*sinth; +f2 = a21*costh-a11*sinth; +f3 = a22*costh-a12*sinth; + % /* Next solve for phi */ - - if (f2 < 1.e-10 && f2 > -1.e-10) - -% /* if f2 = 0, pick phi = 0., set cos phi to 1. */ - cosphisq = 1.; - else - -% /* otherwise, solve quadratic equation, pick the solution that is -% right for the first quadrant */ - afac = (f3-f1)*(f3-f1); - bfac = 4.*f2*f2; - cosphisq = 0.5*(1.+sqrt(1.-bfac/(bfac+afac))); - sinphisq = 1.-cosphisq; - fnum = f1*cosphisq-f3*sinphisq; - if (fnum < 0.) - fnum = -1*fnum; - end - fden = f3*cosphisq-f1*sinphisq; - if (fden < 0.) - fden = -fden; - end - if ((f2 > 0. && fnum < fden) || (f2 < 0. && fnum > fden)) - cosphisq = 1.-cosphisq; - end - end - phi = acosd(sqrt(cosphisq)); + +if (f2 < 1.e-10 && f2 > -1.e-10) + + % /* if f2 = 0, pick phi = 0., set cos phi to 1. */ + cosphisq = 1.; +else + + % /* otherwise, solve quadratic equation, pick the solution that is + % right for the first quadrant */ + afac = (f3-f1)*(f3-f1); + bfac = 4.*f2*f2; + cosphisq = 0.5*(1.+sqrt(1.-bfac/(bfac+afac))); sinphisq = 1.-cosphisq; - -% /* solve for str. */ - - if (cosphisq-0.5 > 0.25 || cosphisq - 0.5 < - 0.25) + fnum = f1*cosphisq-f3*sinphisq; + if (fnum < 0.) + fnum = -1*fnum; + end + fden = f3*cosphisq-f1*sinphisq; + if (fden < 0.) + fden = -fden; + end + if ((f2 > 0. && fnum < fden) || (f2 < 0. && fnum > fden)) + cosphisq = 1.-cosphisq; + end +end +phi = acosd(sqrt(cosphisq)); +sinphisq = 1.-cosphisq; -% /* for angles far from 45 deg, use an equation that is good at 0 -% or 90 deg but blows up at 45 deg. */ - str = (f1*cosphisq-f3*sinphisq)/(f3*cosphisq-f1*sinphisq); +% /* solve for str. */ - else +if (cosphisq-0.5 > 0.25 || cosphisq - 0.5 < - 0.25) + + % /* for angles far from 45 deg, use an equation that is good at 0 + % or 90 deg but blows up at 45 deg. */ + str = (f1*cosphisq-f3*sinphisq)/(f3*cosphisq-f1*sinphisq); + +else + + % /* for angles near 45 deg, use an equation that is good there but + % blows up at 0. */ + factmp = (f1+f3)*sqrt(cosphisq*sinphisq); + str = (factmp+f2)/(factmp-f2); +end -% /* for angles near 45 deg, use an equation that is good there but -% blows up at 0. */ - factmp = (f1+f3)*sqrt(cosphisq*sinphisq); - str = (factmp+f2)/(factmp-f2); - end - % /* solve for smag from the equation for f1, or f2 if that would fail % (which it does with stretch -1 along 45 degree line) */ - - dentmp = str * cosphisq + sinphisq; - if(dentmp > 1.e-5 || dentmp < -1.e-5) - smag = f1/dentmp; - else - smag = 1./((str-1.)*sqrt(cosphisq*sinphisq)); - end - + +dentmp = str * cosphisq + sinphisq; +if(dentmp > 1.e-5 || dentmp < -1.e-5) + smag = f1/dentmp; +else + smag = 1./((str-1.)*sqrt(cosphisq*sinphisq)); +end + % /* if it will make smag closer to 1.0, flip stretch axis 90 deg */ - - f1 = smag - 1; - f2 = str * smag - 1; - if (f1 < 0.) - f1 = -f1; - end - if (f2 < 0.) - f2 = -f2; - end - if(f1 > f2) - smag = smag * str; - str = 1 / str; - phi = phi-90; - end - +f1 = smag - 1; +f2 = str * smag - 1; +if (f1 < 0.) + f1 = -f1; +end +if (f2 < 0.) + f2 = -f2; +end +if(f1 > f2) + smag = smag * str; + str = 1 / str; + phi = phi-90; +end + + % /* Now if there is an inversion, then invert the stretch, mirror the % stretch axis, and add a rotation to bring inverted point along stretch % axis to a point mirrored around X */ -% - if (dtheta < 0) - str = -1*str; - phi = -1*phi; - theta = theta + 180 - 2 * phi; - if (theta > 180) - theta = theta - 180; - end +% +if (dtheta < 0) + str = -1*str; + phi = -1*phi; + theta = theta + 180 - 2 * phi; + if (theta > 180) + theta = theta - 180; end +end diff --git a/coordinates/BH_defineMatrix.m b/coordinates/BH_defineMatrix.m index 6ab1926b..eb1af360 100755 --- a/coordinates/BH_defineMatrix.m +++ b/coordinates/BH_defineMatrix.m @@ -16,7 +16,7 @@ % % DIRECTION = forward : rotation from microscope frame to particle frame. % inverse : rotation from particle frame to microscope frame. -% +% % Output variables: % % ROTATION_MATRIX = 3d rotation matrix @@ -31,10 +31,10 @@ % % These are general, but in the scope of the BH_subTomo programs, they are % generally applied to an ndgrid which is transformed and used as the query to -% an interpolation. +% an interpolation. % % Regardless of how they are used, the angles are interpreted to reflect an -% active, intrinsic transformation on a particle, and the convention and +% active, intrinsic transformation on a particle, and the convention and % direction are taken into account in order for this to work. % % A good test is to create wedge masks of varying orientation because these @@ -57,8 +57,8 @@ % Normalize to unit sphere; randXYZ = randXYZ ./ sqrt(sum(randXYZ.^2,2)); angles = [atan2(randXYZ(2),randXYZ(1)), ... - acos(randXYZ(3)), ... - (2.*pi.*(rand(1) - 0.5))]; % between -pi/pi + acos(randXYZ(3)), ... + (2.*pi.*(rand(1) - 0.5))]; % between -pi/pi % Make sure to override conventions for consistency. CONVENTION = 'Bah'; @@ -72,113 +72,113 @@ % Rx = @(t)[ 1 0 0 ;... % 0 cos(t) -sin(t);... % 0 sin(t) cos(t) ]; -% +% % Ry = @(t)[ cos(t) 0 sin(t);... % 0 1 0;... % -sin(t) 0 cos(t) ]; -% +% % Rz = @(t)[ cos(t) -sin(t) 0;... % sin(t) cos(t) 0;... % 0 0 1 ]; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - + if strcmpi(DIRECTION, 'forward') || strcmpi(DIRECTION, 'fwd') || strcmpi(DIRECTION, 'invVector') - angles = -1.*angles; + angles = -1.*angles; elseif strcmpi(DIRECTION, 'inverse') || strcmpi(DIRECTION, 'inv') || strcmpi(DIRECTION, 'forwardVector') % For interpolation the vectors are applied to a grid, so the sense must - % be inverted to make the final transformation active. - - - % In order to rotate the particle from a position defined by the input - % angles, back to the proper reference frame, the sense is already - % inverted, and just the order must be inverted. - % - % Think of this as taking an average in the proper frame, applying a given - % rotation with 'forward', then this undoes that action. - % - % IMPORTANT NOTE: because the order is flipped, successive rotations by - % multiple matrices must be right multplied for inverse operations. eg: - % R1(e1,e2,e3) & R2(e4,e5,e6) then Rtot = R1 * R2 = e1*e2*e3*e4*e5*e6*Mat - angles = flip(angles); -% angles = [angles(3), angles(2), angles(1)]; - else + % be inverted to make the final transformation active. + + + % In order to rotate the particle from a position defined by the input + % angles, back to the proper reference frame, the sense is already + % inverted, and just the order must be inverted. + % + % Think of this as taking an average in the proper frame, applying a given + % rotation with 'forward', then this undoes that action. + % + % IMPORTANT NOTE: because the order is flipped, successive rotations by + % multiple matrices must be right multplied for inverse operations. eg: + % R1(e1,e2,e3) & R2(e4,e5,e6) then Rtot = R1 * R2 = e1*e2*e3*e4*e5*e6*Mat + angles = flip(angles); + % angles = [angles(3), angles(2), angles(1)]; +else error('Direction must be forward or inv, not %s', DIRECTION) end % Reduce number of trig functions cosA = cos(angles); sinA = sin(angles); - - + + switch CONVENTION case 'Bah' - -% ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Rz(angles(1)); + + % ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Rz(angles(1)); ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [1,0,0; ... - 0,cosA(2),-sinA(2);... - 0,sinA(2),cosA(2)] * ... - [cosA(1),-sinA(1),0;... - sinA(1),cosA(1),0;... - 0,0,1] ; - + sinA(3),cosA(3),0;... + 0,0,1] * ... + [1,0,0; ... + 0,cosA(2),-sinA(2);... + 0,sinA(2),cosA(2)] * ... + [cosA(1),-sinA(1),0;... + sinA(1),cosA(1),0;... + 0,0,1] ; + case 'TILT' - + ROTATION_MATRIX = [cosA,0,sinA; ... - 0,1,0;... - -sinA,0,cosA]; - - + 0,1,0;... + -sinA,0,cosA]; + + case 'SPIDER' -% ROTATION_MATRIX = Rz(angles(3)) * Ry(angles(2)) * Rz(angles(1)); - + % ROTATION_MATRIX = Rz(angles(3)) * Ry(angles(2)) * Rz(angles(1)); + ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [cosA(2),0,sinA(2); ... - 0,1,0;... - -sinA(2),0,cosA(2)] * ... - [cosA(1),-sinA(1),0;... - sinA(1),cosA(1),0;... - 0,0,1] ; - + sinA(3),cosA(3),0;... + 0,0,1] * ... + [cosA(2),0,sinA(2); ... + 0,1,0;... + -sinA(2),0,cosA(2)] * ... + [cosA(1),-sinA(1),0;... + sinA(1),cosA(1),0;... + 0,0,1] ; + case 'Helical' - -% ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Ry(angles(1)); - + + % ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Ry(angles(1)); + ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [1,0,0; ... - 0,cosA(2),-sinA(2);... - 0,sinA(2),cosA(2)] * ... - [cosA(1),0,sinA(1); ... - 0,1,0;... - -sinA(1),0,cosA(1)]; - + sinA(3),cosA(3),0;... + 0,0,1] * ... + [1,0,0; ... + 0,cosA(2),-sinA(2);... + 0,sinA(2),cosA(2)] * ... + [cosA(1),0,sinA(1); ... + 0,1,0;... + -sinA(1),0,cosA(1)]; + case 'IMOD' - + cosA = cos(angles); sinA = sin(angles); -% ROTATION_MATRIX = Rz(angles(3)) * Ry(angles(2)) * Rx(angles(1)); - + % ROTATION_MATRIX = Rz(angles(3)) * Ry(angles(2)) * Rx(angles(1)); + ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [cosA(2),0,sinA(2); ... - 0,1,0;... - -sinA(2),0,cosA(2)] * ... - [cosA(1),-sinA(1),0;... - sinA(1),cosA(1),0;... - 0,0,1] ; - + sinA(3),cosA(3),0;... + 0,0,1] * ... + [cosA(2),0,sinA(2); ... + 0,1,0;... + -sinA(2),0,cosA(2)] * ... + [cosA(1),-sinA(1),0;... + sinA(1),cosA(1),0;... + 0,0,1] ; + otherwise error('Convention must be Bah,SPI,Helical, not %s', CONVENTION) end @@ -188,55 +188,55 @@ % case 'Protomo' % % passive, intrinsic, Z X Z % % i3euler e1 e2 e3 -% +% % if strcmpi(dir,'forward') -% +% % elseif strcmpi(dir, 'inv') % ang = -1.* [ang(3), ang(2), ang(1)]; -% -% +% +% % elseif strcmpi(dir, 'i3') % ang = [ang(3), ang(2), ang(1)]; % end -% +% % RotMat = Rz(ang(3)) * Rx(ang(2)) * Rz(ang(1)); -% -% +% +% % case 'Imod' % % active, extrinsic, Z Y X -% +% % if strcmpi(dir, 'forward') % ang = -1 .* ang ; % end -% +% % RotMat = Rx(ang(3))*Ry(ang(2))*Rz(ang(1)) ; -% +% % case 'Spider' % % passive, extrinsic Z Y Z % % Note that in their documents they refer to the "object" % % rotating clockwise, which sounds active, but this is the -% % same as the CS anti-clockwise, which is just a passive +% % same as the CS anti-clockwise, which is just a passive % % (alias) rotation. I believe Frealign, and Relion also use. -% +% % % Spider puts the origin at top left, with first z on top -% +% % RotMat = Rz(-e3)*Ry(-e2)*Rz(-e1) ; -% +% % case '2d' % % active rotation, second two euler angles are dummy var -% +% % RotMat = Rz(e1); % RotMat = RotMat(1:2,1:2) ; -% +% % case 'NegProtomo' % % passive, intrinsic, Z X Z % % i3euler e1 e2 e3 -% -% +% +% % RotMat = Rz(-ang(1)) * Rx(-ang(2)) * Rz(-ang(3)) ; -% +% % end - - + + diff --git a/coordinates/BH_multi_angularSearch.m b/coordinates/BH_multi_angularSearch.m index fdd6f6d0..f323e32b 100755 --- a/coordinates/BH_multi_angularSearch.m +++ b/coordinates/BH_multi_angularSearch.m @@ -1,13 +1,13 @@ function [ CCC_STORAGE] = BH_multi_angularSearch( ANGLE_STEP, ... - PEAK_LIST, ... - IN_PLANE_SEARCH, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padREF,... - peakMask, peakCOM, IDX, ... - refSym) + PEAK_LIST, ... + IN_PLANE_SEARCH, ... + iClassImg, iClassWdg, ... + ref_FT, refWDG, ... + refRotAvg_FT, ... + volMask, bandpassFilt, ... + padCalc, padREF,... + peakMask, peakCOM, IDX, ... + refSym) %UNTITLED Summary of this function goes her % Detailed explanation goes here @@ -31,8 +31,8 @@ iClassTrim = iClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); + padREF(1,2)+1 : end - padREF(2,2), ... + padREF(1,3)+1 : end - padREF(2,3) ); %[ iClassImg ] = BH_bandLimitCenterNormalize(unMaskedClassImage, bandpassFilt, volMask); @@ -45,7 +45,7 @@ % Either a refinment around top three or top one peaks, or checking all % inplane angles for top ten axially averaged. - if (size(peakList,2) == 10) + if (size(peakList,2) == 10) % Refinementd % Get unique references for peakList wedge weight normalization referenceList = unique(peakList(:,1)); @@ -58,21 +58,21 @@ [ iClassImg2{iRef} ] = BH_bandLimitCenterNormalize(iClassTrim.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); end angCount=1; - + nAngles = 46.*size(peakList,1)+1 - pause(4) + pause(4) cccStorage = zeros(nAngles, 13, 'double', 'gpuArray'); for iAngle = 1 : size(peakList,1) - + iRef = peakList(iAngle, 1); phi = peakList(iAngle, 2); phiInc = peakList(iAngle,5); theta= peakList(iAngle, 3); thetaInc = peakList(iAngle,6); psi = peakList(iAngle, 4); psiInc = peakList(iAngle,7); - + if size(peakList,1) == 1 % This is the final refinement -% superSample = 1; + % superSample = 1; inPlaneSearch = psi-psiInc : psiInc : psi+psiInc; polarSearch = theta-thetaInc :thetaInc : theta+thetaInc; azimuthalSearch= phi-phiInc : phiInc : phi + phiInc; @@ -82,111 +82,111 @@ azimuthalSearch= phi-2*phiInc : phiInc : phi + 2*phiInc; end - - - + + + searchList = zeros(46,3); - nSearch = 1; + nSearch = 1; for iPhi = azimuthalSearch - for iTheta = polarSearch + for iTheta = polarSearch for iPsi = inPlaneSearch searchList(nSearch, :) = [iPhi, iTheta, iPsi]; nSearch = nSearch + 1; end end end - - - + + + for iRefine = 1:nSearch-1 - - + + RotMat = BH_defineMatrix(searchList(iRefine,:),'Bah', 'forward'); - - - [ rotRef ] = BH_resample3d(ref_FT(:,:,:,iRef),RotMat, ... - peakList(iAngle,8:10), ... - 'Bah', 'GPU', 'forward'); - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - peakList(iAngle,8:10), ... - 'Bah', 'GPU', 'forward')); + + + [ rotRef ] = BH_resample3d(ref_FT(:,:,:,iRef),RotMat, ... + peakList(iAngle,8:10), ... + 'Bah', 'GPU', 'forward'); + if isa(refWDG,'cell') + rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... + peakList(iAngle,8:10), ... + 'Bah', 'GPU', 'forward')); else rotWDG = refWDG; end - + rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - - + padREF(1,2)+1 : end - padREF(2,2), ... + padREF(1,3)+1 : end - padREF(2,3) ); + + + + rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef}, volBinary,padCalc,'single'); + BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef}, volBinary,padCalc,'single'); rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - + BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); + rotRef_FT = conj(rotRef_FT); rotRef_FT2= conj(rotRef_FT2); - - - % find translational shift using rotationally averaged tightly masked - % reference, to reduce chance of drift to alternate lattice sites. + + + % find translational shift using rotationally averaged tightly masked + % reference, to reduce chance of drift to alternate lattice sites. try - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); + [ estPeakCoord ] = BH_multi_xcf_Translational( ... + iClassImg2{iRef}, rotRef_FT2, ... + peakMask, peakCOM); catch iRef - - - error('sdfsd') + + + error('sdfsd') end - + % apply only a tranlational shift to the particle [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - - rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - + estPeakCoord,'Bah','GPU','inv'); + + rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... + padREF(1,2)+1 : end - padREF(2,2), ... + padREF(1,3)+1 : end - padREF(2,3) ); + + [ rotClassImg1 ] = BH_bandLimitCenterNormalize(... - rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,... - padCalc,'single'); -% [ rotClassImg2 ] = BH_bandLimitCenterNormalize(... -% rotClassImg.*peakMask, ... -% bandpassFilt, peakBinary, ... -% padCalc,'single'); + rotClassImg.*volMask, ... + bandpassFilt{iRef} , volBinary,... + padCalc,'single'); + % [ rotClassImg2 ] = BH_bandLimitCenterNormalize(... + % rotClassImg.*peakMask, ... + % bandpassFilt, peakBinary, ... + % padCalc,'single'); clear rotClassImg - + % now calc CCC, setting sampling shift to zero [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg, rotWDG, ... - peakMask); - -% [ finalPeakCoord ] = BH_multi_xcf_Translational( ... -% rotClassImg2, rotRef_FT2, ... -% peakMask, peakCOM); + BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... + iClassWdg, rotWDG, ... + peakMask); + + % [ finalPeakCoord ] = BH_multi_xcf_Translational( ... + % rotClassImg2, rotRef_FT2, ... + % peakMask, peakCOM); cccStorage(angCount,:) = [iRef, IDX, searchList(iRefine,:), iCCC, ... - iWeight, peakList(iAngle,8:10)+estPeakCoord, ... - phiInc,thetaInc,psiInc]; - + iWeight, peakList(iAngle,8:10)+estPeakCoord, ... + phiInc,thetaInc,psiInc]; + angCount = angCount + 1; end % end inPlane end % end search over best peaks - + else % top ten % Get unique references for peakList wedge weight normalization - -% referenceList = unique(peakList(:,1)); -% nRefs = length(referenceList); + + % referenceList = unique(peakList(:,1)); + % nRefs = length(referenceList); angCount=1; nRefs = size(ref_FT,4); @@ -203,92 +203,92 @@ %iRef = peakList(iAngle, 1); phi = peakList(iAngle, 2); theta= peakList(iAngle, 3); - - for iInPlane = IN_PLANE_SEARCH + + for iInPlane = IN_PLANE_SEARCH psi = iInPlane; - evaluateRef = ones(1,nRefs); - if (limitSymmetry) - evaluateRef = evaluateRef.*((abs(psi).*evaluateRef) < 180 ./ refSym); - end - for iRef = 1:nRefs - if (evaluateRef(iRef)) - RotMat = BH_defineMatrix([phi, theta, psi - phi],'Bah', 'forward'); - - [ rotRef ] = BH_resample3d(ref_FT(:,:,:,iRef), ... - RotMat,peakList(iAngle,4:6), ... - 'Bah', 'GPU', 'forward'); - - - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - peakList(iAngle,4:6), ... - 'Bah', 'GPU', 'forward')); - else - rotWDG = refWDG; - end - - rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - - - rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef} , volBinary,padCalc,'single'); - rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef} , peakBinary,padCalc,'single'); - - rotRef_FT = conj(rotRef_FT); - rotRef_FT2= conj(rotRef_FT2); - - - - - - % find translational shift - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); - - % apply only a tranlational shift to the particle - [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - - rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - [ rotClassImg1 ] = BH_bandLimitCenterNormalize(rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,padCalc,'single'); - % [ rotClassImg2 ] = BH_bandLimitCenterNormalize(rotClassImg.*peakMask, ... - % bandpassFilt{iRef} , peakBinary,padCalc,'single'); - clear rotClassImg - - - % now calc CCC, setting sampling shift to zero - [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg, rotWDG,... - peakMask); - - % [ finalPeakCoord ] = BH_multi_xcf_Translational( ... - % rotClassImg2, rotRef_FT2, ... - % peakMask, peakCOM); - - - cccStorage(angCount,:) = [iRef, IDX, phi, theta, psi - phi, iCCC, ... - iWeight, estPeakCoord + ... - peakList(iAngle,4:6) ]; - angCount = angCount + 1; + evaluateRef = ones(1,nRefs); + if (limitSymmetry) + evaluateRef = evaluateRef.*((abs(psi).*evaluateRef) < 180 ./ refSym); + end + for iRef = 1:nRefs + if (evaluateRef(iRef)) + RotMat = BH_defineMatrix([phi, theta, psi - phi],'Bah', 'forward'); + + [ rotRef ] = BH_resample3d(ref_FT(:,:,:,iRef), ... + RotMat,peakList(iAngle,4:6), ... + 'Bah', 'GPU', 'forward'); + + + if isa(refWDG,'cell') + rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... + peakList(iAngle,4:6), ... + 'Bah', 'GPU', 'forward')); + else + rotWDG = refWDG; end + + rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... + padREF(1,2)+1 : end - padREF(2,2), ... + padREF(1,3)+1 : end - padREF(2,3) ); + + + + + rotRef_FT = ... + BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef} , volBinary,padCalc,'single'); + rotRef_FT2 = ... + BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef} , peakBinary,padCalc,'single'); + + rotRef_FT = conj(rotRef_FT); + rotRef_FT2= conj(rotRef_FT2); + + + + + + % find translational shift + [ estPeakCoord ] = BH_multi_xcf_Translational( ... + iClassImg2{iRef}, rotRef_FT2, ... + peakMask, peakCOM); + + % apply only a tranlational shift to the particle + [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... + estPeakCoord,'Bah','GPU','inv'); + + rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... + padREF(1,2)+1 : end - padREF(2,2), ... + padREF(1,3)+1 : end - padREF(2,3) ); + + + [ rotClassImg1 ] = BH_bandLimitCenterNormalize(rotClassImg.*volMask, ... + bandpassFilt{iRef} , volBinary,padCalc,'single'); + % [ rotClassImg2 ] = BH_bandLimitCenterNormalize(rotClassImg.*peakMask, ... + % bandpassFilt{iRef} , peakBinary,padCalc,'single'); + clear rotClassImg + + + % now calc CCC, setting sampling shift to zero + [ iCCC, iWeight ] = ... + BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... + iClassWdg, rotWDG,... + peakMask); + + % [ finalPeakCoord ] = BH_multi_xcf_Translational( ... + % rotClassImg2, rotRef_FT2, ... + % peakMask, peakCOM); + + + cccStorage(angCount,:) = [iRef, IDX, phi, theta, psi - phi, iCCC, ... + iWeight, estPeakCoord + ... + peakList(iAngle,4:6) ]; + angCount = angCount + 1; end - end % end inPlane + end + end % end inPlane end % end search over best peaks end - + else % search angles based on grideSearchAngles referenceList = 1:size(ref_FT,4); @@ -304,12 +304,12 @@ nAngles = nRefs.*sum(ANGLE_STEP(:,2)+1).*length(IN_PLANE_SEARCH); cccStorage = zeros(nAngles, 10, 'double', 'gpuArray'); for iAngle = 1:size(ANGLE_STEP,1) - + theta = ANGLE_STEP(iAngle,1); - + % Calculate the increment in phi so that the azimuthal sampling is % consistent and equal to the out of plane increment. - + phiStep = ANGLE_STEP(iAngle,3); % To prevent only searching the same increments each time in a limited @@ -319,42 +319,42 @@ for iAzimuth = 0:ANGLE_STEP(iAngle,2) phi = rem(phiStep * (iAzimuth + azimuthalRandomizer),360) ; - - % For axially averaged this is always zero. - for iInPlane = IN_PLANE_SEARCH - psi = iInPlane; - [phi,theta,psi]; - - for iRef = 1:nRefs + + % For axially averaged this is always zero. + for iInPlane = IN_PLANE_SEARCH + psi = iInPlane; + [phi,theta,psi]; - + for iRef = 1:nRefs + + RotMat = BH_defineMatrix([phi, theta, psi ],'Bah', 'forward'); - - + + [ rotRef ] = BH_resample3d(refRotAvg_FT(:,:,:,iRef),RotMat, ... - [0,0,0], 'Bah', 'GPU', 'forward'); - - - - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - [0,0,0], ... - 'Bah', 'GPU', 'forward')); + [0,0,0], 'Bah', 'GPU', 'forward'); + + + + if isa(refWDG,'cell') + rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... + [0,0,0], ... + 'Bah', 'GPU', 'forward')); else rotWDG = refWDG; - end + end rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - + padREF(1,2)+1 : end - padREF(2,2), ... + padREF(1,3)+1 : end - padREF(2,3) ); + + try rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef} , volBinary,padCalc,'single'); + BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef} , volBinary,padCalc,'single'); rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef} , peakBinary,padCalc,'single'); + BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef} , peakBinary,padCalc,'single'); catch size(volMask) size(peakMask) @@ -366,46 +366,46 @@ rotRef_FT2= conj(rotRef_FT2); - % find translational shift - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); - - % apply only a tranlational shift to the particle - [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - + % find translational shift + [ estPeakCoord ] = BH_multi_xcf_Translational( ... + iClassImg2{iRef}, rotRef_FT2, ... + peakMask, peakCOM); + + % apply only a tranlational shift to the particle + [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... + estPeakCoord,'Bah','GPU','inv'); + rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - [ rotClassImg1 ] = BH_bandLimitCenterNormalize(rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,padCalc,'single'); -% [ rotClassImg2 ] = BH_bandLimitCenterNormalize(rotClassImg.*peakMask, ... -% bandpassFilt{iRef} , peakBinary,padCalc,'single'); - clear rotClassImg - - - % now calc CCC, setting sampling shift to zero - [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg,rotWDG, ... - peakMask); - -% [ finalPeakCoord ] = BH_multi_xcf_Translational( ... -% rotClassImg2, rotRef_FT2, ... -% peakMask, peakCOM); -% - + padREF(1,2)+1 : end - padREF(2,2), ... + padREF(1,3)+1 : end - padREF(2,3) ); + + + [ rotClassImg1 ] = BH_bandLimitCenterNormalize(rotClassImg.*volMask, ... + bandpassFilt{iRef} , volBinary,padCalc,'single'); + % [ rotClassImg2 ] = BH_bandLimitCenterNormalize(rotClassImg.*peakMask, ... + % bandpassFilt{iRef} , peakBinary,padCalc,'single'); + clear rotClassImg + + + % now calc CCC, setting sampling shift to zero + [ iCCC, iWeight ] = ... + BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... + iClassWdg,rotWDG, ... + peakMask); + + % [ finalPeakCoord ] = BH_multi_xcf_Translational( ... + % rotClassImg2, rotRef_FT2, ... + % peakMask, peakCOM); + % + cccStorage(angCount,:) = [iRef, IDX, phi, theta, psi, iCCC, ... - iWeight, estPeakCoord]; - - angCount = angCount + 1; - end % peak search over refs + iWeight, estPeakCoord]; + + angCount = angCount + 1; + end % peak search over refs - end % end inPlane + end % end inPlane end % azimuthal end % polar @@ -414,11 +414,11 @@ end %[ cccStorage ] = BH_multi_peakSearch(referenceList, cccStorage); - cccStorage = cccStorage(( ~(sum(isnan(cccStorage),2)) ),:); - cccStorage = cccStorage(( cccStorage(:,6) ~= 0 ),:); +cccStorage = cccStorage(( ~(sum(isnan(cccStorage),2)) ),:); +cccStorage = cccStorage(( cccStorage(:,6) ~= 0 ),:); - CCC_STORAGE = sortrows(gather(cccStorage),-6); +CCC_STORAGE = sortrows(gather(cccStorage),-6); - clear rotRef rotWDG refWDG rotRef_FT rotRef_FT2 clear rotClassImg1 iClassImg2 +clear rotRef rotWDG refWDG rotRef_FT rotRef_FT2 clear rotClassImg1 iClassImg2 end % end angularSearch function diff --git a/coordinates/BH_multi_calcBinShift.m b/coordinates/BH_multi_calcBinShift.m index a8026029..5bc41d50 100644 --- a/coordinates/BH_multi_calcBinShift.m +++ b/coordinates/BH_multi_calcBinShift.m @@ -9,16 +9,16 @@ if (isTilt) % Expecting just the x,y,z for a tilt series and the binning. Also may - % shift to have an odd dimension so that Imod origin is always the same. - binSize = floor(coords./samplingRate); - binSize = binSize - (1-mod(binSize,2)); - - originFull = floor(coords ./2) + 1; - originBin = floor(binSize./2) + 1; - % This is the shift we need to apply to the binned image to make sure - % that the origin is in the same place. - binShift = -1.*(samplingRate.*originBin - originFull) ./ samplingRate; - + % shift to have an odd dimension so that Imod origin is always the same. + binSize = floor(coords./samplingRate); + binSize = binSize - (1-mod(binSize,2)); + + originFull = floor(coords ./2) + 1; + originBin = floor(binSize./2) + 1; + % This is the shift we need to apply to the binned image to make sure + % that the origin is in the same place. + binShift = -1.*(samplingRate.*originBin - originFull) ./ samplingRate; + end diff --git a/coordinates/BH_multi_calcBinShift.mrc b/coordinates/BH_multi_calcBinShift.mrc deleted file mode 100644 index 01e6d6e8..00000000 --- a/coordinates/BH_multi_calcBinShift.mrc +++ /dev/null @@ -1,10 +0,0 @@ -function [ binShift, binShiftTemplateSearch ] = BH_multi_calcBinShift(coords, samplingRate) -%UNTITLED2 Summary of this function goes here -% Detailed explanation goes here - -tmpShift = (coords-fix(coords./samplingRate).*samplingRate); -binShiftTemplateSearch = [tmpShift(1), tmpShift(3) + tmpShift(2),tmpShift(4)]; -binShift = binShiftTemplateSearch ./ samplingRate; - -end - diff --git a/coordinates/BH_multi_gridCoordinates.m b/coordinates/BH_multi_gridCoordinates.m index f1fa6aaa..7e370f3f 100755 --- a/coordinates/BH_multi_gridCoordinates.m +++ b/coordinates/BH_multi_gridCoordinates.m @@ -1,12 +1,12 @@ function [ Gc1,Gc2,Gc3,g1,g2,g3 ] = BH_multi_gridCoordinates( SIZE, SYSTEM, METHOD, ... - TRANSFORMATION, ... - flgFreqSpace, ... - flgShiftOrigin, flgRad, ... - varargin) + TRANSFORMATION, ... + flgFreqSpace, ... + flgShiftOrigin, flgRad, ... + varargin) %Return grid vectors in R3 for various coordinate systems. % Create grid vectors of dimension SIZE, that are either Cartesian, % Cylindrical, or Spherical. Optionally only return a matrix with radial -% values. Grids are centered with the origin at ceil((N+1)/2). +% values. Grids are centered with the origin at ceil((N+1)/2). if strcmpi(METHOD,'GPU') SIZE = gpuArray(single(SIZE)); else @@ -34,19 +34,19 @@ end end else - if strcmpi(varargin{1}{1},'halfGrid') - doFullGrid = 0; - else - error('1st varargin to grid coords is not numeric or (halfGrid bool) not understood'); - end + if strcmpi(varargin{1}{1},'halfGrid') + doFullGrid = 0; + else + error('1st varargin to grid coords is not numeric or (halfGrid bool) not understood'); + end end end - + if numel(SIZE) == 3 - sX = SIZE(1) ; sY = SIZE(2) ; sZ = SIZE(3); + sX = SIZE(1) ; sY = SIZE(2) ; sZ = SIZE(3); flg3D = 1; elseif numel(SIZE) == 2 - sX = SIZE(1) ; sY = SIZE(2) ; + sX = SIZE(1) ; sY = SIZE(2) ; if strcmpi(METHOD,'GPU'); sZ = gpuArray(single(1)); else sZ = single(1);end flg3D = 0; else @@ -61,7 +61,7 @@ % the origin in IMODs case for an even image is -0.5 relative to mine. % Switching to force odd size - 20171201 conventionShift = [0,0,0]; -% conventionShift = flgShiftOrigin(2:4) .* (1-mod([sX,sY,sZ],2)); + % conventionShift = flgShiftOrigin(2:4) .* (1-mod([sX,sY,sZ],2)); flgShiftOrigin = flgShiftOrigin(1); else conventionShift = [0,0,0]; @@ -75,7 +75,7 @@ symIDX = 0; flgMask = 0; -if iscell(TRANSFORMATION) +if iscell(TRANSFORMATION) switch TRANSFORMATION{1} @@ -96,13 +96,13 @@ DIR = 'forwardVector'; MAG = {TRANSFORMATION{6}}; - case 'single' + case 'single' if numel(TRANSFORMATION{2}) == 9 R = reshape(TRANSFORMATION{2},3,3); else R = reshape(TRANSFORMATION{2},2,2); end - + dXYZ = TRANSFORMATION{3}; if length(dXYZ) == 2 dXYZ = [dXYZ;0]; @@ -113,7 +113,7 @@ symInc = 360 / flgSymmetry; symIDX = 0:flgSymmetry-1; Gc1 = cell(flgSymmetry,1); - Gc2 = cell(flgSymmetry,1); + Gc2 = cell(flgSymmetry,1); Gc3 = cell(flgSymmetry,1); else symIDX = 1; @@ -133,8 +133,8 @@ end end - - + + case 'sequential' flgSequential = 1; @@ -151,77 +151,77 @@ for iTrans = 1:nTrans R_seq{iTrans} = reshape(TRANSFORMATION{iTrans,2},3,3); - + dXYZ_seq{iTrans} = TRANSFORMATION{iTrans,3}; - - + + % Convention is only forward so np need to consider flipping MAG_seq{iTrans} = TRANSFORMATION{iTrans,6}; - + end % For now assuming no symmetry operation on sequential transformations flgSymmetry = TRANSFORMATION{1,5}; DIR = TRANSFORMATION{1,4}; - + symInc = 360 / flgSymmetry; symIDX = 0:flgSymmetry-1; Gc1 = {}; Gc2 = {}; Gc3 ={}; - + otherwise error(['TRANSFORMATION must be a cell,',... - '(none,gridVectors,single,sequential),',... - 'Rotmat, dXYZ, forward|inv, symmetry\n']); + '(none,gridVectors,single,sequential),',... + 'Rotmat, dXYZ, forward|inv, symmetry\n']); end end if ( makeVectors ) - % sX = gpuArray(sX) ; sY = gpuArray(sY) ; sZ = gpuArray(sZ); - if flgShiftOrigin == 1 - if (doFullGrid) - x1 = [-1*floor((sX)/2):floor((sX-1)/2)]; - else - x1 = [0:floor((sX)/2)]; - end - y1 = [-1*floor((sY)/2):floor((sY-1)/2)]; - if (flg3D); z1 = [-1*floor((sZ)/2):floor((sZ-1)/2)]; end - - elseif flgShiftOrigin == -1 - if (doFullGrid) - x1 = [1:sX]; - else - x1 = 1:ceil((sX+1)/2); - end - y1 = [1:sY]; - if (flg3D); z1 = [1:sZ]; end - - elseif flgShiftOrigin == -2 - if (doFullGrid) - x1 = fftshift([1:sX]); - else - x1 = fftshift([1:ceil((sX+1)/2);]); - end - y1 = fftshift([1:sY]); - if (flg3D); z1 = fftshift([1:sZ]); end + % sX = gpuArray(sX) ; sY = gpuArray(sY) ; sZ = gpuArray(sZ); + if flgShiftOrigin == 1 + if (doFullGrid) + x1 = [-1*floor((sX)/2):floor((sX-1)/2)]; else - if (doFullGrid) - x1 = [0:floor(sX/2),-1*floor((sX-1)/2):-1]; - else - x1 = [0:floor(sX/2)]; - end - y1 = [0:floor(sY/2),-1*floor((sY-1)/2):-1]; - if (flg3D); z1 = [0:floor(sZ/2),-1*floor((sZ-1)/2):-1]; end - end - + x1 = [0:floor((sX)/2)]; + end + y1 = [-1*floor((sY)/2):floor((sY-1)/2)]; + if (flg3D); z1 = [-1*floor((sZ)/2):floor((sZ-1)/2)]; end + + elseif flgShiftOrigin == -1 + if (doFullGrid) + x1 = [1:sX]; + else + x1 = 1:ceil((sX+1)/2); + end + y1 = [1:sY]; + if (flg3D); z1 = [1:sZ]; end + + elseif flgShiftOrigin == -2 + if (doFullGrid) + x1 = fftshift([1:sX]); + else + x1 = fftshift([1:ceil((sX+1)/2);]); + end + y1 = fftshift([1:sY]); + if (flg3D); z1 = fftshift([1:sZ]); end + else + if (doFullGrid) + x1 = [0:floor(sX/2),-1*floor((sX-1)/2):-1]; + else + x1 = [0:floor(sX/2)]; + end + y1 = [0:floor(sY/2),-1*floor((sY-1)/2):-1]; + if (flg3D); z1 = [0:floor(sZ/2),-1*floor((sZ-1)/2):-1]; end + end + if strcmpi(METHOD, 'GPU') x1 = gpuArray(x1); y1 = gpuArray(y1); if (flg3D); z1 = gpuArray(z1); end end end - + % Make any needed shifts for convention x1 = x1 - conventionShift(1); y1 = y1 - conventionShift(2); @@ -255,7 +255,7 @@ return end -% Rescale the vectors prior to making gridVectors +% Rescale the vectors prior to making gridVectors if (flgSequential) x1 = x1.*MAG_seq{1}; y1 = y1.*MAG_seq{1}; @@ -263,7 +263,7 @@ else x1 = x1.*MAG{1}; y1 = y1.*MAG{1}; - z1 = z1.*MAG{1}; + z1 = z1.*MAG{1}; end % No matter the case, the cartesian grids are needed @@ -271,7 +271,7 @@ % Optionally evaluate only a smaller masked region -if (flgMask) +if (flgMask) X = X(binaryVol); Y = Y(binaryVol); Z = Z(binaryVol); @@ -281,19 +281,19 @@ for iTrans = 1:1+(flgSequential) if (flgSequential) - rAsym = R_seq{iTrans}; + rAsym = R_seq{iTrans}; if (iTrans == 1) dXyzAsym = 0; % Instead of shifting then shifting back, just note the original shift else - % adding the R2' because the first term doesn't need to be multiplied by - % R2 (in the first action under symmetry loop) but making a change here - % which involves extra multiplications is okay, since this function is - % used much less than 'single' style resampling. - dXyzAsym = ( R_seq{iTrans}'*R_seq{iTrans-1} * ... - dXYZ_seq{iTrans-1}.*MAG_seq{iTrans-1} + ... - R_seq{iTrans-1} * dXYZ_seq{iTrans}.*MAG_seq{iTrans} ); - + % adding the R2' because the first term doesn't need to be multiplied by + % R2 (in the first action under symmetry loop) but making a change here + % which involves extra multiplications is okay, since this function is + % used much less than 'single' style resampling. + dXyzAsym = ( R_seq{iTrans}'*R_seq{iTrans-1} * ... + dXYZ_seq{iTrans-1}.*MAG_seq{iTrans-1} + ... + R_seq{iTrans-1} * dXYZ_seq{iTrans}.*MAG_seq{iTrans} ); + X = Gc1{1}.*MAG_seq{iTrans}; Y = Gc2{1}.*MAG_seq{iTrans}; Z = Gc3{1}.*MAG_seq{iTrans}; @@ -303,18 +303,18 @@ % Note that if symmetric, dXYZ changes each loop after this point dXyzAsym = dXYZ.*MAG{1}; end - + for iSym = symIDX - + % Only in plane symmetries considered anywhere so inv|forward shouldn't % matter. - + R = rAsym * BH_defineMatrix([iSym.*symInc,0,0],'Bah','inv'); - + % Any forward transformations of the grids if (flgTrans) dXYZ = shiftDir .* R*dXyzAsym; - + Xnew = X.*R(1) + Y.*R(4) + Z.*R(7) - dXYZ(1); Ynew = X.*R(2) + Y.*R(5) + Z.*R(8) - dXYZ(2); if (flg3D) @@ -325,13 +325,13 @@ else Xnew = X; Ynew = Y ; Znew = Z; end - + % Only return the radial grid if requested if (flgRad) G1 = sqrt(Xnew.^2 + Ynew.^2 + Znew.^2); G2 = ''; G3 = ''; - + else switch SYSTEM case 'Cartesian' @@ -343,22 +343,22 @@ G2(G2 < 0) = G2(G2 < 0) + 2.*pi; % [0,pi] G3 = acos(Z./G1); - + case 'Cylindrical' - + G1 = sqrt(Xnew.^2 + Ynew.^2); G2 = atan2(Ynew,Xnew); % set from [-pi,pi] --> [0,2pi] G2(G2 < 0) = G2(G2 < 0) + 2.*pi; G3 = Znew; - + otherwise error('SYSTEM must be Cartesian, Spherical, Cylindrical') - end + end end % Only use as cell if symmetry is requested if (flgSymmetry) - % + % Gc1{iSym+1} = G1; Gc2{iSym+1} = G2; @@ -371,7 +371,7 @@ end % loop over symmetric transformations end % loop over sequential transformations - + clear X Y Z Xnew Ynew Znew x1 y1 z1 diff --git a/coordinates/BH_multi_gridSearchAngles.m b/coordinates/BH_multi_gridSearchAngles.m index da635215..1603a578 100755 --- a/coordinates/BH_multi_gridSearchAngles.m +++ b/coordinates/BH_multi_gridSearchAngles.m @@ -1,7 +1,7 @@ function [ nIN_PLANE, IN_PLANE_SEARCH, angleStep, nAngles ] = ... - BH_multi_gridSearchAngles( ANGLE_SEARCH) + BH_multi_gridSearchAngles( ANGLE_SEARCH) %Consolodating function, calculate angular sampling. -% +% % % Called by: % @@ -14,7 +14,7 @@ % BH_alignRaw3d - % doesn't use: nIN_PLANE, ANGLE_INCREMENT % -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % @@ -24,7 +24,7 @@ % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % -% TODO: +% TODO: % - handle helical grid search % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -35,7 +35,7 @@ % For monolayer which can generally be at 0 or 180 +/- create search option % around these two, indicated by a negative value for the out of plane range -if (OUT_OF_PLANE(1) < 0) +if (OUT_OF_PLANE(1) < 0) OUT_OF_PLANE(1)= abs(OUT_OF_PLANE(1)); biPolarSearch = 1; else @@ -49,7 +49,7 @@ symmetryConstrainedSearch = 0; end - + if all(IN_PLANE) @@ -68,14 +68,14 @@ nIN_PLANE = length(IN_PLANE_SEARCH); angleStep(1,:) = [0,0,0,0,psiStep]; else -IN_PLANE_SEARCH = 0; + IN_PLANE_SEARCH = 0; if (symmetryConstrainedSearch) for iSym = 1:symmetryConstrainedSearch-1 IN_PLANE_SEARCH = [IN_PLANE_SEARCH,IN_PLANE_SEARCH + iSym.*(360/symmetryConstrainedSearch)]; end - + end - + psiStep = 0; nIN_PLANE = 1; % Always search the unrotated sample angleStep(1,:) = [0,0,0,0,0]; @@ -89,7 +89,7 @@ else topPolar = OUT_OF_PLANE(1)/OUT_OF_PLANE(2); thetaStep = OUT_OF_PLANE(2); - end + end if (biPolarSearch) polarAngles = 0:OUT_OF_PLANE(2):OUT_OF_PLANE(1);%(0:topPolar).*thetaStep; @@ -109,23 +109,23 @@ if (theta - thetaStep == 0) || (theta + thetaStep == 180) % strict even spacing leaves the first out of plane undersampled phiStep = 0.5 * phiStep; - + end - + nAzimuthal = floor(360/phiStep); - - + + end % first position is psiStep independent of this. if (iPolarAngle) angleStep(iPolarAngle,:) = [theta, nAzimuthal, ... - phiStep, thetaStep, psiStep]; + phiStep, thetaStep, psiStep]; else angleStep(iPolarAngle,:) = [theta, nAzimuthal, ... - phiStep, 0, psiStep]; + phiStep, 0, psiStep]; end - end + end else % Setting top polar limits the angular search to at most the in plane angles, % as the azimuth is also zero for iPolarAngle = 1. @@ -136,28 +136,28 @@ % full search, rotationally averaged, first in plane, refinement1, refinement 2. nAngles = zeros(5,1); if sum(any(angleStep)) - + for i = 1:size(angleStep,1) if angleStep(i,1) == 0 % theta = 0 in plane only - nAngles(1) = nAngles(1) + nIN_PLANE; + nAngles(1) = nAngles(1) + nIN_PLANE; else nAngles(1) = nAngles(1) + (angleStep(i,2) .* nIN_PLANE); end - nAngles(2) = nAngles(2) + angleStep(i,2); + nAngles(2) = nAngles(2) + angleStep(i,2); end else -% Translational only search -nAngles = nAngles + 1; + % Translational only search + nAngles = nAngles + 1; end nAngles(3) = 10 .* nIN_PLANE; nAngles(4) = 175; nAngles(5) = 630; - + end % end of gridSearchAngles function diff --git a/coordinates/BH_multi_iterator.m b/coordinates/BH_multi_iterator.m index c300851e..467c46e7 100755 --- a/coordinates/BH_multi_iterator.m +++ b/coordinates/BH_multi_iterator.m @@ -8,11 +8,11 @@ % nextBest = [64,72,96,108,128,144,160,168,180,192,216,224,256,... % 270,288,300,320,336,360,384,400,432,448,480,512]; nextBest = [64,72,80,84,90,96,108,112,120,126,128,144,160,162, ... - 168,180,192,216,224,240,256,270,288,320,324,336,... - 360,378,384,400,416,432,448,480,486,504,512]; + 168,180,192,216,224,240,256,270,288,320,324,336,... + 360,378,384,400,416,432,448,480,486,504,512]; % fall off between 1 and 0 in masking function APODIZATION = 2.*6; - + switch OPERATION case 'fourier' @@ -27,23 +27,23 @@ sizeTarget= SIZES(1,:); if ( flgDescend ) for i = 1:3 - try - nB(i) = nextBest(find(nextBest <= sizeTarget(i), 1, 'last')); - catch - nB(i) = nextBest(1); - end + try + nB(i) = nextBest(find(nextBest <= sizeTarget(i), 1, 'last')); + catch + nB(i) = nextBest(1); + end end else for i = 1:3 try - nB(i) = nextBest(find(nextBest >= sizeTarget(i), 1, 'first')); + nB(i) = nextBest(find(nextBest >= sizeTarget(i), 1, 'first')); catch nB(i) = nextBest(end); end end end - - + + if ~all(nB) error('next best not found in range 128-512 for [%d,%d,%d]', sizeTarget); end @@ -51,21 +51,21 @@ case 'fourier2d' % Found by -% % % for i = 64:2:3838*2 -% % % if (sum(factor(i).*(factor(i) >= 5))< 10) -% % % f = [f,i]; -% % % end -% % % end + % % % for i = 64:2:3838*2 + % % % if (sum(factor(i).*(factor(i) >= 5))< 10) + % % % f = [f,i]; + % % % end + % % % end nextBest = [64,72,80,84,90,96,108,112,120,126,128,144,160,162, ... - 168,180,192,216,224,240,252,256,270,288,320,324,336,... - 360,378,384,432,448,480,486,504,512,540,576,640,648,... - 720,756,768,810,864,896,960,972,1008,1024,1080,... - 1134,1152,1280,1296,1344,1440,1458,1512,1536,1620,1728,... - 1792,1920,1944,2016,2048,2160,2268,2304,2430,2560,... - 2592,2688,2880,2916,3024,3072,3240,3402,3456,3584,... - 3840,3888,4032,4096,4320,4374,4536,4608,4860,5120,... - 5184,5376,5760,5832,6048,6144,6480,6804,6912,7168,7290,... - 7488,7560,7840,7920,8192]; + 168,180,192,216,224,240,252,256,270,288,320,324,336,... + 360,378,384,432,448,480,486,504,512,540,576,640,648,... + 720,756,768,810,864,896,960,972,1008,1024,1080,... + 1134,1152,1280,1296,1344,1440,1458,1512,1536,1620,1728,... + 1792,1920,1944,2016,2048,2160,2268,2304,2430,2560,... + 2592,2688,2880,2916,3024,3072,3240,3402,3456,3584,... + 3840,3888,4032,4096,4320,4374,4536,4608,4860,5120,... + 5184,5376,5760,5832,6048,6144,6480,6804,6912,7168,7290,... + 7488,7560,7840,7920,8192]; sizeTarget= SIZES(1,:); nB = [0,0]; for i = 1:2 @@ -75,7 +75,7 @@ nB(i) = 0; end end - + if ~all(nB) % For some reason "error" only takes scalar args for print formating. fprintf('next best not found in range 64-7290 for [%d,%d,%d]\n',target); @@ -86,7 +86,7 @@ case 'convolution' % This could be optimized automatically to balance increased target size vs % number of iterations/post padding. - + % the target size, to get the most out of calcs, spend as much time on the % gpu as possible. With finer angular searches, the number of references % needs more memory, so a smaller size here means more transfers, but this @@ -95,48 +95,48 @@ nextBest = [128,144,160,168,192,216,224,256]; elseif all(SIZES(1,:) == 384) nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384]; + 288,300,320,336,360,384]; elseif all(SIZES(1,:) == 432) nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432]; + 288,300,320,336,360,384,400,432]; elseif all(SIZES(1,:) == 512) nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432,480,512]; + 288,300,320,336,360,384,400,432,480,512]; elseif all(SIZES(1,:) > 512) nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432,480,512,... - 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]; + 288,300,320,336,360,384,400,432,480,512,... + 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]; end sizeImage = SIZES(2,:); sizeTemplate = SIZES(3,:); % the mask or kernel sizeParticle = SIZES(4,:) ;% a subregion of sizeTemplate - - + + borderSizeCalc = floor((sizeTemplate + APODIZATION)./2); borderSizeKeep = borderSizeCalc + 2.*sizeParticle; - + abs(sum(sizeImage - sizeTemplate)) sum(0.1.*sizeImage) if abs(sum(sizeImage - sizeTemplate)) < sum(0.1.*sizeImage) OUTPUT = [[0,0,0];[0,0,0] ;sizeImage; ... - sizeImage; sizeImage ; [1,1,1]]; + sizeImage; sizeImage ; [1,1,1]]; return end score = zeros(length(nextBest),6); - + validCalc = repmat(nextBest',1,3) - 2.*repmat(borderSizeCalc,length(nextBest),1); validKeep = repmat(nextBest',1,3) - 2.*repmat(borderSizeKeep,length(nextBest),1); minIter= floor(repmat(sizeImage,length(nextBest),1)./validKeep); postPad= repmat(nextBest',1,3)- ... - (repmat(sizeImage+borderSizeKeep,length(nextBest),1) -minIter.*(validKeep+1)); - + (repmat(sizeImage+borderSizeKeep,length(nextBest),1) -minIter.*(validKeep+1)); + % Added this so I can work with test cases where the volume to be % searched is the same size as the reference - score(:,2:4) = repmat(nextBest',1,3) ./ postPad .* (minIter >= 0) + score(:,2:4) = repmat(nextBest',1,3) ./ postPad .* (minIter >= 0) [~, cX] = max(score(:,2)) ; [~, cY] = max(score(:,3)) ; @@ -150,13 +150,13 @@ validAreaCalc = validCalc([cX,cY,cZ]); nIters = minIter([cX,cY,cZ])+1; postPAD= postPad([cX,cY,cZ]); - + OUTPUT = [borderSizeKeep;postPAD ;chunkSize; ... - validAreaKeep; validAreaCalc ; nIters]; + validAreaKeep; validAreaCalc ; nIters]; case 'binning' - + binFactor = SIZES(1,:); sizeImage = SIZES(2,:); @@ -185,7 +185,7 @@ end end end - + OUTPUT(i) = nextBest(fftVal); end @@ -210,12 +210,12 @@ end end end - + OUTPUT = [finalVal; padVal]; otherwise error('OPERATION is case-sensitive fourier, convolution or binning, not %s', OPERATION); end - + end diff --git a/coordinates/BH_multi_recGeom.m b/coordinates/BH_multi_recGeom.m index a77fb0e9..702000dd 100755 --- a/coordinates/BH_multi_recGeom.m +++ b/coordinates/BH_multi_recGeom.m @@ -11,7 +11,7 @@ recCoords = textscan(recFile,'%f'); fclose(recFile); -%%% Some sanity checks +%%% Some sanity checks % First line should be the name of the tilt-series the tomo is % reconstructed from. [~,tiltNameFromTomo,~] = fileparts(reconCoordName); @@ -23,9 +23,9 @@ recGeom = zeros(nTomos,6); for iSt = 1:nTomos - recGeom(iSt,:) = recCoords{1}(1 + (iSt-1)*6: 6 + (iSt-1)*6); + recGeom(iSt,:) = recCoords{1}(1 + (iSt-1)*6: 6 + (iSt-1)*6); end - + % Note that the x/z shifts (col 5,6) are shifts given to IMOD, which are the opposite of the location of the origin (relative to the center) % To make it more confusing, since the reconstruction is done in a ref frame rotated about X, the Z shift is flipped so it matches the origin in Z end diff --git a/coordinates/EMC_coordGrids.m b/coordinates/EMC_coordGrids.m index 8e40ac64..0bc8a468 100644 --- a/coordinates/EMC_coordGrids.m +++ b/coordinates/EMC_coordGrids.m @@ -104,7 +104,7 @@ gY(gY < 0) = gY(gY < 0) + 2.*pi; % set from [-pi,pi] to [0,2pi] gZ = nan; else - error('EMC:SYSTEM', "SYSTEM should be 'cartesian', 'spherical', 'cylindrical' or 'radial'") + error('EMC:SYSTEM', "SYSTEM should be 'cartesian', 'spherical', 'cylindrical' or 'radial'") end end diff --git a/coordinates/EMC_coordTransform.m b/coordinates/EMC_coordTransform.m index 8cf1d364..92cb6902 100755 --- a/coordinates/EMC_coordTransform.m +++ b/coordinates/EMC_coordTransform.m @@ -46,7 +46,7 @@ % default = 1 % % -> 'offset' (vector): [x, y, z] or [x, y] offset to apply (should correspond to SIZE). -% Offsets are used to adjust the center of rotation defined by 'origin'. +% Offsets are used to adjust the center of rotation defined by 'origin'. % NOTE: this effectively apply a shift on both the vectors and the grids. % NOTE: if there is no rotation or scaling to apply, this has no effect % on the final interpolated image. @@ -118,7 +118,7 @@ if ~isempty(varargin) if length(varargin) ~= 3 error('EMC:varargin', ... - 'varargin should contain 3 row vectors, got %s elements', length(varargin)) + 'varargin should contain 3 row vectors, got %s elements', length(varargin)) elseif ~flg.is3d if ~isscalar(varargin{3}) && ~isnan(varargin{3}) error('EMC:varargin', 'For a 2d case, vZ should be NaN') @@ -128,10 +128,10 @@ end else vX = EMC_setMethod(cast(varargin{1}, OPTION.precision), METHOD); - vY = EMC_setMethod(cast(varargin{2}, OPTION.precision), METHOD); + vY = EMC_setMethod(cast(varargin{2}, OPTION.precision), METHOD); vZ = EMC_setMethod(cast(varargin{3}, OPTION.precision), METHOD); end - + if ~isnumeric(vX) || ~isrow(vX) || SIZE(1) ~= length(vX) error('EMC:varargin', 'varargin{1} (vX) should be a numeric row vector of %d elements', SIZE(1)) elseif ~isnumeric(vY) || ~isrow(vY) || SIZE(2) ~= length(vY) @@ -139,19 +139,19 @@ elseif flg.is3d && ~isnumeric(vZ) || ~isrow(vZ) || SIZE(3) ~= length(vZ) error('EMC:varargin', 'varargin{3} (vZ) should be a numeric row vector of %d elements', SIZE(3)) end - + % Apply offsets and|or normalize if whished. Note: shifts are not applied to vectors. if (flg.offset) - vX = vX - OPTION.offset(1); - vY = vY - OPTION.offset(2); - if flg.is3d; vZ = vZ - OPTION.offset(3); end + vX = vX - OPTION.offset(1); + vY = vY - OPTION.offset(2); + if flg.is3d; vZ = vZ - OPTION.offset(3); end end if (OPTION.normalize) vX = vX ./ SIZE(1); - vY = vY ./ SIZE(2); + vY = vY ./ SIZE(2); if flg.is3d; vZ = vZ ./ SIZE(3); end end - + else % varargin is empty OPTION = EMC_getOption(OPTION, {'offset', 'origin', 'normalize', 'precision'}, true); [vX, vY, vZ] = EMC_coordVectors(SIZE, METHOD, OPTION, false); @@ -163,7 +163,7 @@ % Compute the grids; Optionally evaluate only a smaller masked region. if flg.is3d [X, Y, Z] = ndgrid(vX, vY, vZ); - if flg.binary + if flg.binary X = X(binaryVol); Y = Y(binaryVol); Z = Z(binaryVol); @@ -196,9 +196,9 @@ % Only in plane symmetries considered anywhere % so inverse|forward shouldn't matter. if iSym > 1 - R = OPTION.rotm * BH_defineMatrix([iSym.*symInc,0,0],'Bah','inverse'); + R = OPTION.rotm * BH_defineMatrix([iSym.*symInc,0,0],'Bah','inverse'); else - R = OPTION.rotm; + R = OPTION.rotm; end if flg.transform || iSym > 1 @@ -218,7 +218,7 @@ YTrans = Y; if flg.is3d; ZTrans = Z; else; ZTrans = nan; end end - + % Only use as cell if symmetry is requested if flg.sym gX{iSym+1} = XTrans; % I [TF] hope this doesn't generate a copy. @@ -240,7 +240,7 @@ [flg.is3d, SIZE, ndim] = EMC_is3d(SIZE); if ~(strcmpi(METHOD, 'gpu') || strcmpi(METHOD, 'cpu')) - if isstring(METHOD) || ischar(METHOD) + if isstring(METHOD) || ischar(METHOD) error('EMC:METHOD', "SYSTEM should be 'gpu' or 'cpu', got %s", METHOD) else error('EMC:METHOD', "SYSTEM should be 'gpu' or 'cpu', got %s", clas(METHOD)) @@ -255,16 +255,16 @@ % Extract optional parameters OPTION = EMC_getOption(OPTION, {'rotm', 'shift', 'mag', 'sym', 'direction', ... - 'origin', 'offset', 'binary', 'normalize', 'precision'}, false); + 'origin', 'offset', 'binary', 'normalize', 'precision'}, false); % rotm if isfield(OPTION, 'rotm') - if ~isnumeric(OPTION.rotm) || ~ismatrix(OPTION.rotm) + if ~isnumeric(OPTION.rotm) || ~ismatrix(OPTION.rotm) error('EMC:rotm', 'rotm should be a %dx%d numeric matrix, got %s', ... - ndim, ndim, class(OPTION.rotm)) + ndim, ndim, class(OPTION.rotm)) elseif numel(OPTION.rotm) == ndim^2 error('EMC:rotm', 'rotm should be a %dx%d numeric matrix, got size:%s', ... - ndim, ndim, mat2str(size(OPTION.rotm))) + ndim, ndim, mat2str(size(OPTION.rotm))) end % Most of the time, it will not be an identity matrix, so don't check and do transformation anyway. flg.transform = true; @@ -276,14 +276,14 @@ if isfield(OPTION, 'shift') if ~isnumeric(OPTION.shift) || ~isvector(OPTION.shift) error('EMC:shift', ... - 'shift should be a vector of float|int, got %s', class(OPTION.shift)) + 'shift should be a vector of float|int, got %s', class(OPTION.shift)) elseif any(isnan(OPTION.shift)) || any(isinf(OPTION.shift)) error('EMC:shift', ... - 'shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) + 'shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) elseif numel(OPTION.shift) ~= ndim error('EMC:shift', ... - 'For a %dd SIZE, shift should be a vector of %d float|int, got %s', ... - ndim, ndim, mat2str(OPTION.shift, 2)) + 'For a %dd SIZE, shift should be a vector of %d float|int, got %s', ... + ndim, ndim, mat2str(OPTION.shift, 2)) elseif any(OPTION.shift) flg.shift = true; end @@ -295,22 +295,22 @@ if isfield(OPTION, 'mag') if ~isnumeric(OPTION.mag) error('EMC:mag', ... - 'mag should be a numeric scalar or vector, got %s', class(OPTION.mag)) + 'mag should be a numeric scalar or vector, got %s', class(OPTION.mag)) elseif isvector(OPTION.mag) if length(OPTION.mag) ~= ndim error('EMC:mag', ... - 'mag should be a vector of %d elements, got %d elements', ndim, length(OPTION.mag)) - elseif any(isnan(OPTION.mag)) || any(isinf(OPTION.mag)) + 'mag should be a vector of %d elements, got %d elements', ndim, length(OPTION.mag)) + elseif any(isnan(OPTION.mag)) || any(isinf(OPTION.mag)) error('EMC:mag', ... - 'mag should not have any nan nor inf, got:%s', mat2str(OPTION.mag)) + 'mag should not have any nan nor inf, got:%s', mat2str(OPTION.mag)) end flg.transform = true; elseif isscalar(OPTION.mag) && ~any(isnan(OPTION.mag)) || ~any(isinf(OPTION.mag)) OPTION.mag = zeros(1, ndim) + OPTION.mag; % isotropic scaling flg.transform = true; else - error('EMC:mag', ... - 'mag should be a numeric scalar or a numeric vector of %d elements', ndim) + error('EMC:mag', ... + 'mag should be a numeric scalar or a numeric vector of %d elements', ndim) end else OPTION.mag = ones(1, ndim); % default @@ -320,7 +320,7 @@ if isfield(OPTION, 'sym') if ~isnumeric(OPTION.sym) || ~isscalar(OPTION.sym) || OPTION.sym < 1 || rem(OPTION.sym, 1) error('EMC:sym', ... - 'sym should be a positive integer') + 'sym should be a positive integer') elseif OPTION.sym ~= 1 flg.sym = true; end @@ -344,7 +344,7 @@ % origin if isfield(OPTION, 'origin') if ~isnumeric(OPTION.origin) || ~isscalar(OPTION.origin) || ... - ~(OPTION.origin == 1 || OPTION.origin == -1 || OPTION.origin == 0 || OPTION.origin == 2) + ~(OPTION.origin == 1 || OPTION.origin == -1 || OPTION.origin == 0 || OPTION.origin == 2) error('EMC:origin', 'origin should be 0, 1, 2, or -1, got %d', OPTION.origin) end else @@ -355,14 +355,14 @@ if isfield(OPTION, 'offset') if ~isnumeric(OPTION.offset) || ~isvector(OPTION.offset) error('EMC:offset', ... - 'offset should be a vector of float|int, got %s', class(OPTION.offset)) + 'offset should be a vector of float|int, got %s', class(OPTION.offset)) elseif any(isnan(OPTION.offset)) || any(isinf(OPTION.offset)) error('EMC:offset', ... - 'offset should not contain NaNs or Inf, got %s', mat2str(OPTION.offset, 2)) + 'offset should not contain NaNs or Inf, got %s', mat2str(OPTION.offset, 2)) elseif numel(OPTION.offset) ~= ndim error('EMC:offset', ... - 'For a %dd SIZE, offset should be a vector of %d float|int, got %s', ... - ndim, ndim, mat2str(OPTION.offset, 2)) + 'For a %dd SIZE, offset should be a vector of %d float|int, got %s', ... + ndim, ndim, mat2str(OPTION.offset, 2)) end else OPTION.offset = zeros(1, ndim); % default diff --git a/coordinates/EMC_coordVectors.m b/coordinates/EMC_coordVectors.m index eca2ce51..5f540173 100644 --- a/coordinates/EMC_coordVectors.m +++ b/coordinates/EMC_coordVectors.m @@ -78,7 +78,7 @@ if isfield(OPTION, 'origin') if ~isscalar(OPTION.origin) || ~isnumeric(OPTION.origin) error('EMC:origin', 'OPTION.origin should be an integer, got %s of size: %s', ... - class(OPTION.origin), mat2str(size(OPTION.origin))) + class(OPTION.origin), mat2str(size(OPTION.origin))) elseif OPTION.origin ~= 1 && OPTION.origin ~= -1 && OPTION.origin ~= 0 && OPTION.origin ~= 2 error('EMC:origin', 'OPTION.origin should be 0, 1, 2, or -1, got %d', OPTION.origin) end @@ -97,17 +97,17 @@ if isfield(OPTION, 'shift') if ~isnumeric(OPTION.shift) || ~isrow(OPTION.shift) error('EMC:shift', ... - 'OPTION.shift should be a row vector of float|int, got %s', class(OPTION.shift)) + 'OPTION.shift should be a row vector of float|int, got %s', class(OPTION.shift)) elseif any(isnan(OPTION.shift)) || any(isinf(OPTION.shift)) error('EMC:shift', ... - 'OPTION.shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) + 'OPTION.shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) elseif numel(OPTION.shift) ~= ndim error('EMC:shift', ... - 'For a %dd SIZE, OPTION.shift should be a vector of %d float|int, got %s', ... - ndim, ndim, mat2str(OPTION.shift, 2)) + 'For a %dd SIZE, OPTION.shift should be a vector of %d float|int, got %s', ... + ndim, ndim, mat2str(OPTION.shift, 2)) elseif (OPTION.half || OPTION.origin == -1) && any(OPTION.shift) error('EMC:shift', ... - 'OPTION.shifts are not allowed with half=true or origin=-1 , got %s', mat2str(OPTION.shift, 2)) + 'OPTION.shifts are not allowed with half=true or origin=-1 , got %s', mat2str(OPTION.shift, 2)) end else OPTION.shift = zeros(1, ndim); % default @@ -131,7 +131,7 @@ if isfield(OPTION, 'precision') if ~(ischar(OPTION.precision) || isstring(OPTION.precision)) || ... - ~strcmpi(OPTION.precision, 'single') && ~strcmpi(OPTION.precision, 'double') + ~strcmpi(OPTION.precision, 'single') && ~strcmpi(OPTION.precision, 'double') error('EMC:precision', "OPTION.precision should be 'single' or 'double'") end else @@ -188,7 +188,7 @@ if isDim(2); vY = (limits(1, 2) - OPTION.shift(2)):(limits(2, 2) - OPTION.shift(2)); else; vY = nan; end if is3d; vZ = (limits(1, 3) - OPTION.shift(3)):(limits(2, 3) - OPTION.shift(3)); else; vZ = nan; end -% reciprocal space + % reciprocal space else if isDim(1) if (OPTION.half) diff --git a/coordinates/eulerSearch.m b/coordinates/eulerSearch.m index 17f1e0ee..f58d5836 100644 --- a/coordinates/eulerSearch.m +++ b/coordinates/eulerSearch.m @@ -10,7 +10,7 @@ number_of_out_of_plane_angles = 1; % poorly named. Theta of zero is still searched but not "outofplane" number_of_angles_at_each_theta = []; best_parameters_to_keep = 0; - list_of_search_parameters = {}; + list_of_search_parameters = {}; list_of_best_parameters = {}; symmetry_symbol = 'C1'; number_of_asymmetric_units = 1; @@ -36,13 +36,13 @@ methods function [obj] = eulerSearch(wanted_symmetry_symbol, ... - wanted_theta_max,... - wanted_theta_step,... - wanted_psi_max,... - wanted_psi_step,... - wanted_resolution_limit,... - wanted_parameters_to_keep,... - wanted_random_start_angle) + wanted_theta_max,... + wanted_theta_step,... + wanted_psi_max,... + wanted_psi_step,... + wanted_resolution_limit,... + wanted_parameters_to_keep,... + wanted_random_start_angle) if (wanted_theta_max < 0) wanted_theta_max = abs(wanted_theta_max); @@ -50,7 +50,7 @@ end obj.random_start_angle = wanted_random_start_angle; - + obj.theta_max = wanted_theta_max; obj.theta_step = wanted_theta_step; % emClarity takes -angle:step:angle. This max is based on 0:360 @@ -58,24 +58,24 @@ obj.psi_max = 2.*wanted_psi_max; obj.psi_step = wanted_psi_step; obj.symmetry_symbol = wanted_symmetry_symbol; - + SetSymmetryLimits(obj); CalculateGridSearchPositions(obj); - + end - - + + function [] = CalculateGridSearchPositions(obj) - - + + theta_max_local = obj.theta_max; obj.parameter_map.psi = -obj.psi_max./2 : obj.psi_step : obj.psi_max/2; - obj.number_of_search_positions = 0; - + obj.number_of_search_positions = 0; + theta_search = [ 0 : obj.theta_step : theta_max_local ]; if isempty(theta_search) - theta_search = 0; + theta_search = 0; end if (obj.bipolar_search) theta_search = [theta_search, flip(180-theta_search)]; @@ -85,7 +85,7 @@ obj.number_of_angles_at_each_theta = zeros(obj.number_of_out_of_plane_angles,1); obj.parameter_map.phi = cell(obj.number_of_out_of_plane_angles,1); - + obj.number_of_search_positions = 0; % Change this to include inplane angles explicitly. @@ -97,21 +97,21 @@ else % angular sampling was adapted from Spider subroutine VOEA (Paul Penczek) phi_step = 1.*abs(obj.theta_step / sind(theta)); - if (phi_step > obj.phi_max) + if (phi_step > obj.phi_max) phi_step = obj.phi_max; else phi_step = obj.phi_max / floor(obj.phi_max / phi_step + 0.5); end end - - if (obj.random_start_angle == true) + + if (obj.random_start_angle == true) phi_start_local = phi_step / 2.0 * (rand(1) - 0.5); else phi_start_local = 0.0; end - - obj.parameter_map.phi{iT} = [0:phi_step:obj.phi_max - 1] + phi_start_local; - + + obj.parameter_map.phi{iT} = [0:phi_step:obj.phi_max - 1] + phi_start_local; + obj.number_of_angles_at_each_theta(iT) = length(obj.parameter_map.phi{iT}) .* length(obj.parameter_map.psi); end @@ -122,43 +122,43 @@ end end - - function [] = SetSymmetryLimits(obj) + function [] = SetSymmetryLimits(obj) + switch obj.symmetry_symbol(1) - case 'C' - if (length(obj.symmetry_symbol) < 2) - error('Cyclic symmetry requires an int specifying CX'); - end - + case 'C' + if (length(obj.symmetry_symbol) < 2) + error('Cyclic symmetry requires an int specifying CX'); + end + obj.psi_max = min(obj.psi_max,360.0 / EMC_str2double(obj.symmetry_symbol(2:end))); obj.theta_max = min(180.0, obj.theta_max); obj.phi_max = 360.0; % This will be incompatible with "symmetry_constrained_search" in BH_mutli_gridAngleSEarch (or whatever) obj.number_of_asymmetric_units = EMC_str2double(obj.symmetry_symbol(2:end)); case 'D' % FIXME is this right? - if ((length(obj.symmetry_symbol) < 2)) - error('D symmetry requires an int specifying DX'); - end + if ((length(obj.symmetry_symbol) < 2)) + error('D symmetry requires an int specifying DX'); + end obj.psi_max = min(360.0 / EMC_str2double(obj.symmetry_symbol(2:end))); obj.theta_max = min(obj.theta_max,90.0); obj.phi_max = 360.0; obj.number_of_asymmetric_units = EMC_str2double(obj.symmetry_symbol(2:end)*2); - - case 'O' - if ((length(obj.symmetry_symbol) > 1)) - error('Octahedral symmetry requires no int'); - end - + + case 'O' + if ((length(obj.symmetry_symbol) > 1)) + error('Octahedral symmetry requires no int'); + end + obj.psi_max = min(obj.psi_max,90.0); - obj.theta_max = min(obj.theta_max,54.7); + obj.theta_max = min(obj.theta_max,54.7); obj.phi_max = 90.0; obj.number_of_asymmetric_units = 24; - - case 'I' - % Double check convention: TODO - % 2 fold on Z, 5 fold 31.17 deg around X on Y axis, 3 fold 20.91 - % deg around Y on X. For I2 the X/Y axes are flipped + + case 'I' + % Double check convention: TODO + % 2 fold on Z, 5 fold 31.17 deg around X on Y axis, 3 fold 20.91 + % deg around Y on X. For I2 the X/Y axes are flipped if ((length(obj.symmetry_symbol) < 2)) obj.psi_max = min(obj.psi_max,180.0); obj.theta_max = 31.7; @@ -166,7 +166,7 @@ elseif strcmp(obj.symmetry_symbol,'2') obj.psi_max = min(obj.psi_max,180.0); obj.theta_max = 31.7; - obj.phi_max = 180.0; + obj.phi_max = 180.0; else error('Icosohedral can be I or I2, not (%s)',obj.symmetry_symbol); end @@ -175,8 +175,8 @@ error('symmetry symbol (%s) not recognized', obj.symmetry_symbol); end end - - + + end end diff --git a/ctf/BH_ctfCalc.m b/ctf/BH_ctfCalc.m index ea35c490..dbb360ec 100755 --- a/ctf/BH_ctfCalc.m +++ b/ctf/BH_ctfCalc.m @@ -1,5 +1,5 @@ function [ ctfMask, Hqz] = BH_ctfCalc(PixelSize, Cs, Lamda, Defocus, ... - CTFSIZE, AMPCONT, Phase_Only, varargin) + CTFSIZE, AMPCONT, Phase_Only, varargin) % Calculate a ctf and with the given modification to information at resolution % lower than the first peak. Also return the unmodified ctf. @@ -17,9 +17,9 @@ precision = 'double'; calcOneD = 0; else - maxZ = varargin{1}; - precision = 'double'; - calcOneD = 1; + maxZ = varargin{1}; + precision = 'double'; + calcOneD = 1; end else precision = 'double'; @@ -48,8 +48,8 @@ preShiftedOrigin = PixelSize{2}; phi = PixelSize{3}; calcRad = 0; - - + + % While switching to the default behavior of using ang, check any input radial grid % and make sure it is in Angstrom. Check the middle of the frequency % range that way the input could be shifted or not. @@ -67,7 +67,7 @@ calcRad = 1; end -CS = Cs; +CS = Cs; WL = Lamda; if numel(Defocus) == 1 @@ -83,15 +83,15 @@ - CS = CS * 10^10; - WL = WL * 10^10; - df1 = df1 * 10^10; - df2 = df2 * 10^10; - +CS = CS * 10^10; +WL = WL * 10^10; +df1 = df1 * 10^10; +df2 = df2 * 10^10; -if numel(CTFSIZE) == 1 + +if numel(CTFSIZE) == 1 CTFSIZE(2) = CTFSIZE(1); end @@ -100,27 +100,27 @@ end if ( calcRad ) - + if strcmpi(precision, 'single') [radialGrid,phi,~,~,~,~] = ... - BH_multi_gridCoordinates(CTFSIZE(1:2),'Cylindrical','GPU',{'none'},1,0,0); + BH_multi_gridCoordinates(CTFSIZE(1:2),'Cylindrical','GPU',{'none'},1,0,0); else - + [radialGrid,phi,~,~,~,~] = ... - BH_multi_gridCoordinates(CTFSIZE(1:2),'Cylindrical','GPU',{'none'},1,0,0); + BH_multi_gridCoordinates(CTFSIZE(1:2),'Cylindrical','GPU',{'none'},1,0,0); radialGrid = double(radialGrid); phi = double(phi); end - radialGrid = radialGrid ./ PIXEL_SIZE; - + radialGrid = radialGrid ./ PIXEL_SIZE; + end % Any additional phase shift due to the phase plate is stored with the % amplitude contrast. This will produce very small errors (< 1% for .07,0.1) % for older versions that expect just the amplitude contrast ratio, rather % than the phase shift. -% % % if (abs(AMPCONT - 1.0) < 1e-3) +% % % if (abs(AMPCONT - 1.0) < 1e-3) % % % precomputed_amplitude_contrast_term = pi / 2.0; % % % else % % % precomputed_amplitude_contrast_term = atan2(AMPCONT,sqrt(1.0 - AMPCONT^2)); @@ -131,83 +131,83 @@ % phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 + DF.*WL.*(radialGrid).^2); dfTerm = 0.5.*( (df1+df2) + (df1-df2)*cos(2.*(phi-phi0)) ); phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 + ... - WL.*(radialGrid).^2 .* dfTerm); -% dPdQ = 2*pi*CS*WL^3.*radialGrid.^3 + 2*WL.*radialGrid.*dfTerm; + WL.*(radialGrid).^2 .* dfTerm); +% dPdQ = 2*pi*CS*WL^3.*radialGrid.^3 + 2*WL.*radialGrid.*dfTerm; if ( flgComplex ) ctfMask = exp(-1i.*(phasePerturbation-atan2(AMPCONT,sqrt(1+AMPCONT)))); Hqz = exp(+1i.*(phasePerturbation-atan2(AMPCONT,sqrt(1+AMPCONT)))); return else -% Hqz = (sqrt(1-AMPCONT^2).*sin(phasePerturbation) - AMPCONT.*cos(phasePerturbation)); + % Hqz = (sqrt(1-AMPCONT^2).*sin(phasePerturbation) - AMPCONT.*cos(phasePerturbation)); end Hqz = sin(phasePerturbation - AMPCONT); nanCheck = isnan(Hqz); if gather(sum(nanCheck(:))) - Hqz(nanCheck) = 0; + Hqz(nanCheck) = 0; end if Phase_Only < 0 - - + + % FIXME I need to know if I am a half grid or els this fails! - - oX = floor(CTFSIZE(1)/2)+1; - oY = floor(CTFSIZE(2)/2)+1; - - - if (calcOneD) - if (preShiftedOrigin) - rV = Hqz(oX:end); - else - rV = Hqz(1:oX); % should this be oX-1? - end - elseif (doHalfGrid) - if ( preShiftedOrigin) - rV = Hqz(1:end,oY); - else - rV = Hqz(1:end,1); - end + + oX = floor(CTFSIZE(1)/2)+1; + oY = floor(CTFSIZE(2)/2)+1; + + + if (calcOneD) + if (preShiftedOrigin) + rV = Hqz(oX:end); else - if ( preShiftedOrigin) - rV = Hqz(oX:end,oY); - else - rV = Hqz(1:oX,1); - end + rV = Hqz(1:oX); % should this be oX-1? end - - firstZero = find(rV > 0, 1,'first'); - if isempty(firstZero) || firstZero < floor(0.1.*CTFSIZE(1)) - firstMin = floor(CTFSIZE(1)/2)-6; + elseif (doHalfGrid) + if ( preShiftedOrigin) + rV = Hqz(1:end,oY); else - [~,firstMin]=min(abs(rV(7:firstZero-1)-rV(8:firstZero))); - firstMin = firstMin + 6; + rV = Hqz(1:end,1); end - - - if ( preShiftedOrigin && ~calcOneD) - if doHalfGrid - freqMin = radialGrid(firstMin,ceil((CTFSIZE(1)+1)./2)); - maxRes = 0.5./radialGrid(ceil((CTFSIZE(1)+1)/2),1); - else - try - freqMin = radialGrid(ceil((CTFSIZE(1)+1)./2)+firstMin,ceil((CTFSIZE(2)+1)./2)); - catch - ceil((CTFSIZE(1)+1)./2) - end - maxRes = 0.5./radialGrid(1,ceil((CTFSIZE(2)+1)/2)); - end - + else + if ( preShiftedOrigin) + rV = Hqz(oX:end,oY); else - freqMin = radialGrid(firstMin,1); - freqZero = radialGrid(firstZero,1); + rV = Hqz(1:oX,1); + end + end + + firstZero = find(rV > 0, 1,'first'); + if isempty(firstZero) || firstZero < floor(0.1.*CTFSIZE(1)) + firstMin = floor(CTFSIZE(1)/2)-6; + else + [~,firstMin]=min(abs(rV(7:firstZero-1)-rV(8:firstZero))); + firstMin = firstMin + 6; + end + + + if ( preShiftedOrigin && ~calcOneD) + if doHalfGrid + freqMin = radialGrid(firstMin,ceil((CTFSIZE(1)+1)./2)); maxRes = 0.5./radialGrid(ceil((CTFSIZE(1)+1)/2),1); + else + try + freqMin = radialGrid(ceil((CTFSIZE(1)+1)./2)+firstMin,ceil((CTFSIZE(2)+1)./2)); + catch + ceil((CTFSIZE(1)+1)./2) + end + maxRes = 0.5./radialGrid(1,ceil((CTFSIZE(2)+1)/2)); end - - + + else + freqMin = radialGrid(firstMin,1); + freqZero = radialGrid(firstZero,1); + maxRes = 0.5./radialGrid(ceil((CTFSIZE(1)+1)/2),1); + end + + if (thisZero > 0) lowCut = 1./(0.1*freqMin+0.9*freqZero); if isempty(lowCut) @@ -224,14 +224,14 @@ ctfMask = ctfMask .* Hqz ./ (Hqz.^2 + 1./snr); snr = []; else - if (flgDampen) + if (flgDampen) envelope = (exp(-20.*(radialGrid.*(0.5/max(radialGrid(:)))).^1.25) +0.1)./1.1; else envelope = 1; end - + try - ctfMask = (envelope).*(sign(Hqz).*(radialGrid <= freqMin ).*abs(Hqz).^abs(Phase_Only) + (radialGrid > freqMin).*Hqz); + ctfMask = (envelope).*(sign(Hqz).*(radialGrid <= freqMin ).*abs(Hqz).^abs(Phase_Only) + (radialGrid > freqMin).*Hqz); catch size(rV) size(radialGrid) @@ -242,12 +242,12 @@ maxRes error('sfd') end - end - - -elseif Phase_Only == 1 - - ctfMask = sign(Hqz); + end + + +elseif Phase_Only == 1 + + ctfMask = sign(Hqz); else ctfMask = Hqz; diff --git a/ctf/BH_ctfCalcError.m b/ctf/BH_ctfCalcError.m index 241c09a8..3dc84bb1 100644 --- a/ctf/BH_ctfCalcError.m +++ b/ctf/BH_ctfCalcError.m @@ -1,6 +1,6 @@ - function [ ctfDepth ] = BH_ctfCalcError( pixelSize, Cs, Lambda, Defocus, ... - CTFSIZE, AMPCONT, ... - resCutOff,thicknessAng,dampeningMax,cycleNumber) +function [ ctfDepth ] = BH_ctfCalcError( pixelSize, Cs, Lambda, Defocus, ... + CTFSIZE, AMPCONT, ... + resCutOff,thicknessAng,dampeningMax,cycleNumber) %For a given resolution wanted, calculate the point where destructive %interference drops the CTF amplitude to some given threshold. % Detailed explanation goes here @@ -32,7 +32,7 @@ length([-maxDiff:0.1*maxDiff:maxDiff].*10^-10) for jDelDef = [-maxDiff:0.1*maxDiff:maxDiff].*10^-10 if isempty(ctf1) - ctf1 = BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1,1); + ctf1 = BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1,1); else ctf1 = ctf1 + BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1,1); end @@ -64,7 +64,7 @@ ctf1 = []; for jDelDef = [-maxDiff:0.1*maxDiff:maxDiff].*10^-10 if isempty(ctf1) - ctf1 = BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1); + ctf1 = BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1); else ctf1 = ctf1 + BH_ctfCalc(rad,Cs,Lambda,Defocus+jDelDef,CTFSIZE,AMPCONT,-1); end @@ -86,7 +86,7 @@ if ctfDepth < 0 fprintf('The optimal ctfDepth was not found\n'); fprintf('Inputs %3.3e pix %3.3e cs %3.3e wl %3.3e def %3.3e resTarget %3.3e tomoDepth\n',... - pixelSize*10^-10, Cs, Lambda, Defocus,resCutOff,thicknessAng); + pixelSize*10^-10, Cs, Lambda, Defocus,resCutOff,thicknessAng); ctfDepth = min(thicknessAng/30 * 10^-9,resCutOff(1) * 10 ^-8) elseif ctfDepth < 0.5*10e-9 fprintf('\n\nCapping ctfDepth to 5 nm from a calc %3.3f nm\n\n',ctfDepth*10^9); diff --git a/ctf/BH_ctf_Correct.m b/ctf/BH_ctf_Correct.m index d7bf27e3..291d3dba 100755 --- a/ctf/BH_ctf_Correct.m +++ b/ctf/BH_ctf_Correct.m @@ -2,7 +2,7 @@ %CTF correction for tilt series using general geometry. % Correct for the CTF using a local approach, similar to strip based % periodogram, but with tiles that are smaller allowing for arbitrary -% defocus gradients. +% defocus gradients. % % The full stack is corrected, st if only a small region is to be used, % it would be faster to have trimmed the stack. This should be done @@ -14,7 +14,7 @@ try load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR; + mapBackIter = subTomoMeta.currentTomoCPR; catch mapBackIter = 0; end @@ -48,9 +48,9 @@ pixelSize = emc.('PIXEL_SIZE'); !mkdir -p ctfStacks -try +try EMC_parpool(nGPUs); -catch +catch delete(gcp('nocreate')); EMC_parpool(nGPUs); end @@ -68,25 +68,25 @@ for iTilt = 1:length(ITER_LIST{iGPU}) - + STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - - + + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1); TLT = load(TLTNAME); fprintf('using TLT %s\n', TLTNAME); - + inputStack = sprintf('aliStacks/%s_ali%d.fixed',STACK_PRFX,mapBackIter+1); outputStack = sprintf('ctfStacks/%s_ali%d_ctf.fixed',STACK_PRFX,mapBackIter+1); iMrcObj = MRCImage(inputStack,0); - - - + + + iHeader = getHeader(iMrcObj); iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... - iHeader.cellDimensionY/iHeader.nY, ... - iHeader.cellDimensionZ/iHeader.nZ]; - + iHeader.cellDimensionY/iHeader.nY, ... + iHeader.cellDimensionZ/iHeader.nZ]; + d1 = iHeader.nX; d2 = iHeader.nY; nPrjs = iHeader.nZ; @@ -96,28 +96,28 @@ for iPrj = 1:nPrjs CS = TLT(iPrj,17); - WL = TLT(iPrj,18); - AMPCONT = TLT(iPrj,19); + WL = TLT(iPrj,18); + AMPCONT = TLT(iPrj,19); ddF = TLT(iPrj,12); dPhi = TLT(iPrj,13); D0 = TLT(iPrj,15); - - + + fastFTSize = BH_multi_iterator([d1,d2],'fourier2d'); padVal = BH_multi_padVal([d1,d2],fastFTSize); trimVal = BH_multi_padVal(fastFTSize,[d1,d2]); - - + + initImg = randn(fastFTSize,'single','gpuArray'); f = FFT(initImg); - + ctf = CTF(fastFTSize,pixelSize*10^10,'GPU'); maxZ = 500; maxEval = cosd(TLT(iPrj,4)).*(d1/2) + maxZ./2*abs(sind(TLT(iPrj,4))); oX = ceil((d1+1)./2); oY = ceil((d2+1)./2); iEvalMask = floor(oX-maxEval):ceil(oX+maxEval); - + STRIPWIDTH = 512; STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); % take at least 1200 Ang & include the taper if equal to STRIPWIDTH @@ -126,79 +126,79 @@ %fprintf('stripwidth tilesize %d %d\n',STRIPWIDTH,tileSize); incLow = ceil(tileSize./2); incTop = tileSize - incLow; - - + + iProjection = BH_padZeros3d(getVolume(iMrcObj,[-1],[-1],TLT(iPrj,23),'keep'), ... - padVal(1,:),padVal(2,:),'GPU','singleTaper'); - + padVal(1,:),padVal(2,:),'GPU','singleTaper'); + correctedPrj = zeros([d1,d2],'single','gpuArray'); iProjectionFT = f.fwdFFT(iProjection); - - + + stripDefocusOffset = floor(STRIPWIDTH/2); for i = 1: STRIPWIDTH : d1 - + if (i+tileSize-1) < d1 - endIDX = (i+tileSize-1); - endCUT = i + STRIPWIDTH - 1 + 7; - trimmedSIZE = STRIPWIDTH; + endIDX = (i+tileSize-1); + endCUT = i + STRIPWIDTH - 1 + 7; + trimmedSIZE = STRIPWIDTH; elseif any(ismember(i:d1,iEvalMask)) - endIDX = d1; - endCUT = d1; - trimmedSIZE = endCUT-i+1 -7; - end - + endIDX = d1; + endCUT = d1; + trimmedSIZE = endCUT-i+1 -7; + end + % The eval mask condition can be replaced once the per tomo condition % is trusted. if any(ismember(i:endIDX,iEvalMask)) - - + + DF = D0 +(i + stripDefocusOffset - oX)*pixelSize*-1.*tand(TLT(iPrj,4)); - - + + if ~( isempty(DF) ) - + iDefocus = [DF - ddF, DF + ddF, dPhi]; - - if pixelSize < 2.0e-10 - % use double precision - this is not enabled, but needs to be - - % requires changes to radial grid as well. - ctf.new_img(iDefocus,CS,WL,AMPCONT,-1,-1); - else - ctf.new_img(iDefocus,CS,WL,AMPCONT,-1); - end - - + + if pixelSize < 2.0e-10 + % use double precision - this is not enabled, but needs to be - + % requires changes to radial grid as well. + ctf.new_img(iDefocus,CS,WL,AMPCONT,-1,-1); + else + ctf.new_img(iDefocus,CS,WL,AMPCONT,-1); + end + + tile = ctf.multiply(iProjectionFT); - + tile = BH_padZeros3d(real(f.invFFT(tile,2)), ... - trimVal(1,:),trimVal(2,:),'GPU','single'); - + trimVal(1,:),trimVal(2,:),'GPU','single'); + % trim prior to pulling off gpu to minimize xfer else - + % No particles in this strip, so just replace with simple inversion % to keep the global image statistics ~ correct. - + tile = -1.*BH_padZeros3d(iProjection, trimVal(1,:),trimVal(2,:),'GPU','single'); - + end - - %correctedStack(i + 7 : endCUT,:,TLT(iPrj,1)) = ... - % gather(tile(8:trimmedSIZE+7,:)); - - correctedPrj(i:endIDX,:) = tile(i:endIDX,:); - + + %correctedStack(i + 7 : endCUT,:,TLT(iPrj,1)) = ... + % gather(tile(8:trimmedSIZE+7,:)); + + correctedPrj(i:endIDX,:) = tile(i:endIDX,:); + else - %fprintf('ignoring strip centered on %d for prj %d',i,TLT(iPrj,1)); + %fprintf('ignoring strip centered on %d for prj %d',i,TLT(iPrj,1)); end end % end loop over strips correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj); - + end % end loop over prjs SAVE_IMG(MRCImage(correctedStack), outputStack,iPixelHeader); - + end % end loop over tilt-series end % end parfor diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index e46a10b2..3d74cc25 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -23,7 +23,7 @@ shiftDefocusOrigin = 1; tiltStart = 1; -try +try flgEraseBeads_aferCTF = emc.('erase_beads_after_ctf'); catch flgEraseBeads_aferCTF = false; % If false they SHOULD be erased in ctf estimate/update, but since the user could change parameter, include here. @@ -45,7 +45,7 @@ else super_sample = ''; end - + catch super_sample = ''; end @@ -93,7 +93,7 @@ loadSubTomoMeta = false; flgWhitenPS = [0,0,1.0]; if (bh_global_turn_on_phase_plate(1)) - fprintf('WARNING: the filtered tomogram should only be used for viz, not template matching.'); + fprintf('WARNING: the filtered tomogram should only be used for viz, not template matching.'); end else error('Extra argument to ctf 3d should be a vector [THICKNESS, BINNING] tiltN, or a string templateSearch'); @@ -101,11 +101,11 @@ else % Default to zero for normal use if isempty(bh_global_turn_on_phase_plate) - bh_global_turn_on_phase_plate = 0; + bh_global_turn_on_phase_plate = 0; end end -try +try % -1, whiten before ctf, 1 whiten after - test both. usr_flgWhitenPS = emc.('whitenPS'); if (numel(usr_flgWhitenPS) == 3) @@ -117,8 +117,8 @@ end if (bh_global_turn_on_phase_plate(1) && any(emc.whitenPS)) - fprintf('WARNING: phakePhasePlate and whitening are conflicting preocesses. Turning off whitening.\n') - emc.whitenPS = [0,0,0]; + fprintf('WARNING: phakePhasePlate and whitening are conflicting preocesses. Turning off whitening.\n') + emc.whitenPS = [0,0,0]; end @@ -154,7 +154,7 @@ tmpCache= emc.('fastScratchDisk'); -if strcmpi(tmpCache, 'ram') +if strcmpi(tmpCache, 'ram') if isempty(getenv('EMC_CACHE_MEM')) fprintf('Did not find a variable for EMC_CACHE_MEM\nSkipping ram\n'); tmpCache= ''; @@ -172,14 +172,14 @@ % Check to make sure it even exists if isempty(dir(tmpCache)) - fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); - tmpCache = ''; + fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); + tmpCache = ''; end reconScaling = 1; if isempty(tmpCache) - tmpCache='cache'; + tmpCache='cache'; flgCleanCache = 0; CWD=''; else @@ -191,8 +191,8 @@ % This means the final character was a slash, strip it tmpCache = sprintf('%scache',tmpCache); %strjoin(slashCheck(1:end-1),'/'); else - tmpCache = sprintf('%s/cache',tmpCache); - end + tmpCache = sprintf('%s/cache',tmpCache); + end end % Incase this is launched form another process (synthetic mapback for example, make one level lower in the cache @@ -206,7 +206,7 @@ load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; masterTM = subTomoMeta; clear subTomoMeta - CYCLE = masterTM.currentCycle; + CYCLE = masterTM.currentCycle; else mapBackIter = 0; CYCLE = 0; @@ -242,17 +242,17 @@ flg2dCTF = emc.('flg2dCTF'); catch flg2dCTF = 0; -end +end try - % Part of the experiment with template matching using higher res info, also - % allow for a median filter post CTF correction, pre reconstruction to + % Part of the experiment with template matching using higher res info, also + % allow for a median filter post CTF correction, pre reconstruction to % further denoise prior to template matching. flgMedianFilter = emc.('ctfMedianFilter'); catch flgMedianFilter = 0; end - + % ctf3dDepth=emc.('defocusErrorEst') %mean in case cones. @@ -276,14 +276,14 @@ else % For template search samplingRate = emc.('Tmp_samplingRate'); - try + try resTarget = emc.('lowResCut'); catch resTarget = 12; end end - + end try @@ -347,31 +347,31 @@ getCoords = dir('recon/*.coords'); nTilts = length(getCoords); if (nTilts == 0) - error('Did not find any tomogram coordinates in recon/TS*.coords'); + error('Did not find any tomogram coordinates in recon/TS*.coords'); end tiltList = cell(nTilts,1); nTomosTotal = 0; nTomosPerTilt = cell(nTilts,1); recGeom = cell(nTilts,1); for iStack = 1:nTilts - [ recGeom{iStack}, tiltName, nTomosPossible] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name) ); + [ recGeom{iStack}, tiltName, nTomosPossible] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name) ); nTomosTotal = nTomosTotal + nTomosPossible; nTomosPerTilt{iStack} = nTomosPossible; - tiltList{iStack} = tiltName; + tiltList{iStack} = tiltName; end tomoList = cell(nTomosTotal,1); nTomosAdd = 0; for iStack = 1:nTilts - for iTomo = 1:nTomosPerTilt{iStack} + for iTomo = 1:nTomosPerTilt{iStack} tomoList{nTomosAdd+1} = sprintf('%s_%d',tiltName,iTomo); nTomosAdd = nTomosAdd +1; end end - + end else - [tiltList,nTilts] = BH_returnIncludedTilts(masterTM.mapBackGeometry); + [tiltList,nTilts] = BH_returnIncludedTilts(masterTM.mapBackGeometry); tomoList = fieldnames(masterTM.mapBackGeometry.tomoName); end @@ -385,13 +385,13 @@ iterList{gpuList(iGPU)}; end try - EMC_parpool(nGPUs) + EMC_parpool(nGPUs) catch delete(gcp('nocreate')) EMC_parpool(nGPUs) end - + parfor iGPU = 1:nGPUs for iTilt = iterList{gpuList(iGPU)} @@ -404,7 +404,7 @@ TLT = load(TLTNAME); fprintf('using TLT %s\n', TLTNAME); - + % Get all the tomogram names that belong to a given tilt-series. if (~recWithoutMat) nTomos = 0; @@ -414,61 +414,61 @@ iTomoList{nTomos+1} = tomoList{iTomo}; nTomos = nTomos + 1; end - % The order of tomo num could be off but only if all are present do we - % skip. - if (bh_global_turn_on_phase_plate(1)) - filtered = '_filtered'; - else - filtered = ''; - end - checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', ... - tiltList{iTilt},iTomo,samplingRate,filtered); - if exist(checkRecon, 'file') - fprintf('found %s to already exits\n',checkRecon); - alreadyMade = alreadyMade +1; - end - + % The order of tomo num could be off but only if all are present do we + % skip. + if (bh_global_turn_on_phase_plate(1)) + filtered = '_filtered'; + else + filtered = ''; + end + checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', ... + tiltList{iTilt},iTomo,samplingRate,filtered); + if exist(checkRecon, 'file') + fprintf('found %s to already exits\n',checkRecon); + alreadyMade = alreadyMade +1; + end + end - + if alreadyMade == nTomos fprintf('All tomos 1-%d found to exist for tilt-series %s\n',nTomos,tiltList{iTilt}); continue end end - - - + + + preBinStacks(TLT, tiltList{iTilt}, mapBackIter,1,... - samplingRate,... - PosControl2d,... - tiltWeight,... - flgMedianFilter); - - end + samplingRate,... + PosControl2d,... + tiltWeight,... + flgMedianFilter); + + end end -% All data is handled through disk i/o so everything unique created in the +% All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. parfor iGPU = 1:nGPUs% -%for iGPU = 1:nGPUs + %for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); - % Loop over each tilt + % Loop over each tilt for iTilt = iterList{gpuList(iGPU)} - + if (recWithoutMat) if (loadSubTomoMeta) nTomos = 1; else - % templaterch + % templaterch nTomos = nTomosPerTilt{iTilt}; iCoords = recGeom{iTilt}; end else nTomos = masterTM.mapBackGeometry.(tiltList{iTilt}).nTomos; iCoords = masterTM.mapBackGeometry.(tiltList{iTilt}).coords; - % FIXME + % FIXME end if (recWithoutMat && ~loadSubTomoMeta) || ~recWithoutMat @@ -478,15 +478,15 @@ iCoords(:,3) = iCoords(:,3) - (diff(floor(iCoords(:,2:3)),1,2)+1 - floor(targetSizeY./samplingRate)); end iTomoList = cell(nTomos,1); - - + + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); TLT = load(TLTNAME); fprintf('using TLT %s\n', TLTNAME); - - + + if (~recWithoutMat) % Get all the tomogram names that belong to a given tilt-series. nTomos = 0; @@ -496,87 +496,87 @@ iTomoList{nTomos+1} = tomoList{iTomo}; nTomos = nTomos + 1; end - % The order of tomo num could be off but only if all are present do we - % skip. - if (bh_global_turn_on_phase_plate(1)) - filtered = '_filtered'; - else - filtered = ''; - end - checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', ... - tiltList{iTilt},iTomo,samplingRate,filtered); - if exist(checkRecon, 'file') - fprintf('found %s to already exits\n',checkRecon); - alreadyMade = alreadyMade +1; - end - + % The order of tomo num could be off but only if all are present do we + % skip. + if (bh_global_turn_on_phase_plate(1)) + filtered = '_filtered'; + else + filtered = ''; + end + checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', ... + tiltList{iTilt},iTomo,samplingRate,filtered); + if exist(checkRecon, 'file') + fprintf('found %s to already exits\n',checkRecon); + alreadyMade = alreadyMade +1; + end + end - + if alreadyMade == nTomos fprintf('All tomos 1-%d found to exist for tilt-series %s\n',nTomos,tiltList{iTilt}); continue end end - - - if samplingRate > 1 - fullStack = sprintf('%aliStacks/%s_ali%d.fixed', ... - tiltList{iTilt},mapBackIter+1); - inputStack = sprintf('cache/%s_ali%d_bin%d.fixed',... - tiltList{iTilt},mapBackIter+1,samplingRate); - if ~exist(inputStack, 'file') - % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s > /dev/null',samplingRate,fullStack,inputStack); - % % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s ',samplingRate,fullStack,inputStack); - % - % system(binCMD); - BH_multi_loadOrBin(fullStack,-1.*samplingRate,2); - - end - else - inputStack = sprintf('aliStacks/%s_ali%d.fixed',... - tiltList{iTilt},mapBackIter+1); + + + if samplingRate > 1 + fullStack = sprintf('%aliStacks/%s_ali%d.fixed', ... + tiltList{iTilt},mapBackIter+1); + inputStack = sprintf('cache/%s_ali%d_bin%d.fixed',... + tiltList{iTilt},mapBackIter+1,samplingRate); + if ~exist(inputStack, 'file') + % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s > /dev/null',samplingRate,fullStack,inputStack); + % % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s ',samplingRate,fullStack,inputStack); + % + % system(binCMD); + BH_multi_loadOrBin(fullStack,-1.*samplingRate,2); + end - -% system(sprintf('header %s',inputStack)); - % iHeader = MRCImage(inputStack,0); - % STACK = gpuArray(single(getVolume(iHeader))); - - maskedStack = single(getVolume(MRCImage(inputStack))); - - if (recWithoutMat) - if (reconstructionParameters(1) && loadSubTomoMeta) - NX = size(maskedStack,1); - NY = size(maskedStack,2); - -% NY = size(maskedStack,2)-1; - NZ = floor(reconstructionParameters(1)) - maxZ = NZ; - iCoords = [NX,0,NY-1,NZ,0,0]; - tomoNumber = 1; - else - [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... - samplingRate, cycleNumber,... - 0,1); - end + else + inputStack = sprintf('aliStacks/%s_ali%d.fixed',... + tiltList{iTilt},mapBackIter+1); + end + + % system(sprintf('header %s',inputStack)); + % iHeader = MRCImage(inputStack,0); + % STACK = gpuArray(single(getVolume(iHeader))); + + maskedStack = single(getVolume(MRCImage(inputStack))); + + if (recWithoutMat) + if (reconstructionParameters(1) && loadSubTomoMeta) + NX = size(maskedStack,1); + NY = size(maskedStack,2); + + % NY = size(maskedStack,2)-1; + NZ = floor(reconstructionParameters(1)) + maxZ = NZ; + iCoords = [NX,0,NY-1,NZ,0,0]; + tomoNumber = 1; else - [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... - samplingRate, cycleNumber,... - 0,1); + [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... + iTomoList,nTomos, pixelSize, ... + samplingRate, cycleNumber,... + 0,1); end - + else + [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... + iTomoList,nTomos, pixelSize, ... + samplingRate, cycleNumber,... + 0,1); + end + if ( flg2dCTF || recWithoutMat && loadSubTomoMeta) nSections = 1; ctf3dDepth = maxZ * 10 ^ -9; else - dampeningMax = 0.90; + dampeningMax = 0.90; [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... - TLT(1,17),TLT(1,18),TLT(1,15), ... - 2048, TLT(1,19), ... - resTarget,maxZ*10, ... - dampeningMax,CYCLE); + TLT(1,17),TLT(1,18),TLT(1,15), ... + 2048, TLT(1,19), ... + resTarget,maxZ*10, ... + dampeningMax,CYCLE); fprintf('\n\nCalculated a ctfDepth of %2.2f nm for %s\n\n',ctf3dDepth*10^9,tiltList{iTilt}); if (ctf3dDepth > max_ctf3dDepth) ctf3dDepth = max_ctf3dDepth; @@ -592,25 +592,25 @@ end fprintf('with %3.3f nm sections, correcting %d tilt-series\n',ctf3dDepth*10^9,nSections); - % For each tomo create a list of slices that are to be reconstructed + % For each tomo create a list of slices that are to be reconstructed % for every section section. [ sectionList ] = calcTomoSections(iCoords, tomoNumber,pixelSize, ... - nSections,tiltList{iTilt}, ctf3dDepth); - - + nSections,tiltList{iTilt}, ctf3dDepth); + + if (recWithoutMat) - avgZ = 0; - surfaceFit = 0; - + avgZ = 0; + surfaceFit = 0; + else - [ avgZ, maxZ, tomoNumber, surfaceFit ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... - samplingRate, cycleNumber,... - sectionList,0); - - + [ avgZ, maxZ, tomoNumber, surfaceFit ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... + iTomoList,nTomos, pixelSize, ... + samplingRate, cycleNumber,... + sectionList,0); + + end if ( shiftDefocusOrigin ) @@ -620,23 +620,23 @@ fprintf('Using sample origin as the defocus origin\n'); fprintf('If you want to use the COM of subTomos, set shiftDefocusToSubTomoCOM=1\n'); end - - - - + + + + % Correct a tilt series for earch section which requires writing each to % disk for use of IMOD. if (mapBackIter) tiltErrorFile = sprintf('mapBack%d/%s_ali%d_ctf.beamTiltError', ... - mapBackIter,tiltList{iTilt}, mapBackIter); - try - tiltError = load(tiltErrorFile); + mapBackIter,tiltList{iTilt}, mapBackIter); + try + tiltError = load(tiltErrorFile); if numel(tiltError) ~= 1 error('tiltError should be a single number in degrees.\n'); else fprintf('\nUsing %f degrees for beam tilt error.\n',tiltError) - end + end catch fprintf('\nTiltErrorFile %s not found.\n', tiltErrorFile); @@ -644,19 +644,19 @@ tiltError = 0; end - + else tiltError = 0; end for iSection = 1:nSections - - + + defFitFull = ''; preCombDefocus = 0; if (mapBackIter) defFitFull = sprintf('mapBack%d/%s_ali%d_ctf.defFidFull',mapBackIter, ... - tiltList{iTilt},mapBackIter); + tiltList{iTilt},mapBackIter); if exist(defFitFull,'file') preCombDefocus = load(defFitFull); fprintf('3dCTF using pre calc combined per tilt defocus %s\n',defFitFull); @@ -665,57 +665,57 @@ end end - - + + if (PosControl2d) correctedStack = maskedStack; else - - - % I would have thought the global would be recognized, but it looks - % like there is something odd about its use with a parfor loop - % FIXME, when setting up the iterator, make clean copies for each - % worker that are local in scope.e - - - [ correctedStack ] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,pixelSize,maskedStack,... - maxZ*10/pixelSize,flgDampenAliasedFrequencies,... - preCombDefocus,samplingRate,... - applyExposureFilter,surfaceFit,... - useSurfaceFit,invertDose,... - bh_global_turn_on_phase_plate,... - filterProjectionsForTomoCPRBackground,... - emc.whitenPS); + + + % I would have thought the global would be recognized, but it looks + % like there is something odd about its use with a parfor loop + % FIXME, when setting up the iterator, make clean copies for each + % worker that are local in scope.e + + + [ correctedStack ] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... + avgZ,TLT,pixelSize,maskedStack,... + maxZ*10/pixelSize,flgDampenAliasedFrequencies,... + preCombDefocus,samplingRate,... + applyExposureFilter,surfaceFit,... + useSurfaceFit,invertDose,... + bh_global_turn_on_phase_plate,... + filterProjectionsForTomoCPRBackground,... + emc.whitenPS); end % Write out the stack to the cache directory as a tmp file - + if (flgEraseBeads_aferCTF) scalePixelsBy = samplingRate; correctedStack = BH_eraseBeads(correctedStack,eraseRadius, tiltList{iTilt}, scalePixelsBy,mapBackIter,sortrows(TLT,1)); end - - + + outputStack = sprintf('%s/%s_ali%d_%d.fixed', ... - tmpCache,tiltList{iTilt},mapBackIter+1,iSection) + tmpCache,tiltList{iTilt},mapBackIter+1,iSection) SAVE_IMG(correctedStack,outputStack,pixelSize); correctedStack = []; - - % Loop over tomos reconstructing section and appending a file to + + % Loop over tomos reconstructing section and appending a file to for iT = 1:nTomos - - thisTomo = tomoNumber(iT); - + + thisTomo = tomoNumber(iT); + if any(sectionList{iT}(iSection,:)+9999) - - + + reconName = sprintf('%s/%s_ali%d_%d_%d.rec', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - - + tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); + + if (loadSubTomoMeta) - if (recWithoutMat) + if (recWithoutMat) TA = sortrows(masterTM.tiltGeometry.(tomoList{1}),1); else TA = sortrows(masterTM.tiltGeometry.(sprintf('%s_%d',tiltList{iTilt},thisTomo)),1); @@ -726,26 +726,26 @@ % FIXME: I don't think this block should work, it should only be the tilt angles! error('THis block should not be reached.') TA = load(sprintf('%smapBack%d/%s_ali%d_ctf.tlt',CWD,mapBackIter,tiltList{iTilt},... - mapBackIter)); + mapBackIter)); else TA = load(sprintf('%sfixedStacks/%s.tlt',CWD,tiltList{iTilt})); - end + end end - + rawTLT = sprintf('cache/%s_%d.rawtlt',tiltList{iTilt},thisTomo); rawTLT_file = fopen(rawTLT, 'w'); fprintf(rawTLT_file,'%f\n', TA'); fclose(rawTLT_file); if (mapBackIter) - + LOCAL = sprintf('%smapBack%d/%s_ali%d_ctf.local',CWD,mapBackIter,tiltList{iTilt}, ... - mapBackIter); - else + mapBackIter); + else LOCAL = sprintf('%sfixedStacks/%s.local',CWD,tiltList{iTilt}); end - + fprintf('Local file %s\n',LOCAL); if exist(LOCAL,'file') @@ -754,34 +754,34 @@ fprintf('Did not find local alignment information at %s\n',LOCAL); flgLocal = 0; end - + % hangover from slab padding, remove later. - padRec = 0; - + padRec = 0; + nTiltWorkers = 2; nTotalSlices = (iCoords(thisTomo,3)-iCoords(thisTomo,2)+1); tiltChunkSize = ceil(nTotalSlices/nTiltWorkers); tiltChunks = iCoords(thisTomo,2):tiltChunkSize:iCoords(thisTomo,3); tiltChunks(end) = iCoords(thisTomo,3); totalSlices = [tiltChunks(1),tiltChunks(end)]; - - - + + + rCMD = sprintf(['tilt %s %s -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... - '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... - super_sample, expand_lines, ... - outputStack, reconName, rawTLT, gpuList(iGPU), ... - iCoords(thisTomo,1),floor(sectionList{iT}(iSection,5))+2*padRec,... - iCoords(thisTomo,5),sectionList{iT}(iSection,6)); - - - - % Explicitly set Radial to Nyquist + '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... + super_sample, expand_lines, ... + outputStack, reconName, rawTLT, gpuList(iGPU), ... + iCoords(thisTomo,1),floor(sectionList{iT}(iSection,5))+2*padRec,... + iCoords(thisTomo,5),sectionList{iT}(iSection,6)); + + + + % Explicitly set Radial to Nyquist if (flgLocal) rCMD = [rCMD sprintf('-LOCALFILE %s -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d',LOCAL,reconScaling)]; else rCMD = [rCMD sprintf('-RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d',reconScaling)]; - end + end system(sprintf('rm -f %s.sh',reconName)); @@ -795,14 +795,14 @@ iShift = 0; end % /dev/null fprintf(recScript,'%s -SLICE %d,%d -TOTALSLICES %d,%d > /dev/null &\n',rCMD, ... - tiltChunks(iRecSec),... - tiltChunks(iRecSec+1)-iShift,... - totalSlices); + tiltChunks(iRecSec),... + tiltChunks(iRecSec+1)-iShift,... + totalSlices); end fprintf(recScript,'\n\nwait\n\n'); fclose(recScript); system(sprintf('chmod a=wrx %s.sh',reconName)); - + [recError,~] = system(sprintf('%s.sh > /dev/null ',reconName)); % /dev/null if (recError) system(sprintf('%s.sh',reconName)); @@ -812,83 +812,83 @@ % monitor. For symmetrical padding this doesn't matter, but keep % in mind. /dev/null trimCMD = sprintf('trimvol -rx -y %d,%d %s.TMPPAD %s > /dev/null ' , ... - padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName); -% % % trimCMD = sprintf('newstack -fromone -secs %d-%d %s.TMPPAD %s > /dev/null', ... -% % % padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) + padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName); + % % % trimCMD = sprintf('newstack -fromone -secs %d-%d %s.TMPPAD %s > /dev/null', ... + % % % padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) [msg,~]= system(trimCMD); if (msg) - fprintf('%d from trimCMD\n',msg) + fprintf('%d from trimCMD\n',msg) trimCMDPrintError = sprintf('trimvol -rx -y %d,%d %s.TMPPAD %s', ... - padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) -% % % trimCMDPrintError = sprintf('newstack -fromone -secs %d-%d %s.TMPPAD %s', ... -% % % padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) + padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) + % % % trimCMDPrintError = sprintf('newstack -fromone -secs %d-%d %s.TMPPAD %s', ... + % % % padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) system(trimCMDPrintError); - end + end system(sprintf('rm %s.TMPPAD', reconName)); - % fprintf([trimCMD ' \n']) - - + % fprintf([trimCMD ' \n']) + + end end % end loop over tomos for this section - + system(sprintf('rm %s',outputStack)); - - end % end loop over sections - - deltaZ = []; - evalMask = []; - maskedStack = []; + end % end loop over sections + + deltaZ = []; + evalMask = []; + maskedStack = []; + for iT = 1:nTomos - thisTomo = tomoNumber(iT); + thisTomo = tomoNumber(iT); if (bh_global_turn_on_phase_plate(1)) reconNameFull = sprintf('cache/%s_%d_bin%d_filtered.rec', ... - tiltList{iTilt},thisTomo,samplingRate); + tiltList{iTilt},thisTomo,samplingRate); elseif reconstructionParameters(1) reconNameFull = sprintf('cache/%s_%d_bin%d_backgroundEst.rec', ... - tiltList{iTilt},thisTomo,samplingRate); + tiltList{iTilt},thisTomo,samplingRate); else reconNameFull = sprintf('cache/%s_%d_bin%d.rec', ... - tiltList{iTilt},thisTomo,samplingRate); + tiltList{iTilt},thisTomo,samplingRate); end - + recCMD = 'newstack -fromone'; for iSection = 1:nSections reconName = sprintf('%s/%s_ali%d_%d_%d.rec', ... - tmpCache, tiltList{iTilt},mapBackIter+1,thisTomo,iSection); + tmpCache, tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - if any(sectionList{iT}(iSection,:)+9999) + if any(sectionList{iT}(iSection,:)+9999) recCMD = [recCMD,sprintf(' -secs 1-%d %s', ... - floor(sectionList{iT}(iSection,5)), ... - reconName)]; + floor(sectionList{iT}(iSection,5)), ... + reconName)]; else - fprintf('no info for section %d for tomo %d\n',iSection,thisTomo); + fprintf('no info for section %d for tomo %d\n',iSection,thisTomo); end end system([recCMD, sprintf(' %s > /dev/null ',reconNameFull)]); %/dev/null - + for iSection = 1:nSections cleanUp3 = sprintf('rm %s/%s_ali%d_%d_%d.rec', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); + tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); system(cleanUp3); cleanUp4 = sprintf('rm %s/%s_ali%d_%d_%d.rec.sh', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); + tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); system(cleanUp4); end - - + + end % end of recombination loop - - maskedStack = []; - - end % end of loop over tilt-series + + maskedStack = []; + + end % end of loop over tilt-series end % end of parfor over gpus if (flgCleanCache) @@ -896,7 +896,7 @@ checkDir = dir(tmpCache); if isempty(checkDir) fprintf('not removing the temp cache because it did not eval with dir\n'); - else + else cleanItUp = sprintf('rm %s/*',tmpCache); system(cleanItUp); end @@ -905,14 +905,14 @@ end function [] = preBinStacks(TLT, STACK_PRFX, mapBackIter,usableArea,... - samplingRate,... - PosControl2d,... - tiltWeight,... - flgMedianFilter) - - - -if (PosControl2d) + samplingRate,... + PosControl2d,... + tiltWeight,... + flgMedianFilter) + + + +if (PosControl2d) prefix = 'ctf'; suffix = '_ctf'; else @@ -922,9 +922,9 @@ fullStack = sprintf('%sStacks/%s_ali%d%s.fixed', ... - prefix,STACK_PRFX,mapBackIter+1,suffix); + prefix,STACK_PRFX,mapBackIter+1,suffix); inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed',... - STACK_PRFX,mapBackIter+1,suffix,samplingRate); + STACK_PRFX,mapBackIter+1,suffix,samplingRate); if ~exist(inputStack, 'file') BH_multi_loadOrBin(fullStack,-1.*samplingRate,2,flgMedianFilter); end @@ -934,7 +934,7 @@ function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, pixelSize,... - nSections,tiltName, ctf3Depth) + nSections,tiltName, ctf3Depth) nTomos = length(tomoNumber); sectionList = cell(nTomos,1); @@ -952,20 +952,20 @@ for iT = 1:length(tomoNumber) iTomo = tomoNumber(iT); % Origin + originshift - + -1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6), reconRange = floor([-1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6),0]); reconRange(2) = reconRange(1) + iCoords(iTomo,4) - 1; nZ = 1; flgFirstSec = 1; - + for iSection = 1:nSections sectionCenter = ((nSections-1)/-2+(iSection-1))*(nSec-1); % Check that sectionCenter is within range if sectionCenter + halfSec < reconRange(1) || ... - sectionCenter - halfSec > reconRange(2) + sectionCenter - halfSec > reconRange(2) sectionList{iT}(iSection,:) = -9999; else @@ -975,10 +975,10 @@ sectionList{iT}(iSection,1) = sectionList{iT}(iSection,1) +1; end elseif (sectionCenter - halfSec < 0) - sectionList{iT}(iSection,1) = max(sectionCenter - halfSec,reconRange(1)); + sectionList{iT}(iSection,1) = max(sectionCenter - halfSec,reconRange(1)); if sectionList{iT}(iSection,1) ~= reconRange(1) sectionList{iT}(iSection,1) = sectionList{iT}(iSection,1) +1; - end + end else sectionList{iT}(iSection,1) = max(-halfSec,reconRange(1)); end @@ -987,9 +987,9 @@ sectionList{iT}(iSection,2) = min(sectionCenter + halfSec,reconRange(2)); elseif (sectionCenter - halfSec < 0) sectionList{iT}(iSection,2) = min(sectionCenter + halfSec ,reconRange(2)); - else + else sectionList{iT}(iSection,2) = min(halfSec,reconRange(2)); - end + end % Check that first section starts in the correct place. If a small error, @@ -1001,7 +1001,7 @@ fprintf('\n\nShifting first section %s_n%d\n\n',tiltName,iTomo); else error('section start %d is too far off from expected %d\n', ... - sectionList{iT}(iSection,1), reconRange(1)) + sectionList{iT}(iSection,1), reconRange(1)) end end flgFirstSec = 0; @@ -1012,24 +1012,24 @@ sectionList{iT}(iSection,4) = nZ + secZ; sectionList{iT}(iSection,5) = secZ +1; sectionList{iT}(iSection,6) = (secZ+1)./2 + sectionList{iT}(iSection,1); - nZ = nZ + secZ + 1; + nZ = nZ + secZ + 1; end - - + + % not a good solution, but not sure just yet why I'm getting some occasionally % weird results. if sectionList{iT}(iSection,5) < 3 sectionList{iT}(iSection,:) = -9999; end end % end loop over sections - + % TroubleShoot tSHT = fopen(sprintf('.tblSht_%s_i%d.txt',tiltName,iTomo),'w'); fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords(iTomo,:)'); fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', sectionList{iT}'); fclose(tSHT); end % end loop over tomos - + end @@ -1038,27 +1038,27 @@ function [correctedStack] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,pixelSize,maskedStack,... - maxZ,flgDampenAliasedFrequencies,... - preCombDefocus,samplingRate,... - applyExposureFilter,surfaceFit,... - useSurfaceFit,invertDose, ... - phakePhasePlate, ... - filterProjectionsForTomoCPRBackground,... - flgWhitenPS) -% Correct in strips which is more expensive but (hopefully) more accurate. + avgZ,TLT,pixelSize,maskedStack,... + maxZ,flgDampenAliasedFrequencies,... + preCombDefocus,samplingRate,... + applyExposureFilter,surfaceFit,... + useSurfaceFit,invertDose, ... + phakePhasePlate, ... + filterProjectionsForTomoCPRBackground,... + flgWhitenPS) +% Correct in strips which is more expensive but (hopefully) more accurate. % For sections with too few subTomos to fit, fall back if isa(surfaceFit,'cell') - surfaceFit = surfaceFit{iSection}; + surfaceFit = surfaceFit{iSection}; if ~isa(surfaceFit,'sfit') fprintf('Warning, surfaceFit is not an sfit object\n'); useSurfaceFit = false; end -else +else useSurfaceFit = false; end @@ -1106,26 +1106,26 @@ %end if ( flgDampenAlias ) % Calculate a centered grid b/c real space convolution -[radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... - 'Cylindrical','GPU', ... - {'none'},1,1,0); + [radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... + 'Cylindrical','GPU', ... + {'none'},1,1,0); else -[radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... - 'Cylindrical','GPU', ... - {'none'},1,0,0); + [radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... + 'Cylindrical','GPU', ... + {'none'},1,0,0); end radialGrid = {radialGrid./PIXEL_SIZE,0,phi}; phi = []; fprintf('%f %f\n',filterProjectionsForTomoCPRBackground,pixelSize); if (filterProjectionsForTomoCPRBackground ~= 0) - bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',pixelSize); + bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',pixelSize); else - bpFilter = 1; + bpFilter = 1; end for iPrj = 1:nPrjs - + maxEval = cosd(TLT(iPrj,4)).*(d1/2) + maxZ./2*abs(sind(TLT(iPrj,4))); oX = ceil((d1+1)./2); oY = ceil((d2+1)./2); @@ -1139,8 +1139,8 @@ iExposureFilter = iExposureFilter .* bpFilter; - - + + STRIPWIDTH = min(floor((0.5*ctf3dDepth/PIXEL_SIZE)/abs(tand(TLT(iPrj,4)))),512); STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); @@ -1155,13 +1155,13 @@ ddF = TLT(iPrj,12); dPhi = TLT(iPrj,13); D0 = TLT(iPrj,15); - %TLT(iPrj,16); - - + %TLT(iPrj,16); + + padVal = BH_multi_padVal([d1,d2],fastFTSize); trimVal = BH_multi_padVal(fastFTSize,[d1,d2]); - + iProjection = BH_padZeros3d(maskedStack(:,:,TLT(iPrj,1)),padVal(1,:),padVal(2,:),'GPU','singleTaper'); iProjectionFT = fftn(iProjection).*iExposureFilter; clear iExposureFilter correctedPrj = zeros([d1,d2],'single','gpuArray'); @@ -1169,28 +1169,28 @@ % Gridvectors for the specimen plane [rX,rY,~] = BH_multi_gridCoordinates([d1,d2],'Cartesian','GPU',{'none'},0,1,0); % Assuming the plane fit is from the origin as it is. - - - + + + if (useSurfaceFit) - %rZ = (surfaceFit.p00 + surfaceFit.p10.*(rX+oX)) + surfaceFit.p01.*(rY+oY); - try - rZ = feval(surfaceFit,rX,rY); - catch - d1 - d2 - rX - rY - surfaceFit - error('surface fit failed'); - end + %rZ = (surfaceFit.p00 + surfaceFit.p10.*(rX+oX)) + surfaceFit.p01.*(rY+oY); + try + rZ = feval(surfaceFit,rX,rY); + catch + d1 + d2 + rX + rY + surfaceFit + error('surface fit failed'); + end else rZ = zeros([d1,d2],'single','gpuArray'); end - - + + full_defocusOffset = ((defocusOffset.*cosd(TLT(iPrj,4))) + D0); - + rA = BH_defineMatrix([0,TLT(iPrj,4),0],'SPIDER','inv'); % Transform the specimen plane tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ +oX); @@ -1202,20 +1202,20 @@ % is always < 0) tZ( tX < 1 | tY < 1 | tX > d1 | tY > d2) = 1; - + minDefocus = min(tZ(:)); maxDefocus = max(tZ(tZ<1)); % Spit out some info -% fprintf('Found a min/max defocus of %3.3e/ %3.3e for tilt %d (%3.3f deg)\n',minDefocus,maxDefocus,iPrj,TLT(iPrj,4)); + % fprintf('Found a min/max defocus of %3.3e/ %3.3e for tilt %d (%3.3f deg)\n',minDefocus,maxDefocus,iPrj,TLT(iPrj,4)); % To track sampling in case I put in overlap samplingMask = zeros([d1,d2],'single','gpuArray'); - + for iDefocus = minDefocus-ctf3dDepth/1:ctf3dDepth/1:maxDefocus+ctf3dDepth/1 defVect = [iDefocus - ddF, iDefocus + ddF, dPhi]; - + if (phakePhasePlate(1) > 0) - + if numel(phakePhasePlate) == 2 modPower = floor(phakePhasePlate(2)); SNR = rem(phakePhasePlate(2),1); @@ -1223,13 +1223,13 @@ modPower = 1; SNR = 1; end - - + + [Hqz, ~] = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,1,SNR); - + Hqz = (-1).^modPower.*(phakePhasePlate(1).*Hqz).^1; - - + + modHqz = []; else if PIXEL_SIZE < 2.0e-10 @@ -1238,57 +1238,57 @@ Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,-1); else Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1); - end + end end - - - if (flgWhitenPS(3)) - tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz./(abs(Hqz).^2+flgWhitenPS(3)))),trimVal(1,:),trimVal(2,:),'GPU','single'); - else - tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz)),trimVal(1,:),trimVal(2,:),'GPU','single'); - end - - tmpMask = (tZ > iDefocus - ctf3dDepth/2 & tZ <= iDefocus + ctf3dDepth/2); - -% try - - linearIDX = unique(sub2ind([d1,d2],tX(tmpMask),tY(tmpMask))); -% catch -% -% -% ferr=fopen('err.txt','w'); -% fprintf(ferr,'%f %f\n',[tX(tmpMask),tY(tmpMask)]); -% fclose(ferr); -% error('sdf') -% end - - correctedPrj(linearIDX) = correctedPrj(linearIDX) + tmpCorrection(linearIDX); - samplingMask(linearIDX) = samplingMask(linearIDX) + 1; - + + + if (flgWhitenPS(3)) + tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz./(abs(Hqz).^2+flgWhitenPS(3)))),trimVal(1,:),trimVal(2,:),'GPU','single'); + else + tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz)),trimVal(1,:),trimVal(2,:),'GPU','single'); + end + + tmpMask = (tZ > iDefocus - ctf3dDepth/2 & tZ <= iDefocus + ctf3dDepth/2); + + % try + + linearIDX = unique(sub2ind([d1,d2],tX(tmpMask),tY(tmpMask))); + % catch + % + % + % ferr=fopen('err.txt','w'); + % fprintf(ferr,'%f %f\n',[tX(tmpMask),tY(tmpMask)]); + % fclose(ferr); + % error('sdf') + % end + + correctedPrj(linearIDX) = correctedPrj(linearIDX) + tmpCorrection(linearIDX); + samplingMask(linearIDX) = samplingMask(linearIDX) + 1; + end % end loop over defocus values - - + + samplingMask(samplingMask == 0) = 1; - - if (flgWhitenPS(1)) - correctedStack(:,:,TLT(iPrj,1)) =gather(BH_whitenNoiseSpectrum(correctedPrj./samplingMask,'',pixelSize,1)); - else - - correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj./samplingMask); - end - clear correctedPrj samplingMask tmpMask tmpCorrection - - clear iProjection iProjectionFT + + if (flgWhitenPS(1)) + correctedStack(:,:,TLT(iPrj,1)) =gather(BH_whitenNoiseSpectrum(correctedPrj./samplingMask,'',pixelSize,1)); + else + + correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj./samplingMask); + end + clear correctedPrj samplingMask tmpMask tmpCorrection + + clear iProjection iProjectionFT end % end loop over projections clear tile Hqz end function [avgZ, maxZ, tomoNumber,surfaceFit] = calcAvgZ(masterTM,iCoords, ... - tiltName,tomoList,... - nTomos, pixelSize,... - samplingRate,cycleNumber,... - sectionList,calcMaxZ) + tiltName,tomoList,... + nTomos, pixelSize,... + samplingRate,cycleNumber,... + sectionList,calcMaxZ) % Calculate the maximum extensions in Z and then how many separate sections % need to be corrected. @@ -1300,15 +1300,15 @@ for iTomo = 1:nTomos % The tomograms may not be listed monotonically so explicitly get their % id number - + if isa(masterTM,'struct') tomoNumber(iTomo) = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; nZdZ = iCoords(tomoNumber(iTomo),[4,6]); else tomoNumber(iTomo) = iTomo; - nZdZ = iCoords(iTomo,[4,6]); + nZdZ = iCoords(iTomo,[4,6]); end - + % half the size in z plus the shift back to the microscope coords. sZneeded = 2.*ceil(nZdZ(1)/2+abs(nZdZ(2))+1); if sZneeded > maxZ @@ -1332,12 +1332,12 @@ initGeom = masterTM.(cycleNumber).RawAlign; fprintf('Loaded the geometry for RawAlign %s\n',cycleNumber); catch - fprintf('Failed to load the geometry for RawAlign %s\nTrying cycle000\n',cycleNumber); + fprintf('Failed to load the geometry for RawAlign %s\nTrying cycle000\n',cycleNumber); try initGeom = masterTM.cycle000.geometry; catch error(['Could not load the initial geometry subTomoMeta.%s.geometry\n or--',... - 'subTomoMeta.cycle000.geometry\n'],cycleNumber); + 'subTomoMeta.cycle000.geometry\n'],cycleNumber); end end @@ -1357,57 +1357,57 @@ xFull{iSection} = []; yFull{iSection} = []; zFull{iSection} = []; -end +end for iT = 1:nTomos iTomo = tomoNumber(iT); micDimension = floor(masterTM.tiltGeometry.(tomoList{iT})(1,20:22) ./ samplingRate); - + % Already scaled to sampled pixels tomoOrigin =[ ceil((iCoords(iTomo,1)+1)./2),... - ceil((iCoords(iTomo,3)-iCoords(iTomo,2))./2),... - ceil((iCoords(iTomo,4)+1)/2)]; + ceil((iCoords(iTomo,3)-iCoords(iTomo,2))./2),... + ceil((iCoords(iTomo,4)+1)/2)]; micOrigin = [-1*iCoords(iTomo,5), ... - (iCoords(iTomo,2) + tomoOrigin(2)) - ceil((micDimension(2)+1)/2),... - iCoords(iTomo,6)]; - + (iCoords(iTomo,2) + tomoOrigin(2)) - ceil((micDimension(2)+1)/2),... + iCoords(iTomo,6)]; + iTomoName = sprintf('%s_%d',tiltName,iTomo); - - % shouldn't be any removed particles at this stage but later there would be. + + % shouldn't be any removed particles at this stage but later there would be. zList = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,13)./samplingRate; - + % shift from lower left to centered and include the tomos offset from the % microscope frame zList = zList - tomoOrigin(3) + micOrigin(3); totalZ = totalZ + sum(zList); fprintf('%s tomo has %d subTomos with mean Z %3.3f nm\n', ... - iTomoName, length(zList), mean(zList)*pixelSize./10); + iTomoName, length(zList), mean(zList)*pixelSize./10); nSubTomos = nSubTomos + length(zList); - + for iSection = 1:nSections - + iSecOrigin = sectionList{iT}(iSection,6); iSecRadius = sectionList{iT}(iSection,5)/2; inSectionIDX = zList > iSecOrigin - iSecRadius & zList <= iSecOrigin + iSecRadius; - - + + x = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,11)./samplingRate; - x = x - tomoOrigin(1) + micOrigin(1); + x = x - tomoOrigin(1) + micOrigin(1); y = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,12)./samplingRate; y = y - tomoOrigin(2) + micOrigin(2); - - + + xFull{iSection} = [xFull{iSection} ; x(inSectionIDX)]; yFull{iSection} = [yFull{iSection} ; y(inSectionIDX)]; zFull{iSection} = [zFull{iSection} ; zList(inSectionIDX)]; - - - - + + + + end % loop over sections - clear zList + clear zList end % loop over tomos @@ -1415,25 +1415,25 @@ % sf(x,y) = p00 + p10*x + p01*y; % surfaceFit = fit([xFull, yFull],zFull,'poly11'); - %surfaceFit = fit([xFull, yFull],zFull,'lowess','Span',0.1); -for iSection = 1:nSections - +%surfaceFit = fit([xFull, yFull],zFull,'lowess','Span',0.1); +for iSection = 1:nSections + if length(xFull{iSection}) >= 6 -% try -% try -% exclude = abs(mean(zFull{iSection})-zFull{iSection})>1.5.*std(zFull{iSection}); -% surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'lowess', 'Span', 0.05,'Normalize','on','Exclude',exclude); -% catch -% -% surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'lowess','Robust','on'); -% end - - surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'poly22','Robust','on'); - % end -% -% figure('visible','off'), plot(surfaceFit{iSection},[xFull{iSection},yFull{iSection}],zFull{iSection}); -% saveas(gcf,sprintf('fitThis_%s_%d.pdf',tiltName,iSection)); -% close(gcf); + % try + % try + % exclude = abs(mean(zFull{iSection})-zFull{iSection})>1.5.*std(zFull{iSection}); + % surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'lowess', 'Span', 0.05,'Normalize','on','Exclude',exclude); + % catch + % + % surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'lowess','Robust','on'); + % end + + surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'poly22','Robust','on'); + % end + % + % figure('visible','off'), plot(surfaceFit{iSection},[xFull{iSection},yFull{iSection}],zFull{iSection}); + % saveas(gcf,sprintf('fitThis_%s_%d.pdf',tiltName,iSection)); + % close(gcf); else surfaceFit{iSection} = 0; end @@ -1443,9 +1443,9 @@ fprintf('%s tilt-series has %d subTomos with mean Z %3.3f nm\n', ... - tiltName, nSubTomos,avgZ*10^9); - - + tiltName, nSubTomos,avgZ*10^9); + + diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 3161afb3..c9a3aec0 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -3,7 +3,7 @@ global bh_global_do_2d_fourier_interp; !mkdir -p aliStacks modLocal = false; -if length(varargin) > 3 +if length(varargin) > 3 error('Too many input arguments'); else % PARAMETER_FILE, STACK_BASENAME, gpuIDX @@ -72,7 +72,7 @@ startingDirection = emc.('startingDirection'); doseSymmetricIncrement = emc.('doseSymmetricIncrement'); doseAtMinTilt = emc.('doseAtMinTilt'); - + flgOldDose = 0; tltOrder = calc_dose_scheme(emc,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT); @@ -101,12 +101,12 @@ if (SuperResolution) if SuperResolution == 1 % Standard scenario crop to physical nyquist - scalePixelsBy = 2; + scalePixelsBy = 2; elseif SuperResolution > 10^10*PIXEL_SIZE % Crop to the given pixels size error('Scaling to arbitrary pixel size is not working\n'); % Need to factor in the trunctation to integer pixel size. -% scalePixelsBy = SuperResolution/(10^10*PIXEL_SIZE); + % scalePixelsBy = SuperResolution/(10^10*PIXEL_SIZE); else error('SuperResolution must be 0 (off) 1 (crop to physical Nyquist) or a pixel Size larger than current\n'); end @@ -120,8 +120,8 @@ % useGPU = 0; % METHOD = 'cpu'; % else - useGPU = 1; - METHOD = 'GPU'; +useGPU = 1; +METHOD = 'GPU'; % end % Sanity check @@ -134,13 +134,13 @@ elseif (AMPCONT < 0.025 || AMPCONT > 0.25) fprintf('\nWARNING: AMPCONT probably should be [0.025,0.25]\n'); end - WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; +WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; if Cs == 0 Cs = 1e-6; end - + CUM_e_DOSE = emc.('CUM_e_DOSE'); % test astigmatism vals flgAstigmatism = 1; @@ -160,9 +160,9 @@ eraseRadius = ceil(1.2.*(emc.('beadDiameter')./PIXEL_SIZE.*0.5)); flgImodErase = 0 - - -% Assuming that the first CTF zero is always less than this value + + +% Assuming that the first CTF zero is always less than this value FIXED_FIRSTZERO = PIXEL_SIZE / (70*10^-10) ; highCutoff = PIXEL_SIZE/emc.('defCutOff'); % I still use the def for underfocus < 0 as this places the origin at the @@ -180,7 +180,7 @@ deltaZTolerance = 100e-9; end -try +try zShift = abs(emc.('zShift')); catch zShift = 150e-9; @@ -193,7 +193,7 @@ try maxNumberOfTiles = emc.('ctfMaxNumberOfTiles'); catch - + maxNumberOfTiles = 10000; end @@ -204,7 +204,7 @@ zShift = zShift / PIXEL_SIZE; % Tile size & overlap -try +try tileSize = emc.('ctfTileSize'); catch tileSize = floor(680e-10 / PIXEL_SIZE); @@ -228,7 +228,7 @@ -if exist(stackNameIN, 'file') +if exist(stackNameIN, 'file') if ( flgOldDose ) TLT = zeros(length(rawTLT),23); @@ -258,47 +258,47 @@ TLT(:,19) = tltOrder(:,4); end - - + + [pathName,fileName,extension] = fileparts(stackNameIN); if isempty(pathName) pathName = '.'; end else fprintf('ignoring %s, because the file is not found.\n', stackNameIN) - + end - + % Make ctf directory to store diagnostic images -system(sprintf('mkdir -p %s/ctf', pathName)); +system(sprintf('mkdir -p %s/ctf', pathName)); % if ~(flgResume) - iMrcObj = MRCImage(stackNameIN,0); +iMrcObj = MRCImage(stackNameIN,0); - % The pixel size should be previously set correctly, but if it is not, then we - % must maintain whatever is there in case beads are to be erased. The model - % used for this process depends on the pixel size in the header when it was - % created in IMod alignment. +% The pixel size should be previously set correctly, but if it is not, then we +% must maintain whatever is there in case beads are to be erased. The model +% used for this process depends on the pixel size in the header when it was +% created in IMod alignment. - iHeader = getHeader(iMrcObj); - - iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* scalePixelsBy, ... - iHeader.cellDimensionY/iHeader.nY .* scalePixelsBy, ... - iHeader.cellDimensionZ/iHeader.nZ]; - % Reduce the Z dimension after pixel size is calculated - iHeader.nZ = iHeader.nZ - nSkipped; - - iOriginHeader= [iHeader.xOrigin , ... - iHeader.yOrigin , ... - iHeader.zOrigin ] ./ scalePixelsBy; +iHeader = getHeader(iMrcObj); + +iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* scalePixelsBy, ... + iHeader.cellDimensionY/iHeader.nY .* scalePixelsBy, ... + iHeader.cellDimensionZ/iHeader.nZ]; +% Reduce the Z dimension after pixel size is calculated +iHeader.nZ = iHeader.nZ - nSkipped; + +iOriginHeader= [iHeader.xOrigin , ... + iHeader.yOrigin , ... + iHeader.zOrigin ] ./ scalePixelsBy; + +d1 = iHeader.nX ; d2 = iHeader.nY ; d3 = iHeader.nZ; - d1 = iHeader.nX ; d2 = iHeader.nY ; d3 = iHeader.nZ; - @@ -320,8 +320,8 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 % Part of the switch to listing dose not bfactor to use the optimal exposure -% filter by Grant/Grigorieff - - nPrjs = size(TLT,1); +% filter by Grant/Grigorieff - +nPrjs = size(TLT,1); if ( flgOldDose && flgStandardOrdeDoCalc ) if CUM_e_DOSE < 0 @@ -333,14 +333,14 @@ flgCosineDose = 0; exposure = CUM_e_DOSE./nPrjs; end - + totalExposure = 0; % If the fit tilt angles have moved alot, you may end up with duplicates, alreadyPicked = zeros(nPrjs,1,'single','gpuArray'); largeVect = alreadyPicked + 1000; for iExposure = 1:nPrjs % find the projection angle most closley matching ( - + [~,iTilt] = min((alreadyPicked.*largeVect)+(abs(TLT(:,4)-tltOrder(iExposure)))); alreadyPicked(iTilt) = 1; if flgCosineDose == 0 @@ -350,11 +350,11 @@ end TLT(iTilt,11) = totalExposure; TLT(iTilt,14) = exposure; - + end - -end +end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -368,7 +368,7 @@ outputStackName = sprintf('aliStacks/%s%s',stackNameOUT,extension) -try +try erase_beads_after_ctf = emc.('erase_beads_after_ctf'); catch erase_beads_after_ctf = false; @@ -391,11 +391,11 @@ else sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); end -sizeCropped(3) = d3; +sizeCropped(3) = d3; STACK = zeros(sizeCropped,'single'); - samplingMaskStack = zeros(sizeCropped,'single'); - +samplingMaskStack = zeros(sizeCropped,'single'); + if (flgReOrderMapBack) @@ -406,7 +406,7 @@ % shiftMETHOD = 'cpu'; % fprintf('transforming on cpu b/c > 4096\n') % else - shiftMETHOD = 'GPU'; +shiftMETHOD = 'GPU'; % end % Redefine d3 incase views are ignored @@ -418,111 +418,111 @@ for i = 1:d3 -% fprintf('Transforming prj %d in fourier space oversampled by 2x physical Nyquist\n',i); - - + % fprintf('Transforming prj %d in fourier space oversampled by 2x physical Nyquist\n',i); + + % Stored in row order as output by imod, st transpose is needed. Inversion % of the xform is handled in resample2d. origXF = [1,0;0,1]; newXF = reshape(mbEST(TLT(i,23),1:4),2,2)'; - - + + dXYZ = [(newXF*TLT(i,2:3)')' + mbEST(TLT(i,23),5:6) , 0]; TLT(i,2:3) = dXYZ(1:2); dXYZ = dXYZ ./ scalePixelsBy; - + combinedXF = reshape((newXF*origXF)',1,4); TLT(i,7:10) = combinedXF; - - - - - sizeODD = [d1,d2]-[osX,osY]; - - % If it is even sized, shift up one pixel so that the origin is in the middle - % of the odd output here we can just read it in this way, unlike super res. - - iProjection = ... - single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],TLT(i,23),'keep')); - - iProjection = real(ifftn(fftn(iProjection).* BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'))); - - largeOutliersMean= mean(iProjection(:)); - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); - - largeOutliersIDX = []; - - % Padding to avoid interpolation artifacts. For K3 images this can push - % a 2080 close to or over the limit, so it is been reduced to 1/4 (from - % 1) i.e. the image is paded to 1.25 x unless useFourierInterp is set > - % 1; - sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp*0.25).*max(sizeODD)); -% sizeSQ = floor(([1,1]).*max(sizeODD)); - - padVal = BH_multi_padVal(sizeODD,sizeSQ); - trimVal = BH_multi_padVal(sizeSQ,sizeCropped(1:2)); - - + + + + sizeODD = [d1,d2]-[osX,osY]; + + % If it is even sized, shift up one pixel so that the origin is in the middle + % of the odd output here we can just read it in this way, unlike super res. + + iProjection = ... + single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],TLT(i,23),'keep')); + + iProjection = real(ifftn(fftn(iProjection).* BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'))); + + largeOutliersMean= mean(iProjection(:)); + largeOutliersSTD = std(iProjection(:)); + largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... + iProjection > largeOutliersMean + 6*largeOutliersSTD); + iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); + + largeOutliersIDX = []; + + + % Padding to avoid interpolation artifacts. For K3 images this can push + % a 2080 close to or over the limit, so it is been reduced to 1/4 (from + % 1) i.e. the image is paded to 1.25 x unless useFourierInterp is set > + % 1; + sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp*0.25).*max(sizeODD)); + % sizeSQ = floor(([1,1]).*max(sizeODD)); + + padVal = BH_multi_padVal(sizeODD,sizeSQ); + trimVal = BH_multi_padVal(sizeSQ,sizeCropped(1:2)); + + iProjection = iProjection - mean(iProjection(:)); - + if ( SuperResolution ) iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... - padVal(1,:),padVal(2,:),shiftMETHOD,'singleTaper'); + padVal(1,:),padVal(2,:),shiftMETHOD,'singleTaper'); else iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - shiftMETHOD,'singleTaper'); + shiftMETHOD,'singleTaper'); end - - + + if (i == 1 && bh_global_do_2d_fourier_interp) bhF = fourierTransformer(iProjection,'OddSizeOversampled'); end - - - % Do the phase shift after rotating - need to invert the scaling since - % we are in reciprocal space - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(combinedXF); - - - - if (bh_global_do_2d_fourier_interp) -% combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); - else - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); - end - + + % Do the phase shift after rotating - need to invert the scaling since + % we are in reciprocal space + [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... + BH_decomposeIMODxf(combinedXF); + + + + if (bh_global_do_2d_fourier_interp) + % combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); + combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); + combinedInverted = combinedInverted([1,2,4,5]); + + iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); + else + combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); + combinedInverted = combinedInverted([1,2,4,5]); + iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); + end + iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); iSamplingMask(isnan(iSamplingMask(:))) = 0; samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); iSamplingMask = []; - -% % % % % iProjection = real(fftshift(ifftn(ifftshift(iProjection)))); - STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... - trimVal(1,:),trimVal(2,:),... - shiftMETHOD,'single'))); - - -end + + % % % % % iProjection = real(fftshift(ifftn(ifftshift(iProjection)))); + STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... + trimVal(1,:),trimVal(2,:),... + shiftMETHOD,'single'))); + + +end if ( flgEraseBeads ) - STACK = BH_eraseBeads(STACK,eraseRadius, fileName, scalePixelsBy,0,sortrows(TLT,1)); -end + STACK = BH_eraseBeads(STACK,eraseRadius, fileName, scalePixelsBy,0,sortrows(TLT,1)); +end [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',100,PIXEL_SIZE*10^10,samplingMaskStack); @@ -533,455 +533,455 @@ if ~(flgSkip) -gpuDevice(gpuIDX) -[d1,d2,d3] = size(STACK); -if (PIXEL_SIZE*10^10 < 0) - - flgCrop = 1; - [croppedIMG,pixelOUT] = cropIMG(STACK(:,:,1),PIXEL_SIZE*10^10); - [d1C,d2C] = size(croppedIMG); clear croppedIMG - tltForExp = TLT; - tltForExp(:,16) = pixelOUT*10^-10; - pixelOUT - % Redefining things down hear is a stupid thing to do. Fix this if you - % keep the optino for cropping. - FIXED_FIRSTZERO = pixelOUT / 70 ; - highCutoff = (pixelOUT*10^-10)/emc.('defCutOff'); + gpuDevice(gpuIDX) + [d1,d2,d3] = size(STACK); + if (PIXEL_SIZE*10^10 < 0) + + flgCrop = 1; + [croppedIMG,pixelOUT] = cropIMG(STACK(:,:,1),PIXEL_SIZE*10^10); + [d1C,d2C] = size(croppedIMG); clear croppedIMG + tltForExp = TLT; + tltForExp(:,16) = pixelOUT*10^-10; + pixelOUT + % Redefining things down hear is a stupid thing to do. Fix this if you + % keep the optino for cropping. + FIXED_FIRSTZERO = pixelOUT / 70 ; + highCutoff = (pixelOUT*10^-10)/emc.('defCutOff'); + + else + + flgCrop = 0; + pixelOUT = PIXEL_SIZE*10^10; + d1C = d1; + d2C = d2; + tltForExp = TLT; + + end -else + d3 = size(STACK,3) + % Check for extra large (8k) data which will be too big for the gpu. + % Should set this up to be a hybrid where each slice is on GPU but + % But then pull to the cpu and store there in stack. + if d1C > 4096 || d2C > 4096 || d3 > 40 + prjMaskMethod = 'GPU' + else + prjMaskMethod = 'GPU' + end - flgCrop = 0; - pixelOUT = PIXEL_SIZE*10^10; - d1C = d1; - d2C = d2; - tltForExp = TLT; + % % % % evalMask = zeros(d1C,d2C,d3,'single'); + % % % % for iPrj = 1:d3 + % % % % tmpTLT = tltForExp(iPrj,:); + % % % % % need to write over the projections position in the stack to not expand beyond 2d + % % % % tmpTLT(1) = 1; + % % % % [ iEvalMask, ~ ] = BH_multi_projectionMask([d1C,d2C,1;d1C,d2C,1], tmpTLT, ... + % % % % 'GPU', [zShift,deltaZTolerance] ); + % % % % + % % % % evalMask(:,:,tltForExp(iPrj,1)) = gather(iEvalMask); + % % % % end + % % % % + % % % % + % % % % %evalMask = gather(evalMask); + % % % % + % % % % + % % % % nTiles = zeros(size(STACK,3),1); + % % % % + % % % % + % % % % for i = 1+tileSize/2:overlap:d1C-tileSize/2 + % % % % for j = 1+tileSize/2:overlap:d2C-tileSize/2 + % % % % for k = 1:size(STACK,3) + % % % % if evalMask(i,j,k) + % % % % nTiles(k) = nTiles(k) + 1; + % % % % end + % % % % end + % % % % end + % % % % end -end - -d3 = size(STACK,3) -% Check for extra large (8k) data which will be too big for the gpu. -% Should set this up to be a hybrid where each slice is on GPU but -% But then pull to the cpu and store there in stack. -if d1C > 4096 || d2C > 4096 || d3 > 40 - prjMaskMethod = 'GPU' - else - prjMaskMethod = 'GPU' -end - -% % % % evalMask = zeros(d1C,d2C,d3,'single'); -% % % % for iPrj = 1:d3 -% % % % tmpTLT = tltForExp(iPrj,:); -% % % % % need to write over the projections position in the stack to not expand beyond 2d -% % % % tmpTLT(1) = 1; -% % % % [ iEvalMask, ~ ] = BH_multi_projectionMask([d1C,d2C,1;d1C,d2C,1], tmpTLT, ... -% % % % 'GPU', [zShift,deltaZTolerance] ); -% % % % -% % % % evalMask(:,:,tltForExp(iPrj,1)) = gather(iEvalMask); -% % % % end -% % % % -% % % % -% % % % %evalMask = gather(evalMask); -% % % % -% % % % -% % % % nTiles = zeros(size(STACK,3),1); -% % % % -% % % % -% % % % for i = 1+tileSize/2:overlap:d1C-tileSize/2 -% % % % for j = 1+tileSize/2:overlap:d2C-tileSize/2 -% % % % for k = 1:size(STACK,3) -% % % % if evalMask(i,j,k) -% % % % nTiles(k) = nTiles(k) + 1; -% % % % end -% % % % end -% % % % end -% % % % end - - - -[radialForCTF,phi,~,~,~,~] = ... - BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU', ... - {'none'},1,0,0); - -radialForCTF = {radialForCTF./(pixelOUT.*10^-10),0,phi}; clear phi - -flgExpFilter = 0; - -inc = (0.5 - FIXED_FIRSTZERO) / (paddedSize/2); -freqVector = [inc+FIXED_FIRSTZERO:inc:0.5 ]; -% % % % clear sumVector radialAvg -% % % % sumVector(length(freqVector)) = gpuArray(double(0)); -% % % % radialAvg(length(freqVector)) = gpuArray(double(0)); - - -tic -nT = 1; -nT2=0; -nT3= 0; - -halfX = floor(paddedSize/2) + 1; -% % % % psTile = zeros([(paddedSize).*[1,1],3],'single','gpuArray'); -psTile = zeros([halfX,paddedSize,3],'single','gpuArray'); - -bhF2 = fourierTransformer(randn(paddedSize,paddedSize,'single','gpuArray')); - -for k = 1:d3 - if (skipFitting) - break - end - - tiltIDX = TLT(k,1); - % Center the pixel coordinates - iEvalMask = BH_multi_gridCoordinates([d1C,1,1],'Cartesian','GPU',{'none'},0,1,0); - - % Convert to the z-height in the projection - iEvalMask = iEvalMask.*(-1.*tand(TLT(tiltIDX,4))); - iEvalPos = iEvalMask; - iEvalNeg = iEvalMask; - % Shift by any amount wanted - iEvalPos = iEvalPos - zShift; - iEvalNeg = iEvalNeg + zShift; + [radialForCTF,phi,~,~,~,~] = ... + BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU', ... + {'none'},1,0,0); - % Select region limited by tolerance - iEvalPos = ( iEvalPos > gpuArray(-deltaZTolerance) & iEvalPos < gpuArray(deltaZTolerance)); - iEvalNeg = ( iEvalNeg > gpuArray(-deltaZTolerance) & iEvalNeg < gpuArray(deltaZTolerance)); - iEvalMask = ( iEvalMask > gpuArray(-deltaZTolerance) & iEvalMask < gpuArray(deltaZTolerance)); - + radialForCTF = {radialForCTF./(pixelOUT.*10^-10),0,phi}; clear phi - tmpTile = zeros([halfX,paddedSize,3],'single','gpuArray'); - -% % % % tmpTile = zeros([paddedSize.*[1,1],3],'single','gpuArray'); - - if flgCrop - [iProjection,~] = cropIMG(gpuArray(STACK(:,:,TLT(k,1))),PIXEL_SIZE*10^10); - else - iProjection = (gpuArray(STACK(:,:,TLT(k,1)))); - end + flgExpFilter = 0; + + inc = (0.5 - FIXED_FIRSTZERO) / (paddedSize/2); + freqVector = [inc+FIXED_FIRSTZERO:inc:0.5 ]; + % % % % clear sumVector radialAvg + % % % % sumVector(length(freqVector)) = gpuArray(double(0)); + % % % % radialAvg(length(freqVector)) = gpuArray(double(0)); + + + tic + nT = 1; + nT2=0; + nT3= 0; + + halfX = floor(paddedSize/2) + 1; + % % % % psTile = zeros([(paddedSize).*[1,1],3],'single','gpuArray'); + psTile = zeros([halfX,paddedSize,3],'single','gpuArray'); + + bhF2 = fourierTransformer(randn(paddedSize,paddedSize,'single','gpuArray')); + + for k = 1:d3 + if (skipFitting) + break + end + + tiltIDX = TLT(k,1); + % Center the pixel coordinates + iEvalMask = BH_multi_gridCoordinates([d1C,1,1],'Cartesian','GPU',{'none'},0,1,0); + + % Convert to the z-height in the projection + iEvalMask = iEvalMask.*(-1.*tand(TLT(tiltIDX,4))); + + iEvalPos = iEvalMask; + iEvalNeg = iEvalMask; + + % Shift by any amount wanted + iEvalPos = iEvalPos - zShift; + iEvalNeg = iEvalNeg + zShift; + + % Select region limited by tolerance + iEvalPos = ( iEvalPos > gpuArray(-deltaZTolerance) & iEvalPos < gpuArray(deltaZTolerance)); + iEvalNeg = ( iEvalNeg > gpuArray(-deltaZTolerance) & iEvalNeg < gpuArray(deltaZTolerance)); + iEvalMask = ( iEvalMask > gpuArray(-deltaZTolerance) & iEvalMask < gpuArray(deltaZTolerance)); + - iProjection = iProjection - ... - BH_movingAverage(iProjection,[tileSize,tileSize]); - - iProjection = iProjection ./ ... - BH_movingRMS(iProjection,[tileSize,tileSize]); - - - - - - for i = 1+tileSize/2:overlap:d1C-tileSize/2 - if min([nT,nT2,nT3])< maxNumberOfTiles && (iEvalMask(i) || iEvalPos(i) || iEvalNeg(i)) - for j = 1+tileSize/2:overlap:d2C-tileSize/2 - - thisTile = abs(bhF2.fwdFFT(BH_padZeros3d(... - (iProjection( ... - i-tileSize/2+1:i+tileSize/2,... - j-tileSize/2+1:j+tileSize/2)),... - padVAL(1,:),padVAL(2,:),... - 'GPU','singleTaper'))); - -% % % % -% % % % thisTile = abs(fftn( ... -% % % % BH_padZeros3d(... -% % % % (iProjection( ... -% % % % i-tileSize/2+1:i+tileSize/2,... -% % % % j-tileSize/2+1:j+tileSize/2)),... -% % % % padVAL(1,:),padVAL(2,:),... -% % % % 'GPU','singleTaper'))); - tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; - - - if (iEvalMask(i)) + tmpTile = zeros([halfX,paddedSize,3],'single','gpuArray'); + + % % % % tmpTile = zeros([paddedSize.*[1,1],3],'single','gpuArray'); + + if flgCrop + [iProjection,~] = cropIMG(gpuArray(STACK(:,:,TLT(k,1))),PIXEL_SIZE*10^10); + else + iProjection = (gpuArray(STACK(:,:,TLT(k,1)))); + end + + iProjection = iProjection - ... + BH_movingAverage(iProjection,[tileSize,tileSize]); + + iProjection = iProjection ./ ... + BH_movingRMS(iProjection,[tileSize,tileSize]); + + + + + + for i = 1+tileSize/2:overlap:d1C-tileSize/2 + if min([nT,nT2,nT3])< maxNumberOfTiles && (iEvalMask(i) || iEvalPos(i) || iEvalNeg(i)) + for j = 1+tileSize/2:overlap:d2C-tileSize/2 + + thisTile = abs(bhF2.fwdFFT(BH_padZeros3d(... + (iProjection( ... + i-tileSize/2+1:i+tileSize/2,... + j-tileSize/2+1:j+tileSize/2)),... + padVAL(1,:),padVAL(2,:),... + 'GPU','singleTaper'))); + + % % % % + % % % % thisTile = abs(fftn( ... + % % % % BH_padZeros3d(... + % % % % (iProjection( ... + % % % % i-tileSize/2+1:i+tileSize/2,... + % % % % j-tileSize/2+1:j+tileSize/2)),... + % % % % padVAL(1,:),padVAL(2,:),... + % % % % 'GPU','singleTaper'))); + tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; + + + if (iEvalMask(i)) nT = nT+1; tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; - end - if ( iEvalPos(i) ) - nT2 = nT2+1; - tmpTile(:,:,2) = tmpTile(:,:,2) + thisTile; - end - if (iEvalNeg(i) ) - nT3 = nT3+1; - tmpTile(:,:,3) = tmpTile(:,:,3) + thisTile; - end - + end + if ( iEvalPos(i) ) + nT2 = nT2+1; + tmpTile(:,:,2) = tmpTile(:,:,2) + thisTile; + end + if (iEvalNeg(i) ) + nT3 = nT3+1; + tmpTile(:,:,3) = tmpTile(:,:,3) + thisTile; + end + + end end end - end - fprintf('%d tiles at dZ= 0\t%d tiles at dZ > 0\t%d tiles at dZ < 0, after tilt %d\n',nT,nT2,nT3,k); - - % Apply the dose filter to the sum of each projection to save a bunch of - % multiplicaiton - psTile = psTile + tmpTile; - -end -clear tmpTile -toc - -rotAvgPowerSpec = zeros([paddedSize,paddedSize,3],'single','gpuArray'); -for iTile = 1:3 - tmp = bhF2.swapIndexFWD(psTile(:,:,iTile)); - psTile(:,:,iTile) = bhF2.swapIndexFWD(psTile(:,:,iTile)); - rotAvgPowerSpec(:,:,iTile) = BH_multi_makeHermitian(psTile(:,:,iTile),[paddedSize,paddedSize],1); -end - -clear psTile - -if ~(skipFitting) -% % % % for iTile = 1:3 -% % % % rotAvgPowerSpec(:,:,iTile) = (fftshift(rotAvgPowerSpec(:,:,iTile))); -% % % % end - AvgPowerSpec = rotAvgPowerSpec; - - % TODO make a better rotational averaging funciton - [rot1, rot2, ~, r1,r2, ~] = BH_multi_gridCoordinates(paddedSize.*[1,1], ... - 'Cartesian','GPU', ... - {'none'},0,1,0); - - for i = 0.5:0.5:360 - R = BH_defineMatrix([i,0,0],'Bah','forward'); - ROT1 = R(1).*rot1 + R(4).*rot2; - ROT2 = R(2).*rot1 + R(5).*rot2; + fprintf('%d tiles at dZ= 0\t%d tiles at dZ > 0\t%d tiles at dZ < 0, after tilt %d\n',nT,nT2,nT3,k); + + % Apply the dose filter to the sum of each projection to save a bunch of + % multiplicaiton + psTile = psTile + tmpTile; - for iTile = 1:3 - rotAvgPowerSpec(:,:,iTile) = rotAvgPowerSpec(:,:,iTile) + ... - interpn(r1,r2,AvgPowerSpec(:,:,iTile),... - ROT1,ROT2,'linear',0); - end end - clear ROT1 ROT2 - % Normalize on avgerage #, doseWeighting, and a radial filter to account - % for rotational averaging. - rotAvgPowerSpec = rotAvgPowerSpec ./ (720); clear a - % rotAvgPowerSpec = rotAvgPowerSpec ./ (720.*(sqrt(fftshift(radialForCTF{1}.*(pixelOUT.*10^-10))))); clear a - - - is_a_bummer = ~isfinite(rotAvgPowerSpec); - if sum(is_a_bummer,'all') > 0.5*numel(rotAvgPowerSpec) - error('the rotated Avg power spectrum is more than half nan or inf'); - else - rotAvgPowerSpec(is_a_bummer) = 0; + clear tmpTile + toc + + rotAvgPowerSpec = zeros([paddedSize,paddedSize,3],'single','gpuArray'); + for iTile = 1:3 + tmp = bhF2.swapIndexFWD(psTile(:,:,iTile)); + psTile(:,:,iTile) = bhF2.swapIndexFWD(psTile(:,:,iTile)); + rotAvgPowerSpec(:,:,iTile) = BH_multi_makeHermitian(psTile(:,:,iTile),[paddedSize,paddedSize],1); end - - currentDefocusEst = defEST; - currentDefocusWin = defWIN; - measuredVsExpected = zeros(2,3); -end - -for iTilt = 1:3 - - if (skipFitting) + clear psTile + + if ~(skipFitting) + % % % % for iTile = 1:3 + % % % % rotAvgPowerSpec(:,:,iTile) = (fftshift(rotAvgPowerSpec(:,:,iTile))); + % % % % end + AvgPowerSpec = rotAvgPowerSpec; + + % TODO make a better rotational averaging funciton + [rot1, rot2, ~, r1,r2, ~] = BH_multi_gridCoordinates(paddedSize.*[1,1], ... + 'Cartesian','GPU', ... + {'none'},0,1,0); + + for i = 0.5:0.5:360 + R = BH_defineMatrix([i,0,0],'Bah','forward'); + ROT1 = R(1).*rot1 + R(4).*rot2; + ROT2 = R(2).*rot1 + R(5).*rot2; + + for iTile = 1:3 + rotAvgPowerSpec(:,:,iTile) = rotAvgPowerSpec(:,:,iTile) + ... + interpn(r1,r2,AvgPowerSpec(:,:,iTile),... + ROT1,ROT2,'linear',0); + end + end + clear ROT1 ROT2 + % Normalize on avgerage #, doseWeighting, and a radial filter to account + % for rotational averaging. + rotAvgPowerSpec = rotAvgPowerSpec ./ (720); clear a + % rotAvgPowerSpec = rotAvgPowerSpec ./ (720.*(sqrt(fftshift(radialForCTF{1}.*(pixelOUT.*10^-10))))); clear a + + + is_a_bummer = ~isfinite(rotAvgPowerSpec); + if sum(is_a_bummer,'all') > 0.5*numel(rotAvgPowerSpec) + error('the rotated Avg power spectrum is more than half nan or inf'); + else + rotAvgPowerSpec(is_a_bummer) = 0; + end + + currentDefocusEst = defEST; - - % Add the determined defocus, and write out with mic paramters as well. - TLT(:,15) = repmat(-1.*defEST,size(TLT,1),1); - - [~, idx] = sortrows(abs(TLT(:,4)), -1); - TLT = TLT(idx,:); - % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt - % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast - fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); - fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%8.2f\n'], TLT'); - fclose(fileID); - fprintf('\n\nUsing the estimated value for defocus and phase shift provided\n\n'); - return; + currentDefocusWin = defWIN; + measuredVsExpected = zeros(2,3); end - - radialAvg = [rotAvgPowerSpec((paddedSize/2)+1:end,(paddedSize/2)+1,iTilt)]'; - - - defRange = [currentDefocusEst-currentDefocusWin,currentDefocusEst+currentDefocusWin]; - - defInc = 0.01; - - defVal = (defRange(1):defInc:defRange(2))'; - - cccStorage = zeros(length(defVal),2); - cccStorage(:,1) = defVal; - nDF = 1; - for iDF = defVal' - DF = iDF*10^-6; - - % TODO add a global switch for the damping - - if (PIXEL_SIZE < 1*10^-10) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0,-1); - else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); + + for iTilt = 1:3 + + if (skipFitting) + currentDefocusEst = defEST; + + % Add the determined defocus, and write out with mic paramters as well. + TLT(:,15) = repmat(-1.*defEST,size(TLT,1),1); + + [~, idx] = sortrows(abs(TLT(:,4)), -1); + TLT = TLT(idx,:); + % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt + % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY + % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast + fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); + fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%8.2f\n'], TLT'); + fclose(fileID); + fprintf('\n\nUsing the estimated value for defocus and phase shift provided\n\n'); + return; end - - try - [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, 0); - catch -% figure, imshow3D(gather(Hqz)); -% figure, imshow3D(gather(rotAvgPowerSpec)); -% highCutoff -% figure, plot(freqVector); -% figure, plot(radialAvg); + + radialAvg = [rotAvgPowerSpec((paddedSize/2)+1:end,(paddedSize/2)+1,iTilt)]'; + + + defRange = [currentDefocusEst-currentDefocusWin,currentDefocusEst+currentDefocusWin]; + + defInc = 0.01; + + defVal = (defRange(1):defInc:defRange(2))'; + + cccStorage = zeros(length(defVal),2); + cccStorage(:,1) = defVal; + nDF = 1; + for iDF = defVal' + DF = iDF*10^-6; + + % TODO add a global switch for the damping + + if (PIXEL_SIZE < 1*10^-10) + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0,-1); + else + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); + end + + try + [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, 0); + catch + % figure, imshow3D(gather(Hqz)); + % figure, imshow3D(gather(rotAvgPowerSpec)); + % highCutoff + % figure, plot(freqVector); + % figure, plot(radialAvg); + end + + [ iCCC ] = calc_CCC( freqVector, bg, bandpass, radialAvg, rV, cccScale); + + cccStorage(nDF, 2) = gather(iCCC); + nDF = nDF +1; end - - [ iCCC ] = calc_CCC( freqVector, bg, bandpass, radialAvg, rV, cccScale); - - cccStorage(nDF, 2) = gather(iCCC); - nDF = nDF +1; - end - - [~,maxVal] = max(cccStorage(:,2)); - maxDef = cccStorage(maxVal,1); - - if (iTilt == 1) + + [~,maxVal] = max(cccStorage(:,2)); + maxDef = cccStorage(maxVal,1); + + if (iTilt == 1) % Only save for the "true" defocus at the tilt-axes - figure('Visible','off'), scatter(cccStorage(:,1), cccStorage(:,2)); - title(sprintf('CCC\nmax = %03.3f micron', maxDef)); - xlabel('defocus (micron)'); ylabel('CCC'); - + figure('Visible','off'), scatter(cccStorage(:,1), cccStorage(:,2)); + title(sprintf('CCC\nmax = %03.3f micron', maxDef)); + xlabel('defocus (micron)'); ylabel('CCC'); + saveas(gcf,sprintf('%s/ctf/%s_ccFIT.pdf',pathName,stackNameOUT), 'pdf') - end + end DF = maxDef*10^-6; - + if (PIXEL_SIZE < 1*10^-10) [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0,-1); else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); end - + [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, 0); - - - if (iTilt == 1) + + + if (iTilt == 1) % Only save for the "true" defocus at the tilt-axes - figure('Visible','off'), plot(freqVector(bandpass),backGroundBuffer.*bg(freqVector(bandpass)),freqVector(bandpass),abs(radialAvg(bandpass))); - title('Background fitting') - - saveas(gcf,sprintf('%s/ctf/%s_bgFit.pdf',pathName,stackNameOUT), 'pdf') - end - - - clear STACK exposureFilter - + figure('Visible','off'), plot(freqVector(bandpass),backGroundBuffer.*bg(freqVector(bandpass)),freqVector(bandpass),abs(radialAvg(bandpass))); + title('Background fitting') + + saveas(gcf,sprintf('%s/ctf/%s_bgFit.pdf',pathName,stackNameOUT), 'pdf') + end + + + clear STACK exposureFilter + pdfOUT = sprintf('%s/ctf/%s_psRadial_%d.pdf',pathName,stackNameOUT,iTilt); - - + + bgSubPS = (abs(radialAvg) - bg(freqVector)').*bandpass; bgSubPS = bgSubPS ./ max(bgSubPS(:)); figure('Visible','off'), plot(freqVector(bandpass)./(pixelOUT),bgSubPS(bandpass), freqVector(bandpass)./(pixelOUT),abs(rV(bandpass)).^2./max(abs(rV(bandpass)).^2),'-g'); title(sprintf('CTF fit\n%03.3f μm ', maxDef)); xlabel('Spatial Frequency (1/Å)'); ylabel('Relative Power'); - + saveas(gcf,pdfOUT, 'pdf') - - - + + + if (flgAstigmatism) - - radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; + + radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; [radialAstig,~,~,~,~,~] = ... - BH_multi_gridCoordinates(size(Hqz),'Cartesian',... - 'GPU',{'none'},1,1,1); - + BH_multi_gridCoordinates(size(Hqz),'Cartesian',... + 'GPU',{'none'},1,1,1); + % Hqz from max defocus still exisists - + [ bg, bandpass, ~ ] = prepare_spectrum( Hqz, highCutoff ,... - freqVector, AvgPowerSpec(:,:,iTilt), radialAstig); - - [ bgSubPS ] = calc_CCC( radialAstig, bg, bandpass, AvgPowerSpec(:,:,iTilt),-9999, cccScale) ; - + freqVector, AvgPowerSpec(:,:,iTilt), radialAstig); + + [ bgSubPS ] = calc_CCC( radialAstig, bg, bandpass, AvgPowerSpec(:,:,iTilt),-9999, cccScale) ; + SAVE_IMG(MRCImage(gather(bgSubPS)), ... - sprintf('%s/ctf/%s_avgPS-bgSub%s',pathName,fileName,extension)); - - - - + sprintf('%s/ctf/%s_avgPS-bgSub%s',pathName,fileName,extension)); + + + + SAVE_IMG(MRCImage(gather(AvgPowerSpec)), ... - sprintf('%s/ctf/%s_avgPS%s',pathName,fileName,extension)); - - + sprintf('%s/ctf/%s_avgPS%s',pathName,fileName,extension)); + + coarseDefSearch = 0:floor(maxAstig/astigStep); coarseAngSearch = -pi/2:coarseAngStep:pi/2; - + fineAngSearch = -1*coarseAngStep/2:fineAngStep:coarseAngStep/2; fineDefSearch = -astigStep/2:astigStep/4:astigStep/2; - - - + + + initAstigCCC = zeros(length(coarseDefSearch)*length(coarseAngSearch),3, 'gpuArray'); - - + + n=1; - + for iAng = coarseAngSearch for iDelDF = coarseDefSearch df1 = maxDef*10^-6 - iDelDF*astigStep; df2 = maxDef*10^-6 + iDelDF*astigStep; - - if (PIXEL_SIZE < 1*10^-10) - - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0,-1); - else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0); - end - - - [ iCCC ] = calc_CCC( radialAstig, bgSubPS, bandpass, AvgPowerSpec, Hqz, cccScale); - - initAstigCCC(n,:) = [iAng,iDelDF*astigStep,iCCC]; - n = n + 1; + + if (PIXEL_SIZE < 1*10^-10) + + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0,-1); + else + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0); + end + + + [ iCCC ] = calc_CCC( radialAstig, bgSubPS, bandpass, AvgPowerSpec, Hqz, cccScale); + + initAstigCCC(n,:) = [iAng,iDelDF*astigStep,iCCC]; + n = n + 1; fprintf('%d / %d coarse astigmatism search\n',n,size(initAstigCCC,1)); end end - + nPeaks = 1; top3 = zeros(nPeaks,3,'gpuArray'); - + for iCCC = 1:nPeaks [~,c] = max(initAstigCCC(:,3)); top3(iCCC,:) = initAstigCCC(c,:); initAstigCCC = initAstigCCC(initAstigCCC(:,1)~=initAstigCCC(c,1),:); end - + refineCCC = zeros(length(fineDefSearch)*length(fineAngSearch)*nPeaks,3,'gpuArray'); - + n=1; for iPeak = 1:nPeaks mAng = top3(iPeak,1); mDef = top3(iPeak,2); - - for iAng = fineAngSearch + + for iAng = fineAngSearch for iDelDF = fineDefSearch - + df1 = maxDef*10^-6 - (mDef + iDelDF); df2 = maxDef*10^-6 + (mDef + iDelDF); - + % For values very close to zero, the search range may include % values |df1| < |df2| which is against convention. if abs(df1) >= abs(df2) - + if (PIXEL_SIZE < 1*10^-10) - - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng+mAng], ... - size(radialForCTF{1}), -AMPCONT,-1.0,-1); + + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [df1,df2,iAng+mAng], ... + size(radialForCTF{1}), -AMPCONT,-1.0,-1); else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng+mAng], ... - size(radialForCTF{1}), -AMPCONT,-1.0); + [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [df1,df2,iAng+mAng], ... + size(radialForCTF{1}), -AMPCONT,-1.0); end - - + + [ iCCC ] = calc_CCC( radialAstig,bgSubPS, bandpass, ... - AvgPowerSpec, Hqz,cccScale ); - + AvgPowerSpec, Hqz,cccScale ); + else iCCC = -9999; end - - refineCCC(n,:) = [iAng+mAng,mDef + iDelDF,iCCC]; - n = n + 1; + + refineCCC(n,:) = [iAng+mAng,mDef + iDelDF,iCCC]; + n = n + 1; fprintf('%d / %d fine astigmatism search\n',n,size(refineCCC,1)); end end @@ -992,303 +992,303 @@ topScore = fopen(sprintf('%s/ctf/%s_astig.txt',pathName,fileName),'w'); fprintf(topScore,'%7.7e %7.7e %2.7f\n',refineCCC(c,:)); fclose(topScore); - - - - - end - - if ( iTilt == 1) - - radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; - currentDefocusEst = maxDef; - currentDefocusWin = (defWIN*.25); - measuredVsExpected(1,:) = [maxDef + zShift*PIXEL_SIZE*10^6, maxDef, maxDef - zShift*PIXEL_SIZE*10^6]; - measuredVsExpected(2,2) = maxDef; - % Add the determined defocus, and write out with mic paramters as well. - TLT(:,15) = repmat(maxDef*10^-6,size(TLT,1),1); - if (flgAstigmatism) && (refineCCC(c,3)~=-9999) - TLT(:,12) = repmat(gather(refineCCC(c,2)),size(TLT,1),1); - TLT(:,13) = repmat(gather(refineCCC(c,1)),size(TLT,1),1); + + + + end - % Turn off astigmatism and restrict search range for handedness check. - flgAstigmatism = 0; - - % Sort descending along the magnitude of the tilt angles because higher tilts take - % longer on CTF correction. If more processor available than projections, - % this doesn't affect anything. - [~, idx] = sortrows(abs(TLT(:,4)), -1); - TLT = TLT(idx,:); - % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt - % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast - fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); - fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%8.2f\n'], TLT'); - fclose(fileID); - - elseif iTilt == 2 - measuredVsExpected(2,1) = maxDef; - elseif iTilt == 3 - measuredVsExpected(2,3) = maxDef; - end % Stuff we only do on the full determin (tilt1) - -end % Loop on handedness check -if sum(abs(diff(measuredVsExpected,1))) > sum(abs(measuredVsExpected(1,:) - flip(measuredVsExpected(2,:)))) - warnInvertedHand = 1; -else - warnInvertedHand = 0; -end - -fprintf('\n******************************************************\n\n'); -fprintf('\nCloser to focus |\tAt focus |\tFarther from focus\n\n'); -fprintf('Expected defocus %3.2f %3.2f %3.2f\n\n', abs(measuredVsExpected(1,:))); -fprintf('Measured defocus %3.2f %3.2f %3.2f\n\n' ,abs(measuredVsExpected(2,:))); -if ( warnInvertedHand ) - fprintf('\nIt looks like your handedness may be inverted!!\n'); -else - fprintf('\nIt looks like your handedness is probably correct.\n'); -end -fprintf('\n******************************************************\n\n\n'); - -else - + if ( iTilt == 1) + + radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; + currentDefocusEst = maxDef; + currentDefocusWin = (defWIN*.25); + measuredVsExpected(1,:) = [maxDef + zShift*PIXEL_SIZE*10^6, maxDef, maxDef - zShift*PIXEL_SIZE*10^6]; + measuredVsExpected(2,2) = maxDef; + % Add the determined defocus, and write out with mic paramters as well. + TLT(:,15) = repmat(maxDef*10^-6,size(TLT,1),1); + if (flgAstigmatism) && (refineCCC(c,3)~=-9999) + TLT(:,12) = repmat(gather(refineCCC(c,2)),size(TLT,1),1); + TLT(:,13) = repmat(gather(refineCCC(c,1)),size(TLT,1),1); + end + + % Turn off astigmatism and restrict search range for handedness check. + flgAstigmatism = 0; + + % Sort descending along the magnitude of the tilt angles because higher tilts take + % longer on CTF correction. If more processor available than projections, + % this doesn't affect anything. [~, idx] = sortrows(abs(TLT(:,4)), -1); - TLT = TLT(idx,:); - % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt - % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast - fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); - fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%8.2f\n'], TLT'); - fclose(fileID); - + TLT = TLT(idx,:); + % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt + % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY + % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast + fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); + fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%8.2f\n'], TLT'); + fclose(fileID); + + elseif iTilt == 2 + measuredVsExpected(2,1) = maxDef; + elseif iTilt == 3 + measuredVsExpected(2,3) = maxDef; + end % Stuff we only do on the full determin (tilt1) + end % Loop on handedness check + if sum(abs(diff(measuredVsExpected,1))) > sum(abs(measuredVsExpected(1,:) - flip(measuredVsExpected(2,:)))) + warnInvertedHand = 1; + else + warnInvertedHand = 0; + end + + fprintf('\n******************************************************\n\n'); + fprintf('\nCloser to focus |\tAt focus |\tFarther from focus\n\n'); + fprintf('Expected defocus %3.2f %3.2f %3.2f\n\n', abs(measuredVsExpected(1,:))); + fprintf('Measured defocus %3.2f %3.2f %3.2f\n\n' ,abs(measuredVsExpected(2,:))); + if ( warnInvertedHand ) + fprintf('\nIt looks like your handedness may be inverted!!\n'); + else + fprintf('\nIt looks like your handedness is probably correct.\n'); + end + fprintf('\n******************************************************\n\n\n'); + +else + + [~, idx] = sortrows(abs(TLT(:,4)), -1); + TLT = TLT(idx,:); + % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt + % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY + % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast + fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); + fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%8.2f\n'], TLT'); + fclose(fileID); + + end % end flgSkip - % TODO should I restart the parallel pool - BH_ctf_Refine2(varargin{1},varargin{2}); +% TODO should I restart the parallel pool +BH_ctf_Refine2(varargin{1},varargin{2}); end % end of ctf estimate function function [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, dualAxis) - - if numel(dualAxis) ~= max(size(dualAxis)) - bandpass2d = dualAxis; - dualAxis = 1; - end - - + +if numel(dualAxis) ~= max(size(dualAxis)) + bandpass2d = dualAxis; + dualAxis = 1; +end - paddedSize = size(Hqz, 1); + + +paddedSize = size(Hqz, 1); % if (dualAxis) % rV = Hqz(1+paddedSize/2,1+paddedSize/2:end); % else - rV = Hqz(1,1:paddedSize/2); +rV = Hqz(1,1:paddedSize/2); % end - - % smooth the spectrum a little to fit the zeros, and more to fit the max - % values - rVmin = convn(rV,[0.0180,0.0891,0.2327,0.3204,0.2327,0.0891,0.0180],'same'); - [~,firstAbsMax] = max(abs(rV)); - knots = []; - n = 1; +% smooth the spectrum a little to fit the zeros, and more to fit the max +% values +rVmin = convn(rV,[0.0180,0.0891,0.2327,0.3204,0.2327,0.0891,0.0180],'same'); + +[~,firstAbsMax] = max(abs(rV)); +knots = []; +n = 1; +phs = -1; +while n < length(rV) + + % should start negative + + + if phs < 0 + kn = find(rVmin(n:end) >= 0, 1, 'first'); + if isempty(kn) + break + end + phs = 1; + n = n + kn + 2 ; + knots = [knots; n-3]; + else + kn = find(rVmin(n:end) <= 0, 1, 'first'); + if isempty(kn) + break + end phs = -1; - while n < length(rV) + n = n + kn + 2; + knots = [knots; n-3]; + end + +end - % should start negative +% Take 15% of the first peak +halfPastFirstMax = knots(1) - floor(0.15*(knots(1) - firstAbsMax)); +% halfPastFirstMax = knots(2); - if phs < 0 - kn = find(rVmin(n:end) >= 0, 1, 'first'); - if isempty(kn) - break - end - phs = 1; - n = n + kn + 2 ; - knots = [knots; n-3]; - else - kn = find(rVmin(n:end) <= 0, 1, 'first'); - if isempty(kn) - break - end - phs = -1; - n = n + kn + 2; - knots = [knots; n-3]; - end - end +if (dualAxis) + + + nCone = 1; + + for iCone = 2.5:5:90-2.5 + rImg = BH_resample2d(radialAvg,[0,0,iCone],[0,0,0],'Bah','GPU','inv',1,[paddedSize,paddedSize]); + rAvg = abs(gather(rImg(1+paddedSize/2,1+paddedSize/2:end))); - % Take 15% of the first peak - halfPastFirstMax = knots(1) - floor(0.15*(knots(1) - firstAbsMax)); -% halfPastFirstMax = knots(2); - + try + bg{nCone} = fit(gather(freqVector(knots))', ... + double(gather(rAvg(knots))'), ... + 'smoothingSpline'); + catch + bg{nCone} = fit(gather(freqVector(knots(1:end-1))'), ... + double(gather(rAvg(knots(1:end-1)))'), ... + 'smoothingSpline'); + end + nCone = nCone +1; - if (dualAxis) - - - nCone = 1; + end + +else + + if length(knots) >= 2 + try + bg = fit(gather(freqVector(knots))', ... + double(gather(abs(radialAvg(knots)))'), ... + 'smoothingSpline'); - for iCone = 2.5:5:90-2.5 - rImg = BH_resample2d(radialAvg,[0,0,iCone],[0,0,0],'Bah','GPU','inv',1,[paddedSize,paddedSize]); - rAvg = abs(gather(rImg(1+paddedSize/2,1+paddedSize/2:end))); - - - try - bg{nCone} = fit(gather(freqVector(knots))', ... - double(gather(rAvg(knots))'), ... - 'smoothingSpline'); - catch - bg{nCone} = fit(gather(freqVector(knots(1:end-1))'), ... - double(gather(rAvg(knots(1:end-1)))'), ... - 'smoothingSpline'); - end - nCone = nCone +1; - - end + catch + bg = fit(gather(freqVector(knots(1:end-1)))', ... + double(gather(abs(radialAvg(knots(1:end-1))))'), ... + 'smoothingSpline'); - else - - if length(knots) >= 2 - try - bg = fit(gather(freqVector(knots))', ... - double(gather(abs(radialAvg(knots)))'), ... - 'smoothingSpline'); - - catch - bg = fit(gather(freqVector(knots(1:end-1)))', ... - double(gather(abs(radialAvg(knots(1:end-1))))'), ... - 'smoothingSpline'); - - end - - - else - fprintf('fewer than 2 minima detected\n') - end end - bandpass = false(1,length(freqVector)); - bandpass(halfPastFirstMax:find(freqVector > highCutoff, 1, 'first')) = true; - - if (dualAxis) - avg = zeros(size(freqVector))'; - for iFit = 1:nCone-1 - avg = avg + bg{iFit}(freqVector); - end + + else + fprintf('fewer than 2 minima detected\n') + end +end - avg = avg ./ (nCone-1); +bandpass = false(1,length(freqVector)); +bandpass(halfPastFirstMax:find(freqVector > highCutoff, 1, 'first')) = true; - avgFit = fit(gather(freqVector)',avg,'cubicSpline'); - bg = avgFit; - end - +if (dualAxis) + avg = zeros(size(freqVector))'; + for iFit = 1:nCone-1 + avg = avg + bg{iFit}(freqVector); + end + + avg = avg ./ (nCone-1); - if (dualAxis) + avgFit = fit(gather(freqVector)',avg,'cubicSpline'); + bg = avgFit; +end - freqLow = freqVector(find(bandpass,1,'first')); - freqTop = freqVector(find(bandpass,1,'last')); - bandpass = (bandpass2d > freqLow & bandpass2d < freqTop); - end +if (dualAxis) + + freqLow = freqVector(find(bandpass,1,'first')); + freqTop = freqVector(find(bandpass,1,'last')); + bandpass = (bandpass2d > freqLow & bandpass2d < freqTop); + +end end function [ iCCC ] = calc_CCC( freqVector, bg, bandpass, radialAvg, rV, cccScale) + +if (numel(radialAvg) == max(size(radialAvg))) + % One dimensional case + bgSubPS = (abs(radialAvg) - bg(freqVector)').*bandpass; + %bgSubPS = (bgSubPS - mean(bgSubPS(bandpass))) .* bandpass; + if cccScale == 0.5 + % amplify the contribution of higher frequency information that may + % otherwise be overwhelmed - particularly in CCD images. Only option is to + % take the sqrt prior to normalizing. Provides some balance without really + % risking amplifying too much noise. + bgSubPS = (bgSubPS - min(bgSubPS(:)) + 1).^0.5; + elseif cccScale ~= 1 + error('cccScale must be 1 or 0.5') + end - if (numel(radialAvg) == max(size(radialAvg))) - % One dimensional case - bgSubPS = (abs(radialAvg) - bg(freqVector)').*bandpass; - %bgSubPS = (bgSubPS - mean(bgSubPS(bandpass))) .* bandpass; - if cccScale == 0.5 - % amplify the contribution of higher frequency information that may - % otherwise be overwhelmed - particularly in CCD images. Only option is to - % take the sqrt prior to normalizing. Provides some balance without really - % risking amplifying too much noise. - bgSubPS = (bgSubPS - min(bgSubPS(:)) + 1).^0.5; - elseif cccScale ~= 1 - error('cccScale must be 1 or 0.5') - end - - bgSubPS = bgSubPS ./ max(bgSubPS(:)); -% % % % % bgSubPS = bgSubPS - mean(bgSubPS(:)); + bgSubPS = bgSubPS ./ max(bgSubPS(:)); + % % % % % bgSubPS = bgSubPS - mean(bgSubPS(:)); +else + if isnumeric(bg) + % Use existing (otherwise bg is a cfit object) + bgSubPS = bg; else - if isnumeric(bg) - % Use existing (otherwise bg is a cfit object) - bgSubPS = bg; - else bgSubPS = abs(radialAvg) - reshape(bg(freqVector),size(radialAvg)); - + bgSubPS = bgSubPS .* bandpass; - - S = std2(bgSubPS(bandpass)); + + S = std2(bgSubPS(bandpass)); bgSubPS(bgSubPS > S*2.5) = bgSubPS(bgSubPS>S*2.5).* ... - (rand(gather(sum(bgSubPS(:)>S*2.5)),1)+0.5)./2 ; - -% bgSubPS = bgSubPS - BH_movingAverage(bgSubPS,[8,8]); -% bgSubPS = bgSubPS ./ BH_movingRMS(bgSubPS,[8,8]); - -% % % % % bgSubPS = bgSubPS ./ max(abs(bgSubPS(bandpass))); -% % % % % bgSubPS = (bgSubPS - mean(bgSubPS(bandpass))).*bandpass; + (rand(gather(sum(bgSubPS(:)>S*2.5)),1)+0.5)./2 ; + + % bgSubPS = bgSubPS - BH_movingAverage(bgSubPS,[8,8]); + % bgSubPS = bgSubPS ./ BH_movingRMS(bgSubPS,[8,8]); + + % % % % % bgSubPS = bgSubPS ./ max(abs(bgSubPS(bandpass))); + % % % % % bgSubPS = (bgSubPS - mean(bgSubPS(bandpass))).*bandpass; bgSubPS = bgSubPS ./ max(abs(bgSubPS(bandpass))).*bandpass; - end end +end - - if gather(rV(1)) == -9999 - % return the bgSubPS to save - iCCC = bgSubPS; - else - ctfSQ = abs(rV).*bandpass; -% % % % % ctfSQ = (ctfSQ- mean(ctfSQ(bandpass)).*bandpass); - ctfSQ = ctfSQ ./ max(abs(ctfSQ(:))); - iCCC = sum(sum((bgSubPS .* ctfSQ))) ./ ... - ( numel(ctfSQ(bandpass)).*... - std2(ctfSQ(bandpass)).*... - std2(bgSubPS(bandpass)) ); - end +if gather(rV(1)) == -9999 + % return the bgSubPS to save + iCCC = bgSubPS; +else + ctfSQ = abs(rV).*bandpass; + % % % % % ctfSQ = (ctfSQ- mean(ctfSQ(bandpass)).*bandpass); + ctfSQ = ctfSQ ./ max(abs(ctfSQ(:))); + iCCC = sum(sum((bgSubPS .* ctfSQ))) ./ ... + ( numel(ctfSQ(bandpass)).*... + std2(ctfSQ(bandpass)).*... + std2(bgSubPS(bandpass)) ); + +end end function [ diagnosticIMG ] = make_diagnosticIMG( Hqz, pixelSize, bandpass, bg, IMG) - - iImg = 1; - Hqz = fftshift(Hqz); - paddedSize = size(Hqz,1); - [radialGrid,~,~,~,~,~] = ... - BH_multi_gridCoordinates(size(Hqz),'Cartesian',... - 'GPU',{'none'},1,0,1); - - radialGrid = radialGrid ./ pixelSize; - lowCut = radialGrid(1, find(bandpass , 1,'first')); - highCut= radialGrid(1, find(bandpass , 1,'last')); - - radialGrid = fftshift(radialGrid); - bandpass2d = (radialGrid < highCut & radialGrid > lowCut); - bgSubPS2d = (abs(IMG{iImg}) - reshape(bg(radialGrid),size(Hqz))).*bandpass2d; - diagnosticIMG = zeros(size(Hqz)); - - Hqz = abs(Hqz).* bandpass2d; - Hqz = 1.0.*Hqz ./ max(Hqz(bandpass2d)).*bandpass2d; - diagnosticIMG(1:(paddedSize/2),:,1) = gather(Hqz(1:(paddedSize/2),:)); +iImg = 1; +Hqz = fftshift(Hqz); +paddedSize = size(Hqz,1); +[radialGrid,~,~,~,~,~] = ... + BH_multi_gridCoordinates(size(Hqz),'Cartesian',... + 'GPU',{'none'},1,0,1); + +radialGrid = radialGrid ./ pixelSize; +lowCut = radialGrid(1, find(bandpass , 1,'first')); +highCut= radialGrid(1, find(bandpass , 1,'last')); + +radialGrid = fftshift(radialGrid); +bandpass2d = (radialGrid < highCut & radialGrid > lowCut); +bgSubPS2d = (abs(IMG{iImg}) - reshape(bg(radialGrid),size(Hqz))).*bandpass2d; + +diagnosticIMG = zeros(size(Hqz)); +Hqz = abs(Hqz).* bandpass2d; +Hqz = 1.0.*Hqz ./ max(Hqz(bandpass2d)).*bandpass2d; +diagnosticIMG(1:(paddedSize/2),:,1) = gather(Hqz(1:(paddedSize/2),:)); - bgSubPS2d = bgSubPS2d ./ max(bgSubPS2d(:)); - diagnosticIMG((paddedSize/2)+1:(paddedSize/2)*2,:,1) = gather(bgSubPS2d((paddedSize/2)+1:(paddedSize/2)*2,:)); + +bgSubPS2d = bgSubPS2d ./ max(bgSubPS2d(:)); +diagnosticIMG((paddedSize/2)+1:(paddedSize/2)*2,:,1) = gather(bgSubPS2d((paddedSize/2)+1:(paddedSize/2)*2,:)); end @@ -1296,215 +1296,215 @@ function [ croppedIMG,pixelOUT ] = cropIMG(IMG,pixelIN) - maxRes = 3.3; - targetNyquist = (0.45*maxRes)/0.5; - [d1,d2] = size(IMG); - - [radialGrid] = BH_multi_gridCoordinates( [d1,d2,1],'Cartesian','GPU', ... - {'none'},1,0,1); - - % For now, pixelIN only applies to pre-fixedpattern noise removal, while - % pixelOUT is the desired final cropping. - radialGrid = radialGrid./pixelIN; - rVX = radialGrid(1:ceil((d1+1)/2)); - rVY = radialGrid(1:ceil((d2+1)/2)); - clear radialGrid +maxRes = 3.3; +targetNyquist = (0.45*maxRes)/0.5; +[d1,d2] = size(IMG); - cutX = find(rVX >= 1/targetNyquist, 1,'first'); - cutY = find(rVY >= 1/targetNyquist, 1,'first'); +[radialGrid] = BH_multi_gridCoordinates( [d1,d2,1],'Cartesian','GPU', ... + {'none'},1,0,1); - % The actual nyquist will deviate from the target since it is trunctated - % to some pixel value - newNyquist = 1/rVX(cutX); - pixelOUT = gather(0.5*newNyquist); - +% For now, pixelIN only applies to pre-fixedpattern noise removal, while +% pixelOUT is the desired final cropping. +radialGrid = radialGrid./pixelIN; +rVX = radialGrid(1:ceil((d1+1)/2)); +rVY = radialGrid(1:ceil((d2+1)/2)); +clear radialGrid +cutX = find(rVX >= 1/targetNyquist, 1,'first'); +cutY = find(rVY >= 1/targetNyquist, 1,'first'); + +% The actual nyquist will deviate from the target since it is trunctated +% to some pixel value +newNyquist = 1/rVX(cutX); +pixelOUT = gather(0.5*newNyquist); + + + + +% Prior approach: fftshift, trim (with taper and bandpass), ifftshift +% Trial approach: shift and trim with logical fftMask, ifftshift +fftMask = BH_fftShift([cutX,cutY],[d1,d2],1); +ifftMask = BH_fftShift(0,-2.*[cutX,cutY],1); + +croppedIMG = fftn(gpuArray(IMG)); +croppedIMG = croppedIMG(fftMask); + +croppedIMG = real(ifftn(croppedIMG(ifftMask))); + +clear fftMask ifftMask - - % Prior approach: fftshift, trim (with taper and bandpass), ifftshift - % Trial approach: shift and trim with logical fftMask, ifftshift - fftMask = BH_fftShift([cutX,cutY],[d1,d2],1); - ifftMask = BH_fftShift(0,-2.*[cutX,cutY],1); - croppedIMG = fftn(gpuArray(IMG)); - croppedIMG = croppedIMG(fftMask); - croppedIMG = real(ifftn(croppedIMG(ifftMask))); - - clear fftMask ifftMask - - - end function [ tltOrder ] = calc_dose_scheme(emc,rawTLT,anglesSkipped,PHASE_PLATE_SHIFT) - flgCosineDose = emc.('oneOverCosineDose'); - startingAngle = emc.('startingAngle'); - startingDirection = emc.('startingDirection'); - doseSymmetricIncrement = emc.('doseSymmetricIncrement'); - doseAtMinTilt = emc.('doseAtMinTilt'); - nPrjs = length(rawTLT); - tltOrder = zeros(nPrjs,5); - - nAngle = 2; - - if (doseSymmetricIncrement < 0) - % For doseSymmetricIncrement = 2, 3 deg - % 0, 3, -3, -6, 6, 9 ... - doseSymmetricIncrement = abs(doseSymmetricIncrement); - flgFirstTilt = 0; +flgCosineDose = emc.('oneOverCosineDose'); +startingAngle = emc.('startingAngle'); +startingDirection = emc.('startingDirection'); +doseSymmetricIncrement = emc.('doseSymmetricIncrement'); +doseAtMinTilt = emc.('doseAtMinTilt'); +nPrjs = length(rawTLT); +tltOrder = zeros(nPrjs,5); + +nAngle = 2; + +if (doseSymmetricIncrement < 0) + % For doseSymmetricIncrement = 2, 3 deg + % 0, 3, -3, -6, 6, 9 ... + doseSymmetricIncrement = abs(doseSymmetricIncrement); + flgFirstTilt = 0; +else + % For doseSymmetricIncrement = 2, 3 deg + % 0, 3, 6, -3, -6, 9 ... + flgFirstTilt=1; +end + +if any(PHASE_PLATE_SHIFT) + if diff(PHASE_PLATE_SHIFT) < 1e-3 + PHASE_PLATE_SHIFT(2) = PHASE_PLATE_SHIFT(1) + 1e-3; + end + extraPhaseShift = [PHASE_PLATE_SHIFT(1):(PHASE_PLATE_SHIFT(2) - PHASE_PLATE_SHIFT(1))./nPrjs:PHASE_PLATE_SHIFT(2)]; +else + extraPhaseShift = zeros(nPrjs,1); +end + +totalDose = doseAtMinTilt; + +if (anglesSkipped) + % Get the actual angles from the index + anglesToSkip = rawTLT(anglesSkipped); + anglesToKeep = ~ismember(1:nPrjs,anglesSkipped); +else + anglesToSkip = []; + anglesToKeep = true(nPrjs,1); +end + + +% We always start from the first tilt. +[~,firstTilt] = min(abs(rawTLT-startingAngle)); +tltOrder(firstTilt,:) = [firstTilt,rawTLT(firstTilt),totalDose,extraPhaseShift(1),0]; +% Remove this angle to get those remaining +tmpTLT = rawTLT([1:firstTilt-1,firstTilt+1:end]); +largerAngles = tmpTLT(tmpTLT-startingAngle > 0); +smallerAngles= tmpTLT(tmpTLT-startingAngle < 0); + +% Now split into thos that are larger or smaller than the min tilt +if ( startingAngle >= 0 ) + largerAngles = sort(largerAngles,'ascend'); + smallerAngles = sort(smallerAngles,'descend'); +else + largerAngles = sort(largerAngles,'ascend'); + smallerAngles = sort(smallerAngles,'descend'); +end + +clear tmpTLT + +if ( doseSymmetricIncrement ) + % It is assumed that blocks of this many tilts are collected NOT + % including the first tilt. If the original dose symmetric scheme is + % requested (negative Increment) then the first tilt IS included, and + % so we need to subtract one from the counter. + switchAfterNTilts = doseSymmetricIncrement - (1-flgFirstTilt) + flgFirstTilt=0; +else + if strcmpi(startingDirection,'pos') + switchAfterNTilts = length(largerAngles); + elseif strcmpi(startingDirection,'neg') + switchAfterNTilts = length(smallerAngles); + else + error('flgDose symmetric is 0 and starting direction must be pos or neg'); + end +end + + +% while (~isempty(largerAngles) || ~isempty(smallerAngles)) && nAngle <= nPrjs +for iPrj = 1:nPrjs + if iPrj == firstTilt + continue; + end + + if strcmpi(startingDirection,'pos') + try + nextTilt = largerAngles(1); + if length(largerAngles) > 1 + largerAngles = largerAngles(2:end); + else + largerAngles = []; + end + catch + nextTilt = smallerAngles(1); + if length(smallerAngles) > 1 + smallerAngles = smallerAngles(2:end); + else + smallerAngles = []; + end + end + else - % For doseSymmetricIncrement = 2, 3 deg - % 0, 3, 6, -3, -6, 9 ... - flgFirstTilt=1; + try + nextTilt = smallerAngles(1); + if length(smallerAngles) > 1 + smallerAngles = smallerAngles(2:end); + else + smallerAngles = []; + end + catch + nextTilt = largerAngles(1); + if length(largerAngles) > 1 + largerAngles = largerAngles(2:end); + else + largerAngles = []; + end + end end - if any(PHASE_PLATE_SHIFT) - if diff(PHASE_PLATE_SHIFT) < 1e-3 - PHASE_PLATE_SHIFT(2) = PHASE_PLATE_SHIFT(1) + 1e-3; + [~,iTilt] = min(abs(rawTLT-nextTilt)); + + switchAfterNTilts = switchAfterNTilts -1; + + if (switchAfterNTilts == 0) + if strcmpi(startingDirection,'pos') + startingDirection = 'neg'; + elseif strcmpi(startingDirection,'neg') + startingDirection = 'pos'; + end + if ( doseSymmetricIncrement ) + switchAfterNTilts = doseSymmetricIncrement; end - extraPhaseShift = [PHASE_PLATE_SHIFT(1):(PHASE_PLATE_SHIFT(2) - PHASE_PLATE_SHIFT(1))./nPrjs:PHASE_PLATE_SHIFT(2)]; + end + + if (flgCosineDose) + totalDose = totalDose + (1/cosd(rawTLT(iTilt)))*doseAtMinTilt; else - extraPhaseShift = zeros(nPrjs,1); + totalDose = totalDose + doseAtMinTilt; end - totalDose = doseAtMinTilt; - if (anglesSkipped) - % Get the actual angles from the index - anglesToSkip = rawTLT(anglesSkipped); - anglesToKeep = ~ismember(1:nPrjs,anglesSkipped); + % The dose is incremented but don't add to the list. + + if ~ismember(rawTLT(iTilt),anglesToSkip) + tltOrder(iTilt,:) = [iTilt,rawTLT(iTilt),totalDose,extraPhaseShift(nAngle),-1]; else - anglesToSkip = []; - anglesToKeep = true(nPrjs,1); + tltOrder(iTilt,:) = [iTilt,rawTLT(iTilt),-1,-1,-1]; end + nAngle = nAngle + 1; + +end - % We always start from the first tilt. - [~,firstTilt] = min(abs(rawTLT-startingAngle)); - tltOrder(firstTilt,:) = [firstTilt,rawTLT(firstTilt),totalDose,extraPhaseShift(1),0]; - % Remove this angle to get those remaining - tmpTLT = rawTLT([1:firstTilt-1,firstTilt+1:end]); - largerAngles = tmpTLT(tmpTLT-startingAngle > 0); - smallerAngles= tmpTLT(tmpTLT-startingAngle < 0); - - % Now split into thos that are larger or smaller than the min tilt - if ( startingAngle >= 0 ) - largerAngles = sort(largerAngles,'ascend'); - smallerAngles = sort(smallerAngles,'descend'); - else - largerAngles = sort(largerAngles,'ascend'); - smallerAngles = sort(smallerAngles,'descend'); - end - - clear tmpTLT - - if ( doseSymmetricIncrement ) - % It is assumed that blocks of this many tilts are collected NOT - % including the first tilt. If the original dose symmetric scheme is - % requested (negative Increment) then the first tilt IS included, and - % so we need to subtract one from the counter. - switchAfterNTilts = doseSymmetricIncrement - (1-flgFirstTilt) - flgFirstTilt=0; - else - if strcmpi(startingDirection,'pos') - switchAfterNTilts = length(largerAngles); - elseif strcmpi(startingDirection,'neg') - switchAfterNTilts = length(smallerAngles); - else - error('flgDose symmetric is 0 and starting direction must be pos or neg'); - end - end - - -% while (~isempty(largerAngles) || ~isempty(smallerAngles)) && nAngle <= nPrjs - for iPrj = 1:nPrjs - if iPrj == firstTilt - continue; - end - - if strcmpi(startingDirection,'pos') - try - nextTilt = largerAngles(1); - if length(largerAngles) > 1 - largerAngles = largerAngles(2:end); - else - largerAngles = []; - end - catch - nextTilt = smallerAngles(1); - if length(smallerAngles) > 1 - smallerAngles = smallerAngles(2:end); - else - smallerAngles = []; - end - end - - else - try - nextTilt = smallerAngles(1); - if length(smallerAngles) > 1 - smallerAngles = smallerAngles(2:end); - else - smallerAngles = []; - end - catch - nextTilt = largerAngles(1); - if length(largerAngles) > 1 - largerAngles = largerAngles(2:end); - else - largerAngles = []; - end - end - end - - [~,iTilt] = min(abs(rawTLT-nextTilt)); - - switchAfterNTilts = switchAfterNTilts -1; - - if (switchAfterNTilts == 0) - if strcmpi(startingDirection,'pos') - startingDirection = 'neg'; - elseif strcmpi(startingDirection,'neg') - startingDirection = 'pos'; - end - if ( doseSymmetricIncrement ) - switchAfterNTilts = doseSymmetricIncrement; - end - end - - if (flgCosineDose) - totalDose = totalDose + (1/cosd(rawTLT(iTilt)))*doseAtMinTilt; - else - totalDose = totalDose + doseAtMinTilt; - end - - % The dose is incremented but don't add to the list. +tltOrder = tltOrder(anglesToKeep,:); +tltOrder(:,5) = tltOrder(:,1); +tltOrder(:,1) = 1:size(tltOrder,1); +tltOrder - if ~ismember(rawTLT(iTilt),anglesToSkip) - tltOrder(iTilt,:) = [iTilt,rawTLT(iTilt),totalDose,extraPhaseShift(nAngle),-1]; - else - tltOrder(iTilt,:) = [iTilt,rawTLT(iTilt),-1,-1,-1]; - end - nAngle = nAngle + 1; +size(tltOrder) - end - - - tltOrder = tltOrder(anglesToKeep,:); - tltOrder(:,5) = tltOrder(:,1); - tltOrder(:,1) = 1:size(tltOrder,1); - tltOrder - size(tltOrder) +% This will be a naive run through that works only if the angles are in +% order. - - - % This will be a naive run through that works only if the angles are in - % order. - end - + diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index f530312d..00fb1119 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -1,7 +1,7 @@ function [ cccStorage, maxAst, maxAng, astigAngSearch] = BH_ctf_Refine2(PARAMETER_FILE, STACK_PRFX) % Script to test refinement of ctf estimate by scaling tiles from tilted images % to change their nominal magnification so that the defocus matches that of the -% mean +% mean % Load in the tomo and tilt info emc = BH_parseParameterFile(PARAMETER_FILE); @@ -24,8 +24,8 @@ % As long as the material coming into view at high tilt is at the same % plane and does not have wildly different image stats, using it could % improve the thon rings on tilted data. Zero will produce the "normal" -% process, 1 will use the same area as the min tilt, -try +% process, 1 will use the same area as the min tilt, +try fraction_of_extra_tilt_data = emc.('fraction_of_extra_tilt_data') catch fraction_of_extra_tilt_data = 0.25 @@ -78,7 +78,7 @@ Cs = 5e-6; end -% Assuming that the first CTF zero is always less than this value +% Assuming that the first CTF zero is always less than this value FIXED_FIRSTZERO = PIXEL_SIZE / 40*10^-10 ; % Size to padTile to should be even, large, and preferably a power of 2 @@ -120,15 +120,15 @@ INPUT_CELL{iStack,4} = fileName; INPUT_CELL{iStack,5} = extension; %INPUT_CELL{iStack,6} = PRJ_OUT; - + else fprintf('ignoring %s, because the file is not found.\n', tlt{iStack}) end -end +end for iStack = 1%stacksFound - + STACK = single(getVolume(MRCImage(INPUT_CELL{iStack,2}))); % The pixel size should be previously set correctly, but if it is not, then we % must maintain whatever is there in case beads are to be erased. The model @@ -137,291 +137,291 @@ [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); iPixelHeader = EMC_str2double(iPixelHeader); [d1,d2,d3] = size(STACK) - - - + + + TLT = INPUT_CELL{iStack,1}; pathName = INPUT_CELL{iStack,3}; fileName = INPUT_CELL{iStack,4}; extension = INPUT_CELL{iStack,5}; - - + + SIZEOUT = [d1,d2]; - + [radialForCTF,phi,~,~,~,~] = ... BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU',{'none'},1,1,0); - - - radialForCTF = {radialForCTF./PIXEL_SIZE,1,phi} ; - clear phi - + + radialForCTF = {radialForCTF./PIXEL_SIZE,1,phi} ; + + clear phi + clear sumVector radialAvg sumVector(length(freqVector)) = gpuArray(double(0)); radialAvg(length(freqVector)) = gpuArray(double(0)); - + tic - + psTile = zeros([paddedSize,paddedSize,d3],'single'); - + flgReplaceStack = 0; for iPrj = 1:d3 iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); iProjection = iProjection - ... - BH_movingAverage(iProjection,[tileSize,tileSize]); + BH_movingAverage(iProjection,[tileSize,tileSize]); iProjection = iProjection ./ ... - BH_movingRMS(iProjection,[tileSize,tileSize]); + BH_movingRMS(iProjection,[tileSize,tileSize]); % Taking a cue from Alexis maxPixelSizeWanted = 2.0e-10; if TLT(iPrj,16) < maxPixelSizeWanted - %fprintf(ftmp,'Resampling pixel size\n'); - % Resample to 2Ang/pix - padSq = BH_multi_padVal(size(iProjection),max(size(iProjection)).*[1,1]); - - iProjection = BH_padZeros3d(iProjection,padSq(1,:),padSq(2,:),'GPU','singleTaper'); - sizeIN = size(iProjection,1); - % Replace with BH_fftShift if this works - iProjection = fftshift(fftn(iProjection)); - trimVal = BH_multi_padVal(size(iProjection), floor(size(iProjection).*(TLT(iPrj,16)./maxPixelSizeWanted))); - iProjection = real(ifftn(ifftshift(BH_padZeros3d(iProjection,trimVal(1,:),trimVal(2,:),'GPU','single')))); - sizeOUT = size(iProjection); - if iPrj == 1 - flgReplaceStack = 1; - newSTACK = zeros([sizeOUT,d3],'single'); - end - newSTACK(:,:,TLT(iPrj,1)) = gather(iProjection); - clear iProjection - % Actual new pixel size - pixelSize = sizeIN./sizeOUT(1).*TLT(iPrj,16); - - % Update the CTF params with the new pixelSize - ctfParams(1) = pixelSize.*10^10; - - %fprintf(ftmp,'%d %d %d %d %d %d\n',trimVal); - %fprintf(ftmp,'pixelOld %3.3e, pixelNew %3.3e\n',TLT(iPrj,16),pixelSize); - - - else - pixelSize = TLT(iPrj,16); + %fprintf(ftmp,'Resampling pixel size\n'); + % Resample to 2Ang/pix + padSq = BH_multi_padVal(size(iProjection),max(size(iProjection)).*[1,1]); + + iProjection = BH_padZeros3d(iProjection,padSq(1,:),padSq(2,:),'GPU','singleTaper'); + sizeIN = size(iProjection,1); + % Replace with BH_fftShift if this works + iProjection = fftshift(fftn(iProjection)); + trimVal = BH_multi_padVal(size(iProjection), floor(size(iProjection).*(TLT(iPrj,16)./maxPixelSizeWanted))); + iProjection = real(ifftn(ifftshift(BH_padZeros3d(iProjection,trimVal(1,:),trimVal(2,:),'GPU','single')))); + sizeOUT = size(iProjection); + if iPrj == 1 + flgReplaceStack = 1; + newSTACK = zeros([sizeOUT,d3],'single'); end - end % iPrj 1:d3 - - if ( flgReplaceStack ) - STACK = newSTACK ; clear newSTACK; + newSTACK(:,:,TLT(iPrj,1)) = gather(iProjection); + clear iProjection + % Actual new pixel size + pixelSize = sizeIN./sizeOUT(1).*TLT(iPrj,16); + + % Update the CTF params with the new pixelSize + ctfParams(1) = pixelSize.*10^10; + + %fprintf(ftmp,'%d %d %d %d %d %d\n',trimVal); + %fprintf(ftmp,'pixelOld %3.3e, pixelNew %3.3e\n',TLT(iPrj,16),pixelSize); + + + else + pixelSize = TLT(iPrj,16); end - [d1,d2,d3] = size(STACK); - [Xnew, Ynew, ~, x1,y1, ~] = BH_multi_gridCoordinates([tileSize,d2], ... + end % iPrj 1:d3 + + if ( flgReplaceStack ) + STACK = newSTACK ; clear newSTACK; + end + [d1,d2,d3] = size(STACK); + [Xnew, Ynew, ~, x1,y1, ~] = BH_multi_gridCoordinates([tileSize,d2], ... 'Cartesian','GPU', ... {'none'},0,1,0); - - [X, Y, ~,~,~, ~] = BH_multi_gridCoordinates([tileSize,tileSize], ... + + [X, Y, ~,~,~, ~] = BH_multi_gridCoordinates([tileSize,tileSize], ... 'Cartesian','GPU', ... {'none'},0,1,0); - - coordShift = (-1).^(X+Y); - clear X Y - - try - ppool = EMC_parpool(nWorkers); - catch - delete(gcp('nocreate')); - ppool = EMC_parpool(nWorkers); - end - - for iPrj = 1:d3 - - pFuture(iPrj) = parfeval(ppool,@runAvgTiles,2, TLT, paddedSize, tileSize, ... - d1,d2, iPrj, overlap, ... - STACK(:,:,TLT(iPrj,1)), ... - 1, ... - 1, ... - x1, y1, Xnew, Ynew,coordShift, ... - reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); - - - end - for iWorker = 1:d3 - fprintf('Refining defocus on prj %d/ %d\n',iWorker,d3); - [iPrj, ctfCorr,pixelSize] = fetchNext(pFuture); - - psTile(:,:,TLT(iPrj,1)) = ctfCorr; - end - - pixelSize = pixelSize*10^10; - SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS.mrc',fileName),pixelSize); - bpLog = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); - bpLog = bpLog > 0.99; - bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); - bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); + coordShift = (-1).^(X+Y); + clear X Y - for iPrj = 1:d3 - iTile = gpuArray(psTile(:,:,iPrj)); - iTile = iTile.*bp.*bp2; - iTile(~bpLog) = mean(iTile(bpLog)); - psTile(:,:,iPrj) = gather(iTile); - - end - - SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName),pixelSize); - delete(ppool); - delete(gcp('nocreate')) + try + ppool = EMC_parpool(nWorkers); + catch + delete(gcp('nocreate')); + ppool = EMC_parpool(nWorkers); + end - % exit an fit the PS using CTFFIND4 - BH_runCtfFind(sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName), ... - sprintf('%s_ctf.tlt',fileName), ctfParams,TLT) - - + for iPrj = 1:d3 + + pFuture(iPrj) = parfeval(ppool,@runAvgTiles,2, TLT, paddedSize, tileSize, ... + d1,d2, iPrj, overlap, ... + STACK(:,:,TLT(iPrj,1)), ... + 1, ... + 1, ... + x1, y1, Xnew, Ynew,coordShift, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); + + + end + + for iWorker = 1:d3 + fprintf('Refining defocus on prj %d/ %d\n',iWorker,d3); + [iPrj, ctfCorr,pixelSize] = fetchNext(pFuture); + + psTile(:,:,TLT(iPrj,1)) = ctfCorr; end + + pixelSize = pixelSize*10^10; + SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS.mrc',fileName),pixelSize); + bpLog = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); + bpLog = bpLog > 0.99; + bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); + bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); + + for iPrj = 1:d3 + iTile = gpuArray(psTile(:,:,iPrj)); + iTile = iTile.*bp.*bp2; + iTile(~bpLog) = mean(iTile(bpLog)); + psTile(:,:,iPrj) = gather(iTile); + + end + + SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName),pixelSize); + delete(ppool); + delete(gcp('nocreate')) + + % exit an fit the PS using CTFFIND4 + BH_runCtfFind(sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName), ... + sprintf('%s_ctf.tlt',fileName), ctfParams,TLT) + + +end end function [psTile,pixelSize] = runAvgTiles(TLT, paddedSize, tileSize, d1,d2, iPrj, overlap, ... - iProjection, evalMask, ... - ddZ, x1, y1, Xnew, Ynew, coordShift, ... - reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine) + iProjection, evalMask, ... + ddZ, x1, y1, Xnew, Ynew, coordShift, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine) - DFo = TLT(iPrj,15); +DFo = TLT(iPrj,15); - padTileOver = 256; - tmpTile = zeros(paddedSize.*[1,1]+2*padTileOver,'single','gpuArray'); - - tiltOrigin = ceil((size(iProjection,1)+1)./2); - - oXprj = ceil((size(iProjection,1)+1)./2); - % Don't worry about extending the edges for thickness - half_width = (size(iProjection,1)/2); +padTileOver = 256; +tmpTile = zeros(paddedSize.*[1,1]+2*padTileOver,'single','gpuArray'); - maxEval = (fraction_of_extra_tilt_data + ... - cosd(TLT(iPrj,4)).*(1-fraction_of_extra_tilt_data)) .* half_width; +tiltOrigin = ceil((size(iProjection,1)+1)./2); - iEvalMask = floor(oXprj-maxEval):ceil(oXprj+maxEval); - - % Since I'm enforcing Y-tilt axis, then this could be dramatically sped up - % by resampling strips along the sampling +oXprj = ceil((size(iProjection,1)+1)./2); +% Don't worry about extending the edges for thickness +half_width = (size(iProjection,1)/2); - for iOuter = 1+tileSize/2:overlap:d1-tileSize/2 - randSize = randi(floor(overlap/2),1); - if (randi(2,1) == 2) - randSize = -1*randSize; - end - i = iOuter + randSize; - if (i < tileSize/2 || i > d1-tileSize/2) - continue; - end +maxEval = (fraction_of_extra_tilt_data + ... + cosd(TLT(iPrj,4)).*(1-fraction_of_extra_tilt_data)) .* half_width; + +iEvalMask = floor(oXprj-maxEval):ceil(oXprj+maxEval); + +% Since I'm enforcing Y-tilt axis, then this could be dramatically sped up +% by resampling strips along the sampling + +for iOuter = 1+tileSize/2:overlap:d1-tileSize/2 + randSize = randi(floor(overlap/2),1); + if (randi(2,1) == 2) + randSize = -1*randSize; + end + i = iOuter + randSize; + if (i < tileSize/2 || i > d1-tileSize/2) + continue; + end + + % Slightly randomize the step size to avoid a Moire like effect that + % presents particulary strongly with a continuous carbon layer. + + iDeltaZ = (i - tiltOrigin)*pixelSize*-1.*tand(TLT(iPrj,4)); + if any(ismember(i-tileSize/2+1:i+tileSize/2,iEvalMask)) %evalMask(i,paddedSize/2+1) + + mag = (1+iDeltaZ./DFo).^0.5; + + estSize = 2048; + ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); + ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),iDeltaZ+DFo,estSize,TLT(iPrj,19),-1,1); + ctf1 = ctf1(1:estSize/2); + ctf2 = ctf2(1:estSize/2); + firstZero = find(ctf1 > 0, 1, 'first'); + % secondZero= find(ctf1(firstZero:end) < 0 , 1, 'first') + firstZero - 1; + + %fprintf(ftmp,'firstZero %d %2.2f\n',firstZero,estSize/firstZero*pixelSize); + % This range will depend on the size of the field of view. For now, setting manually for Florian's HIV + % data, but will derive a formula to make sure the search is appropriate. Here we expect at most ~ 300 nm + % deltaZ, the strongest difference is at the lowest defocus which is ~ 1500 nm, which gives an estimated mag + % ~ 1.095 + defRange = mag-.1:.001:mag+.1; + nDef = length(defRange); + scoreDef = zeros(nDef,1,'gpuArray'); + for iDef = 1:nDef + ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); + % Larger scalings will have zeros rather than extroplation, so% + % % don't let this influence the score. + % lastZero = find(abs(ci) > 0 , 1, 'last'); + %fprintf(ftmp,'%d %d %d %d',size(ci),size(ctf1)); + scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - % Slightly randomize the step size to avoid a Moire like effect that - % presents particulary strongly with a continuous carbon layer. + end + [~,maxCoord] = max(scoreDef); + mag = defRange(maxCoord); + + defRange = mag-.01:.0001:mag+.01; + nDef = length(defRange); + scoreDef = zeros(nDef,1,'gpuArray'); + for iDef = 1:nDef + ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); + scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); + end + [~,maxCoord] = max(scoreDef); + mag = defRange(maxCoord); + + if (testNoRefine) + mag = 1; + end + + scaledStrip = iProjection(i-tileSize/2+1:i+tileSize/2,:); + + for j = 1+tileSize/2:overlap:d2-tileSize/2 - iDeltaZ = (i - tiltOrigin)*pixelSize*-1.*tand(TLT(iPrj,4)); - if any(ismember(i-tileSize/2+1:i+tileSize/2,iEvalMask)) %evalMask(i,paddedSize/2+1) - - mag = (1+iDeltaZ./DFo).^0.5; - - estSize = 2048; - ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); - ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),iDeltaZ+DFo,estSize,TLT(iPrj,19),-1,1); - ctf1 = ctf1(1:estSize/2); - ctf2 = ctf2(1:estSize/2); - firstZero = find(ctf1 > 0, 1, 'first'); - % secondZero= find(ctf1(firstZero:end) < 0 , 1, 'first') + firstZero - 1; - - %fprintf(ftmp,'firstZero %d %2.2f\n',firstZero,estSize/firstZero*pixelSize); - % This range will depend on the size of the field of view. For now, setting manually for Florian's HIV - % data, but will derive a formula to make sure the search is appropriate. Here we expect at most ~ 300 nm - % deltaZ, the strongest difference is at the lowest defocus which is ~ 1500 nm, which gives an estimated mag - % ~ 1.095 - defRange = mag-.1:.001:mag+.1; - nDef = length(defRange); - scoreDef = zeros(nDef,1,'gpuArray'); - for iDef = 1:nDef - ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); - % Larger scalings will have zeros rather than extroplation, so% - % % don't let this influence the score. - % lastZero = find(abs(ci) > 0 , 1, 'last'); - %fprintf(ftmp,'%d %d %d %d',size(ci),size(ctf1)); - scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - - end - [~,maxCoord] = max(scoreDef); - mag = defRange(maxCoord); - - defRange = mag-.01:.0001:mag+.01; - nDef = length(defRange); - scoreDef = zeros(nDef,1,'gpuArray'); - for iDef = 1:nDef - ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); - scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - end - [~,maxCoord] = max(scoreDef); - mag = defRange(maxCoord); - - if (testNoRefine) - mag = 1; + iTile = scaledStrip(:,j-tileSize/2+1:j+tileSize/2);%.*coordShift; + if (reScaleRealSpace) + scaledSize = paddedSize; + else + % Slightly randomize scaling + if (randi(2,1) == 2) + scaledSize = ceil(paddedSize .* mag) + randi(2,1) -1; + else + scaledSize = floor(paddedSize .* mag)+ randi(2,1) -1; end + %scaledSize = floor(paddedSize ./ mag); + end + + [oX,oY] = size(tmpTile); + oX = ceil((oX+1)./2); + oY = ceil((oY+1)./2); + + iPadVal = BH_multi_padVal(size(iTile),[scaledSize,scaledSize]); + + + oupSize = [floor(scaledSize./2),ceil(scaledSize./2); ... + floor(scaledSize./2),ceil(scaledSize./2)]; + + % Get rid of th fftshift + tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... + oY-oupSize(2,1):oY+oupSize(2,2)-1) = ... + tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... + oY-oupSize(2,1):oY+oupSize(2,2)-1) + ... + fftshift(abs(fftn(BH_padZeros3d(iTile,iPadVal(1,:),iPadVal(2,:), ... + 'GPU','singleTaper', mean(iTile(:)))))); + + end % loop over j + end % if over eval mask +end % over tiles - scaledStrip = iProjection(i-tileSize/2+1:i+tileSize/2,:); - - for j = 1+tileSize/2:overlap:d2-tileSize/2 - - iTile = scaledStrip(:,j-tileSize/2+1:j+tileSize/2);%.*coordShift; - if (reScaleRealSpace) - scaledSize = paddedSize; - else - % Slightly randomize scaling - if (randi(2,1) == 2) - scaledSize = ceil(paddedSize .* mag) + randi(2,1) -1; - else - scaledSize = floor(paddedSize .* mag)+ randi(2,1) -1; - end - %scaledSize = floor(paddedSize ./ mag); - end - - [oX,oY] = size(tmpTile); - oX = ceil((oX+1)./2); - oY = ceil((oY+1)./2); - - iPadVal = BH_multi_padVal(size(iTile),[scaledSize,scaledSize]); - - - oupSize = [floor(scaledSize./2),ceil(scaledSize./2); ... - floor(scaledSize./2),ceil(scaledSize./2)]; - - % Get rid of th fftshift - tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... - oY-oupSize(2,1):oY+oupSize(2,2)-1) = ... - tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... - oY-oupSize(2,1):oY+oupSize(2,2)-1) + ... - fftshift(abs(fftn(BH_padZeros3d(iTile,iPadVal(1,:),iPadVal(2,:), ... - 'GPU','singleTaper', mean(iTile(:)))))); - - end % loop over j - end % if over eval mask - end % over tiles - - - psTile = gather(BH_padZeros3d(tmpTile, [-1,-1].* ... - padTileOver,[-1,-1].*padTileOver,... - 'GPU','single')); - clear tmpTile iProjection ddZ evalMask Xnew Ynew x1 y1 + +psTile = gather(BH_padZeros3d(tmpTile, [-1,-1].* ... + padTileOver,[-1,-1].*padTileOver,... + 'GPU','single')); +clear tmpTile iProjection ddZ evalMask Xnew Ynew x1 y1 end - + function [minRes] = calcMinResolution(TLT, radialForCTF,Cs,WAVELENGTH,AMPCONT) - meanDef = mean(TLT(:,15)); - meanAst = mean(TLT(:,12)); - meanAng = mean(TLT(:,13)); +meanDef = mean(TLT(:,15)); +meanAst = mean(TLT(:,12)); +meanAng = mean(TLT(:,13)); - df1 = meanDef - meanAst; - df2 = meanDef + meanAst; +df1 = meanDef - meanAst; +df2 = meanDef + meanAst; - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,meanAng],size(radialForCTF{1}), ... - AMPCONT,-1.0); - rV = Hqz(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); - freqVector = radialForCTF{1}(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); - firstZero = find(rV > 0, 1,'first'); +[ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [df1,df2,meanAng],size(radialForCTF{1}), ... + AMPCONT,-1.0); +rV = Hqz(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); +freqVector = radialForCTF{1}(1+size(Hqz, 1)/2,1+size(Hqz, 1)/2:end); +firstZero = find(rV > 0, 1,'first'); - minRes = 1/freqVector(firstZero) * 10^10; +minRes = 1/freqVector(firstZero) * 10^10; end diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index b4a0f280..629a347e 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -19,15 +19,15 @@ flgShiftEucentric = 0; end try - % Should be negative, but to test. - defShiftSign = emc.('testFlipSign'); + % Should be negative, but to test. + defShiftSign = emc.('testFlipSign'); catch - defShiftSign = -1; + defShiftSign = -1; end try load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR; + mapBackIter = subTomoMeta.currentTomoCPR; catch mapBackIter = 0; end @@ -40,14 +40,14 @@ ITER_LIST = {STACK_LIST}; else flgParallel = 1; - + updateCMD = sprintf('%s,%d,TiltAlignment,UpdateTilts,[%d,0,0],STD', ... - PARAMETER_FILE,subTomoMeta.currentCycle, ... - subTomoMeta.currentCycle); - % fprintf('%s/n',updateCMD); + PARAMETER_FILE,subTomoMeta.currentCycle, ... + subTomoMeta.currentCycle); + % fprintf('%s/n',updateCMD); nGPUs = emc.('nGPUs'); ITER_LIST = cell(nGPUs,1); - + [STACK_LIST, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); clear STACK_LIST_tmp for iGPU = 1:nGPUs @@ -84,8 +84,8 @@ % loop, getting confused about whether it is a variable or a function. recGeomForThickness = subTomoMeta.reconGeometry; parfor iGPU = 1:nGPUs -% for iTilt = 1:length(ITER_LIST{iGPU}) - + % for iTilt = 1:length(ITER_LIST{iGPU}) + if ( flgParallel ) useGPU = iGPU; gDev = gpuDevice(useGPU); @@ -95,16 +95,16 @@ end for iTilt = 1:length(ITER_LIST{iGPU}) - + STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - + if (mapBackIter) mapBackPrfx = sprintf('mapBack%d/%s_ali%d_ctf',mapBackIter,STACK_PRFX,mapBackIter) else mbEST=''; end - - + + if strcmpi(applyFullorUpdate, 'full') % Combine old and new transformations and apply as well as erasing beads, % e.g. go from raw stack to preCTF. @@ -113,21 +113,21 @@ PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); - outputDirectory = 'aliStacks' + outputDirectory = 'aliStacks' elseif strcmpi(applyFullorUpdate, 'fullScale') - % Combine old and new transformations and apply as well as erasing beads, - % e.g. go from raw stack to preCTF. - - % Also remove shifts due to the sample being non-eucentric. This is a - % test, and if it helps, it would be even better to just apply this shift - % to all the subtomos pre-emptivel. - flgSkipErase = 0; - flgApplyFullXform = 1; - PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} - PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} - PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); - - outputDirectory = 'aliStacks'; + % Combine old and new transformations and apply as well as erasing beads, + % e.g. go from raw stack to preCTF. + + % Also remove shifts due to the sample being non-eucentric. This is a + % test, and if it helps, it would be even better to just apply this shift + % to all the subtomos pre-emptivel. + flgSkipErase = 0; + flgApplyFullXform = 1; + PRJ_STACK ={sprintf('fixedStacks/%s.fixed',STACK_PRFX)} + PRJ_OUT = {sprintf('%s_ali%d',STACK_PRFX,mapBackIter+1)} + PRJ_OLD = sprintf('%s_ali%d',STACK_PRFX,mapBackIter); + + outputDirectory = 'aliStacks'; elseif strcmpi(applyFullorUpdate, 'refine') % Don't combine, just use tlt with new defocus values and erase beads @@ -151,290 +151,290 @@ outputDirectory = 'ctfStacks'; else - error('applyFullorUpdate should be [full], [fullScale] or [update]') - end - - - tlt = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+flgInitResample)}; - tlt_OUT = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1)}; - - eraseStack = sprintf('rm cache/%s_*.fixed',STACK_PRFX); - eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); - % Converte bead diameter to pixels and add a little to be safe. - PIXEL_SIZE = emc.('PIXEL_SIZE'); - SuperResolution = emc.('SuperResolution'); - - % Don't apply any fourier cropping of super-res data if only updating, - % as it would already be done. - if strcmpi(applyFullorUpdate,'update') - SuperResolution = 0; - end - - if (SuperResolution) - % Transform the raw images at full sampling then crop the fft to physical - % nyquist - PIXEL_SIZE = 2.* PIXEL_SIZE; - end - - eraseSigma = 3;%emc.('beadSigma'); - - eraseRadius = ceil(1.2.*(emc.('beadDiameter')./PIXEL_SIZE.*0.5)); - flgImodErase = 0 - - % FIXME, this should be stored from previous mask calc and accessed there. - % For now just take based on tomogram (which will be larger than the true specimen thickness) - %THICKNESS = recGeomForThickness.(sprintf('%s_1',STACK_PRFX)); - %THICKNESS = min(10,abs(THICKNESS(1,3)-THICKNESS(2,3)).*PIXEL_SIZE.*10^9); - THICKNESS = 100; - % Assuming all extreme pixels have already been removed from the stack. - %PRJ_STACK = {sprintf('%s_local04_18.mrc',mjIDX)};%,sprintf('%s_local14_18.mrc',mjIDX),sprintf('%s_local24_18.mrc',mjIDX),sprintf('%s_local34_18.mrc',mjIDX)}; - nStacks = length(tlt); - INPUT_CELL = cell(nStacks,7); - TLT_Trans = cell(nStacks,1); - - % killed the loop, clean up later - if exist(tlt{1}, 'file') && exist(PRJ_STACK{1}, 'file') - INPUT_CELL{1,1} = load(tlt{1}); - INPUT_CELL{1,2} = PRJ_STACK{1}; - [pathName,fileName,extension] = fileparts(PRJ_STACK{1}); - if isempty(pathName) - pathName = '.'; + error('applyFullorUpdate should be [full], [fullScale] or [update]') end - [ctfPath,~,~] = fileparts(tlt{1}); - INPUT_CELL{1,3} = pathName; - INPUT_CELL{1,4} = fileName; - INPUT_CELL{1,5} = extension; - INPUT_CELL{1,6} = PRJ_OUT{1}; - INPUT_CELL{1,7} = ctfPath; - else - if ~exist(tlt{1}, 'file') - fprintf('\nignoring %s, because the file is not found.\n', tlt{1}); - end - if ~exist(PRJ_STACK{1}, 'file') - fprintf('\nignoring %s, because the file is not found.\n',PRJ_STACK{1}); - end - - end - - - % FIXME: removed the loop, clean up later cells etc later - iStack=1; - - iMrcObj = MRCImage(INPUT_CELL{iStack,2},0); - - % The pixel size should be previously set correctly, but if it is not, then we - % must maintain whatever is there in case beads are to be erased. The model - % used for this process depends on the pixel size in the header when it was - % created in IMod alignment. - - iHeader = getHeader(iMrcObj); - iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* (1+abs(SuperResolution)), ... - iHeader.cellDimensionY/iHeader.nY .* (1+abs(SuperResolution)), ... - iHeader.cellDimensionZ/iHeader.nZ]; - - iOriginHeader= [iHeader.xOrigin , ... - iHeader.yOrigin , ... - iHeader.zOrigin ] ./ (1+abs(SuperResolution)); - - d1 = iHeader.nX; d2 = iHeader.nY; d3 = size(INPUT_CELL{iStack,1},1);%iHeader.nZ; - - osX = 1-mod(d1,2); osY = 1-mod(d2,2); - - if (SuperResolution) - gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,-0.235,'GPU','nyquistHigh'); - else - gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); - end - - TLT = INPUT_CELL{iStack,1}; - pathName = INPUT_CELL{iStack,3} - fileName = INPUT_CELL{iStack,4} - extension = INPUT_CELL{iStack,5} - - - % Optionally address magnification changes. - % system(sprintf('mkdir -p %s/recon',INPUT_CELL{i,3})); - system('mkdir -p aliStacks'); - - if (mapBackIter) - fprintf('Combining tranformations\n\n'); - % Load in the mapBack alignment - try - mbEST = load(sprintf('%s.tltxf',mapBackPrfx)); - catch - error('WARNING: did not load %s.tltxf, cannot update alignments',mapBackPrfx) - system(sprintf('cp fixedStacks/ctf/%s_ali1_ctf.tlt fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,STACK_PRFX,mapBackIter+1)); - continue; + + + tlt = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+flgInitResample)}; + tlt_OUT = {sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1)}; + + eraseStack = sprintf('rm cache/%s_*.fixed',STACK_PRFX); + eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); + % Converte bead diameter to pixels and add a little to be safe. + PIXEL_SIZE = emc.('PIXEL_SIZE'); + SuperResolution = emc.('SuperResolution'); + + % Don't apply any fourier cropping of super-res data if only updating, + % as it would already be done. + if strcmpi(applyFullorUpdate,'update') + SuperResolution = 0; end - mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); - defShifts = sprintf('%s.defShifts',mapBackPrfx); - if exist(defShifts,'file') - % tomoCPR is now using mexCTF so updated to def > 0 and in Angstrom, - % which are added to the base value. - % ctf 3d is still using orig def < 0 and in SI so convert here - defShifts = load(defShifts) .* (defShiftSign*10^-10); - fprintf('Updating defocus shifts from tomoCPR\n'); - else - defShifts = 0; - fprintf('Did not find updated defocus estimate from tomoCPR\n'); + + if (SuperResolution) + % Transform the raw images at full sampling then crop the fft to physical + % nyquist + PIXEL_SIZE = 2.* PIXEL_SIZE; end - end - - if ( flgShiftEucentric && mapBackIter ) - toFit = abs(mbTLT) > eucentric_minTilt; - - % For now take the mean, but it would probably be better to fit a line, - % use the Y intercept, and use the deviation from 0 of the slope as a - % measure of quality. - fprintf('\n\nYou suspect a eucentric drift\n\n') - eucShift = fit(mbTLT(toFit),1 .* mbEST(toFit,5) ./ sind(mbTLT(toFit)),'poly1'); - - fprintf('\n\nFound a possible eucentric shift of %3.3f\nThe slope (%3.3f) should be close to zero.\n\n',eucShift.p2,eucShift.p1); - eucShiftsResults{iGPU}{iTilt} = eucShift.p2; - end - - outputStackName = sprintf('%s/%s%s',outputDirectory,INPUT_CELL{iStack,6},INPUT_CELL{iStack,5}); - oldStackName = sprintf('%s/%s%s',outputDirectory,PRJ_OLD,INPUT_CELL{iStack,5}); - - try - erase_beads_after_ctf = emc.('erase_beads_after_ctf'); - catch - erase_beads_after_ctf = false; - end - - if (erase_beads_after_ctf) - flgEraseBeads = 0; - else - if exist(sprintf('fixedStacks/%s.erase',fileName),'file') - flgEraseBeads = 1; - % create and later run a script to erase gold beads using imods - % ccderaser and the present fiducial model. - + eraseSigma = 3;%emc.('beadSigma'); + + eraseRadius = ceil(1.2.*(emc.('beadDiameter')./PIXEL_SIZE.*0.5)); + flgImodErase = 0 + + % FIXME, this should be stored from previous mask calc and accessed there. + % For now just take based on tomogram (which will be larger than the true specimen thickness) + %THICKNESS = recGeomForThickness.(sprintf('%s_1',STACK_PRFX)); + %THICKNESS = min(10,abs(THICKNESS(1,3)-THICKNESS(2,3)).*PIXEL_SIZE.*10^9); + THICKNESS = 100; + % Assuming all extreme pixels have already been removed from the stack. + %PRJ_STACK = {sprintf('%s_local04_18.mrc',mjIDX)};%,sprintf('%s_local14_18.mrc',mjIDX),sprintf('%s_local24_18.mrc',mjIDX),sprintf('%s_local34_18.mrc',mjIDX)}; + nStacks = length(tlt); + INPUT_CELL = cell(nStacks,7); + TLT_Trans = cell(nStacks,1); + + % killed the loop, clean up later + if exist(tlt{1}, 'file') && exist(PRJ_STACK{1}, 'file') + INPUT_CELL{1,1} = load(tlt{1}); + INPUT_CELL{1,2} = PRJ_STACK{1}; + [pathName,fileName,extension] = fileparts(PRJ_STACK{1}); + if isempty(pathName) + pathName = '.'; + end + [ctfPath,~,~] = fileparts(tlt{1}); + INPUT_CELL{1,3} = pathName; + INPUT_CELL{1,4} = fileName; + INPUT_CELL{1,5} = extension; + INPUT_CELL{1,6} = PRJ_OUT{1}; + INPUT_CELL{1,7} = ctfPath; else - flgEraseBeads = 0; + if ~exist(tlt{1}, 'file') + fprintf('\nignoring %s, because the file is not found.\n', tlt{1}); + end + if ~exist(PRJ_STACK{1}, 'file') + fprintf('\nignoring %s, because the file is not found.\n',PRJ_STACK{1}); + end + end - end - - tlt_tmp = cell(d3,1); - out_tmp = cell(d3,1); - - origOrder = TLT(:,1); - TLT = sortrows(TLT,1); - - for i = 1:d3 - tlt_tmp{i} = TLT(i,:); - end - - if (flgSkipUpdate) - continue; - end - - if (SuperResolution) - % Forcing output to odd size. - sizeCropped = floor([d1,d2,d3]./2)-(1-mod(floor([d1,d2,d3]./2),2)); - else - sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); - end - sizeCropped(3) = d3; - - STACK = zeros(sizeCropped,'single'); - samplingMaskStack = zeros(sizeCropped,'single'); - - for i = 1:d3 - + + + % FIXME: removed the loop, clean up later cells etc later + iStack=1; + + iMrcObj = MRCImage(INPUT_CELL{iStack,2},0); + + % The pixel size should be previously set correctly, but if it is not, then we + % must maintain whatever is there in case beads are to be erased. The model + % used for this process depends on the pixel size in the header when it was + % created in IMod alignment. + + iHeader = getHeader(iMrcObj); + iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* (1+abs(SuperResolution)), ... + iHeader.cellDimensionY/iHeader.nY .* (1+abs(SuperResolution)), ... + iHeader.cellDimensionZ/iHeader.nZ]; + + iOriginHeader= [iHeader.xOrigin , ... + iHeader.yOrigin , ... + iHeader.zOrigin ] ./ (1+abs(SuperResolution)); + + d1 = iHeader.nX; d2 = iHeader.nY; d3 = size(INPUT_CELL{iStack,1},1);%iHeader.nZ; + + osX = 1-mod(d1,2); osY = 1-mod(d2,2); + if (SuperResolution) - % The transform shifts need to be scaled by 2 since the stored values - % are relative to full sampling, while the tomoCPR are relative to - % physical pixel size. - updateScale = 2; + gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,-0.235,'GPU','nyquistHigh'); else - updateScale = 1; + gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); end - + + TLT = INPUT_CELL{iStack,1}; + pathName = INPUT_CELL{iStack,3} + fileName = INPUT_CELL{iStack,4} + extension = INPUT_CELL{iStack,5} + + + % Optionally address magnification changes. + % system(sprintf('mkdir -p %s/recon',INPUT_CELL{i,3})); + system('mkdir -p aliStacks'); + if (mapBackIter) - - % Stored in row order as output by imod, st transpose is needed. Inversion - % of the xform is handled in resample2d. - origXF = reshape(tlt_tmp{i}(7:10),2,2)'; - newXF = reshape(mbEST(i,1:4),2,2)'; - - - dXYZ = [(newXF*tlt_tmp{i}(2:3)')' + mbEST(i,5:6).*updateScale , 0]; - if ~isvector(dXYZ) - % In case some implicit expansion were to happen for whatever reason. - error('dXYZ is a matrix and should be a vector'); + fprintf('Combining tranformations\n\n'); + % Load in the mapBack alignment + try + mbEST = load(sprintf('%s.tltxf',mapBackPrfx)); + catch + error('WARNING: did not load %s.tltxf, cannot update alignments',mapBackPrfx) + system(sprintf('cp fixedStacks/ctf/%s_ali1_ctf.tlt fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,STACK_PRFX,mapBackIter+1)); + continue; + end + mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); + defShifts = sprintf('%s.defShifts',mapBackPrfx); + if exist(defShifts,'file') + % tomoCPR is now using mexCTF so updated to def > 0 and in Angstrom, + % which are added to the base value. + % ctf 3d is still using orig def < 0 and in SI so convert here + defShifts = load(defShifts) .* (defShiftSign*10^-10); + fprintf('Updating defocus shifts from tomoCPR\n'); + else + defShifts = 0; + fprintf('Did not find updated defocus estimate from tomoCPR\n'); end - tlt_tmp{i}(2:3) = dXYZ(1:2); + end + + + if ( flgShiftEucentric && mapBackIter ) + toFit = abs(mbTLT) > eucentric_minTilt; - - - combinedXF = reshape((newXF*origXF)',1,4); - tlt_tmp{i}(7:10) = combinedXF; + % For now take the mean, but it would probably be better to fit a line, + % use the Y intercept, and use the deviation from 0 of the slope as a + % measure of quality. + fprintf('\n\nYou suspect a eucentric drift\n\n') + eucShift = fit(mbTLT(toFit),1 .* mbEST(toFit,5) ./ sind(mbTLT(toFit)),'poly1'); + + fprintf('\n\nFound a possible eucentric shift of %3.3f\nThe slope (%3.3f) should be close to zero.\n\n',eucShift.p2,eucShift.p1); + eucShiftsResults{iGPU}{iTilt} = eucShift.p2; + end + + outputStackName = sprintf('%s/%s%s',outputDirectory,INPUT_CELL{iStack,6},INPUT_CELL{iStack,5}); + oldStackName = sprintf('%s/%s%s',outputDirectory,PRJ_OLD,INPUT_CELL{iStack,5}); + + try + erase_beads_after_ctf = emc.('erase_beads_after_ctf'); + catch + erase_beads_after_ctf = false; + end + + if (erase_beads_after_ctf) + flgEraseBeads = 0; else - combinedXF = tlt_tmp{i}(7:10); - dXYZ = [tlt_tmp{i}(2:3),0]; + if exist(sprintf('fixedStacks/%s.erase',fileName),'file') + flgEraseBeads = 1; + % create and later run a script to erase gold beads using imods + % ccderaser and the present fiducial model. + + else + flgEraseBeads = 0; + end end - - % Now that we are always cropping prior to transforming, reduce the - % scale. Probable should just instruct to fourier crop prior to tilt - % alignment.flgSkipUpdate - dXYZ = dXYZ ./ updateScale; - - - % Pad the projection prior to xforming in Fourier space. + + tlt_tmp = cell(d3,1); + out_tmp = cell(d3,1); + + origOrder = TLT(:,1); + TLT = sortrows(TLT,1); + + for i = 1:d3 + tlt_tmp{i} = TLT(i,:); + end + + if (flgSkipUpdate) + continue; + end + if (SuperResolution) + % Forcing output to odd size. + sizeCropped = floor([d1,d2,d3]./2)-(1-mod(floor([d1,d2,d3]./2),2)); + else + sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); + end + sizeCropped(3) = d3; + + STACK = zeros(sizeCropped,'single'); + samplingMaskStack = zeros(sizeCropped,'single'); + + for i = 1:d3 - iProjection = single(getVolume(iMrcObj,[],[],tlt_tmp{i}(23),'keep')); - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); + if (SuperResolution) + % The transform shifts need to be scaled by 2 since the stored values + % are relative to full sampling, while the tomoCPR are relative to + % physical pixel size. + updateScale = 2; + else + updateScale = 1; + end - % Information beyond the physical nyquist should be removed to limit - % aliasing of noise prior tto interpolation. - iProjection = BH_padZeros3d(iProjection,[0,0],[0,0],'GPU','singleTaper',mean(iProjection(:))); - trimVal = BH_multi_padVal(1.*size(iProjection),sizeCropped(1:2)); - - largeOutliersMean= mean(iProjection(:)); - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single','gpuArray'); - - iProjection = real(ifftn(ifftshift(... - BH_padZeros3d(fftshift(... - fftn(iProjection)), ... - trimVal(1,:),trimVal(2,:),... - 'GPU','single')))); + if (mapBackIter) - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),[0,0,0],[0,0],'Bah','GPU','forward',1/2,sizeCropped(1:2)); - sizeODD = size(iProjection)-[osX,osY]; - else - sizeODD = [d1,d2]-[osX,osY]; - + % Stored in row order as output by imod, st transpose is needed. Inversion + % of the xform is handled in resample2d. + origXF = reshape(tlt_tmp{i}(7:10),2,2)'; + newXF = reshape(mbEST(i,1:4),2,2)'; + + + dXYZ = [(newXF*tlt_tmp{i}(2:3)')' + mbEST(i,5:6).*updateScale , 0]; + if ~isvector(dXYZ) + % In case some implicit expansion were to happen for whatever reason. + error('dXYZ is a matrix and should be a vector'); + end + tlt_tmp{i}(2:3) = dXYZ(1:2); + + + + combinedXF = reshape((newXF*origXF)',1,4); + tlt_tmp{i}(7:10) = combinedXF; + else + combinedXF = tlt_tmp{i}(7:10); + dXYZ = [tlt_tmp{i}(2:3),0]; + end + % Now that we are always cropping prior to transforming, reduce the + % scale. Probable should just instruct to fourier crop prior to tilt + % alignment.flgSkipUpdate + dXYZ = dXYZ ./ updateScale; - % If it is even sized, shift up one pixel so that the origin is in the middle - % of the odd output here we can just read it in this way, unlike super res. - iProjection = ... - single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep')); - - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); - - largeOutliersMean= mean(iProjection(:)); + % Pad the projection prior to xforming in Fourier space. + if (SuperResolution) + + iProjection = single(getVolume(iMrcObj,[],[],tlt_tmp{i}(23),'keep')); + iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); + + % Information beyond the physical nyquist should be removed to limit + % aliasing of noise prior tto interpolation. + iProjection = BH_padZeros3d(iProjection,[0,0],[0,0],'GPU','singleTaper',mean(iProjection(:))); + trimVal = BH_multi_padVal(1.*size(iProjection),sizeCropped(1:2)); + + largeOutliersMean= mean(iProjection(:)); + largeOutliersSTD = std(iProjection(:)); + largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... + iProjection > largeOutliersMean + 6*largeOutliersSTD); + iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single','gpuArray'); + + iProjection = real(ifftn(ifftshift(... + BH_padZeros3d(fftshift(... + fftn(iProjection)), ... + trimVal(1,:),trimVal(2,:),... + 'GPU','single')))); + + + iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),[0,0,0],[0,0],'Bah','GPU','forward',1/2,sizeCropped(1:2)); + sizeODD = size(iProjection)-[osX,osY]; + else + sizeODD = [d1,d2]-[osX,osY]; + + + + % If it is even sized, shift up one pixel so that the origin is in the middle + % of the odd output here we can just read it in this way, unlike super res. + + iProjection = ... + single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep')); + + iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); + + largeOutliersMean= mean(iProjection(:)); + + largeOutliersSTD = std(iProjection(:)); + largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... + iProjection > largeOutliersMean + 6*largeOutliersSTD); + iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); + + + + end - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); - - - - end - - % Because the rotation/scaling and translation are done separately, - % we must use a square transform; otherwise, a rotation angle dependent - % anisotropic distortion (like mag distortion) is introduced. + % Because the rotation/scaling and translation are done separately, + % we must use a square transform; otherwise, a rotation angle dependent + % anisotropic distortion (like mag distortion) is introduced. sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp.*0.25).*max(sizeODD)); @@ -443,109 +443,109 @@ iProjection = iProjection - mean(iProjection(:)); iProjection = iProjection ./ std(iProjection(:)); - - - if ( SuperResolution ) - iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... - padVal(1,:),padVal(2,:),'GPU','singleTaper'); - else - iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - 'GPU','singleTaper'); - end - - if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); - end - - - if (flgApplyFullXform) - % Do the phase shift after rotating - need to invert the scaling since - % we are in reciprocal space - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(combinedXF); - % Assuming stretch and skew are not fit, leave defined for possible - % later consideration. - - if (bh_global_do_2d_fourier_interp) - if (i == 1) - fprintf('resampling at 2x padding with fourier interp\n'); - end - % combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - combinedInverted = combinedInverted([1,2,4,5]); - - - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); + + if ( SuperResolution ) + iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... + padVal(1,:),padVal(2,:),'GPU','singleTaper'); else - if (i == 1) - fprintf('resampling at 1x padding with linear interp\n'); - end - % Real space, do not invert mag - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); + iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... + 'GPU','singleTaper'); end - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - - else - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(mbEST(i,1:4)); - % Assuming stretch and skew are not fit, leave defined for possible - % later consideration. - - % NOTE mag is ignored when the rotation matrix has 4 elements (IMOD) + if (i == 1 && bh_global_do_2d_fourier_interp) + bhF = fourierTransformer(iProjection,'OddSizeOversampled'); + end + + + if (flgApplyFullXform) + % Do the phase shift after rotating - need to invert the scaling since + % we are in reciprocal space + [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... + BH_decomposeIMODxf(combinedXF); + % Assuming stretch and skew are not fit, leave defined for possible + % later consideration. + + if (bh_global_do_2d_fourier_interp) - if (i == 1) - fprintf('resampling at 2x padding with fourier interp\n'); + if (i == 1) + fprintf('resampling at 2x padding with fourier interp\n'); + end + % combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); + combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); + combinedInverted = combinedInverted([1,2,4,5]); + + + iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); + else + if (i == 1) + fprintf('resampling at 1x padding with linear interp\n'); + end + % Real space, do not invert mag + combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); + combinedInverted = combinedInverted([1,2,4,5]); + iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); end - % mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); + + iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); + + else + [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... + BH_decomposeIMODxf(mbEST(i,1:4)); + % Assuming stretch and skew are not fit, leave defined for possible + % later consideration. + + % NOTE mag is ignored when the rotation matrix has 4 elements (IMOD) + + if (bh_global_do_2d_fourier_interp) + if (i == 1) + fprintf('resampling at 2x padding with fourier interp\n'); + end + % mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); mbEstInverted = mbEstInverted([1,2,4,5]); iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); else - if (i == 1) - fprintf('resampling at 1x padding with linear interp\n'); - end + if (i == 1) + fprintf('resampling at 1x padding with linear interp\n'); + end mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); mbEstInverted = mbEstInverted([1,2,4,5]); iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); end - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),mbEST(i,1:4),dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); + iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),mbEST(i,1:4),dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); + end + + + + STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... + trimVal(1,:),trimVal(2,:),... + 'GPU','single'))); + + iSamplingMask(isnan(iSamplingMask(:))) = 0; + samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); + iSamplingMask = []; + % + % end + end - - - STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... - trimVal(1,:),trimVal(2,:),... - 'GPU','single'))); - - iSamplingMask(isnan(iSamplingMask(:))) = 0; - samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); - iSamplingMask = []; - % - % end - - end - for i= 1:d3 TLT(i,:) = tlt_tmp{i}; - % STACK(:,:,TLT(i,1)) = out_tmp{i}; + % STACK(:,:,TLT(i,1)) = out_tmp{i}; end - - out_tmp = []; - - if (mapBackIter) - % Update the tilt angles - TLT(:,4) = mbTLT; - if (defShifts) - TLT(:,15) = TLT(:,15) + defShifts; - TLT(:,16) = PIXEL_SIZE; - end - - + + out_tmp = []; + + if (mapBackIter) + % Update the tilt angles + TLT(:,4) = mbTLT; + if (defShifts) + TLT(:,15) = TLT(:,15) + defShifts; + TLT(:,16) = PIXEL_SIZE; + end + + % Sort descending along the magnitude of the tilt angles because higher tilts take % longer on CTF correction. If more processor available than projections, % this doesn't affect anything. @@ -558,69 +558,69 @@ sprintf('%s/%s.tlt',INPUT_CELL{iStack,7},INPUT_CELL{iStack,6}) fileID = fopen(tlt_OUT{iStack}, 'w'); fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%5e\t%5e\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%3.2f\n'], TLT'); - - - if ( flgEraseBeads ) - STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); - end - - + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%3.2f\n'], TLT'); + + + if ( flgEraseBeads ) + STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); + end + + fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',... - THICKNESS, STACK_PRFX); - + THICKNESS, STACK_PRFX); + [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,PIXEL_SIZE*10^10,gpuArray(samplingMaskStack)); SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName)); - xShift= []; yShift = []; scale = []; angleShift = []; - dZ = []; recZ= []; rotMat = []; angX = []; angY = []; - else - - if ( flgEraseBeads ) - STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); - end - - + xShift= []; yShift = []; scale = []; angleShift = []; + dZ = []; recZ= []; rotMat = []; angX = []; angY = []; + else + + if ( flgEraseBeads ) + STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); + end + + fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',... - THICKNESS, STACK_PRFX); - + THICKNESS, STACK_PRFX); + [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,PIXEL_SIZE*10^10,gpuArray(samplingMaskStack)); SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName),iPixelHeader,iOriginHeader); - - end - if (mapBackIter && conserveDiskSpace) - system(sprintf('rm %s',oldStackName)); - end - - - % Once updated the reconstructions are no longer valid - system(eraseStack); - system(eraseRec); + + end + if (mapBackIter && conserveDiskSpace) + system(sprintf('rm %s',oldStackName)); + end + + + % Once updated the reconstructions are no longer valid + system(eraseStack); + system(eraseRec); end % end of loop over tilts end % end of par for loop -% +% if (flgShiftEucentric && mapBackIter) % Update the sub tomo z coords with an estimate of the shift cycle_to_update = subTomoMeta.('tomoCPR_run_in_cycle')(find(subTomoMeta.('tomoCPR_run_in_cycle')(:,1) == subTomoMeta.currentTomoCPR),2); for iGPU = 1:nGPUs - - for iTilt = 1:length(ITER_LIST{iGPU}) - + + for iTilt = 1:length(ITER_LIST{iGPU}) + STACK_PRFX = ITER_LIST{iGPU}{iTilt}; eucShift = eucShiftsResults{iGPU}{iTilt}; - + % Workaround for partial numbers - need something better. FIXME n_tomos_found = 0; - for jTomo = 1:size(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords,1) + for jTomo = 1:size(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords,1) if any(subTomoMeta.mapBackGeometry.(STACK_PRFX).coords(jTomo,:)) - n_tomos_found = n_tomos_found + 1; + n_tomos_found = n_tomos_found + 1; % We might have skipped the update if tomoCPR failed. if isempty(eucShift) fprintf('No updated shifts for %s_%d\n',STACK_PRFX,jTomo); @@ -629,28 +629,28 @@ fprintf('shifting %s_%d\n',STACK_PRFX,jTomo); subTomoMeta.(sprintf('cycle%0.3d',cycle_to_update)).('eucentric_shifts').(sprintf('%s_%d',STACK_PRFX,jTomo)) = [eucShift]; % The tomogram is shifted by the calculated amount -% subTomoMeta.(sprintf('cycle%0.3d',subTomoMeta.currentCycle)).RawAlign.(sprintf('%s_%d',STACK_PRFX,jTomo))(:,13) = ... -% eucShift + subTomoMeta.(sprintf('cycle%0.3d',subTomoMeta.currentCycle)).RawAlign.(sprintf('%s_%d',STACK_PRFX,jTomo))(:,13); + % subTomoMeta.(sprintf('cycle%0.3d',subTomoMeta.currentCycle)).RawAlign.(sprintf('%s_%d',STACK_PRFX,jTomo))(:,13) = ... + % eucShift + subTomoMeta.(sprintf('cycle%0.3d',subTomoMeta.currentCycle)).RawAlign.(sprintf('%s_%d',STACK_PRFX,jTomo))(:,13); end - + end end if (n_tomos_found ~= subTomoMeta.mapBackGeometry.(STACK_PRFX).nTomos) - error('The number of tomos found (%d) does not match (%d) for tomo %s\n', ... - n_tomos_found, subTomoMeta.mapBackGeometry.(STACK_PRFX).nTomos, STACK_PRFX); - end + error('The number of tomos found (%d) does not match (%d) for tomo %s\n', ... + n_tomos_found, subTomoMeta.mapBackGeometry.(STACK_PRFX).nTomos, STACK_PRFX); + end end end save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); end - + if ( flgParallel ) fprintf('\nAuto updating the tilt geometry\n'); - % BH_geometryAnalysis(updateCMD) - BH_geometryAnalysis(sprintf('%s',PARAMETER_FILE),sprintf('%d',subTomoMeta.currentCycle),'TiltAlignment','UpdateTilts',sprintf('[%d,0,0]',subTomoMeta.currentCycle),'STD'); + % BH_geometryAnalysis(updateCMD) + BH_geometryAnalysis(sprintf('%s',PARAMETER_FILE),sprintf('%d',subTomoMeta.currentCycle),'TiltAlignment','UpdateTilts',sprintf('[%d,0,0]',subTomoMeta.currentCycle),'STD'); else fprintf('\n\nSince you are updating each tilt series manually, you must'); fprintf(' run\nemClarity geometry [param] [cycle] TiltAlignment UpdateTilts [cycle,0,0] STD\n\n'); end - + diff --git a/ctf/BH_runCtfFind.m b/ctf/BH_runCtfFind.m index 1d44e8be..6dd8dbf1 100644 --- a/ctf/BH_runCtfFind.m +++ b/ctf/BH_runCtfFind.m @@ -1,8 +1,8 @@ function [ ] = BH_runCtfFind(stackName, tltName, ctfParams, tiltAngles) %Fit the ctf to a background subtracted PS using ctffind4 % CTF params -% PixelSize (Ang) -% KeV +% PixelSize (Ang) +% KeV % CS (mm) % Amplitude Contrast @@ -23,12 +23,12 @@ % % Check to make sure this hasn't alread been done % if ~exist(sprintf('fixedStacks/ctf/%s_orig',tltName), 'file') - system(sprintf('mv fixedStacks/ctf/%s fixedStacks/ctf/%s_orig',tltName,tltName)); +system(sprintf('mv fixedStacks/ctf/%s fixedStacks/ctf/%s_orig',tltName,tltName)); % end - tmpTLT = load(sprintf('fixedStacks/ctf/%s_orig',tltName)); - meanDefocus = mean(tmpTLT(:,15))*-1.0*10^10; - fprintf('Searching around an estimated mean defocus of %3.6f Angstrom\n'); +tmpTLT = load(sprintf('fixedStacks/ctf/%s_orig',tltName)); +meanDefocus = mean(tmpTLT(:,15))*-1.0*10^10; +fprintf('Searching around an estimated mean defocus of %3.6f Angstrom\n'); % write the run script, this should link to a distributed version with % special name, but for testing use the beta. @@ -38,16 +38,16 @@ fprintf(fID,'#!/bin/bash\n\n'); for iPrj = 1:d3 % I want to fit to lower resolution at higher tilts tltIDX = find(tiltAngles(:,1) == iPrj); - + % put in a line to limit number of cores, or use the threaded version fprintf(fID,'\n%s --amplitude-spectrum-input << eof &',ctfFindPath); fprintf(fID,'\nfixedStacks/ctf/forCtfFind/%s_%d.mrc\n',randPrfx,iPrj); fprintf(fID,'fixedStacks/ctf/forCtfFind/%s_diagnostic_%d.mrc\n',randPrfx,iPrj); fprintf(fID,'%f\n%f\n%f\n%f\n%d\n%f\n%f\n%d\n%d\n%d\n',ctfParams(1:4), ... - d1,30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)),... - 0.75*meanDefocus,... - 1.25*meanDefocus,... - 25.0); + d1,30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)),... + 0.75*meanDefocus,... + 1.25*meanDefocus,... + 25.0); fprintf(fID,'no\nno\nyes\n500.0\nno\nno\nno\neof\n\n'); end fprintf(fID,'wait\n'); @@ -77,8 +77,8 @@ for iPrj = 1:d3 - system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10,3.141592/180*$4,-1*(($2+$3)/2)*10^-10 }'' >> %s', baseName,iPrj,tmpName)); - + system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10,3.141592/180*$4,-1*(($2+$3)/2)*10^-10 }'' >> %s', baseName,iPrj,tmpName)); + end % TODO ground truth to confirm orientation of astigmatism diff --git a/logicals/BH_imodWait.m b/logicals/BH_imodWait.m index bb5aa594..30c098cd 100755 --- a/logicals/BH_imodWait.m +++ b/logicals/BH_imodWait.m @@ -5,29 +5,29 @@ % 1) in observed cases the header is not written so use this as a check % for completion % 2) Make sure the file is actually growing or throw an error. - fSize = 0; - failed = 1; - while (failed) - [failed, ~] = system(sprintf('header %s',fileName)); - fInfo = dir(fileName); - if fInfo.bytes > fSize; - fSize = fInfo.bytes - pause(5); - else - % give it another longer pause - fprintf('System seems slow, extended pause.\n') - pause(30) - [failedMeTwice, ~] = system(sprintf('header %s',fileName)); - if (failedMeTwice) - if fInfo.bytes > fSize - fSize = fInfo.bytes - pause(5) - else - error('##### system impatience.') - end - end - end +fSize = 0; +failed = 1; +while (failed) + [failed, ~] = system(sprintf('header %s',fileName)); + fInfo = dir(fileName); + if fInfo.bytes > fSize; + fSize = fInfo.bytes + pause(5); + else + % give it another longer pause + fprintf('System seems slow, extended pause.\n') + pause(30) + [failedMeTwice, ~] = system(sprintf('header %s',fileName)); + if (failedMeTwice) + if fInfo.bytes > fSize + fSize = fInfo.bytes + pause(5) + else + error('##### system impatience.') end - + end + end +end + end diff --git a/logicals/BH_isWindowValid.m b/logicals/BH_isWindowValid.m index b22979fa..de725fd0 100755 --- a/logicals/BH_isWindowValid.m +++ b/logicals/BH_isWindowValid.m @@ -1,12 +1,12 @@ function [ INDICES, PADVALUES, SHIFTS ] = ... - BH_isWindowValid( VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, CENTER ) + BH_isWindowValid( VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, CENTER ) %Address out of bounds conditions. -% +% % % Input Variables: % % VOLUMES_SIZE = Size of the volume to extract from. -% +% % WINDOW_SIZE = Size of the window to be extracted. % % CENTER = Coordinates of the center of the window in the tomogram frame. @@ -17,7 +17,7 @@ % INDICES = 2,3 array with [x1, y1, z1 ; x2, y2, z2] % % PADVALUES = 2,3 array with pre and post padding values, same order. -% +% % SHIFTS = fractional shifts % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -64,44 +64,44 @@ PADVALUES = [padLOW ; padTOP]; SHIFTS = deltaWinCenter; - % The window is often much larger than the particle. This is to ensure that all of the - % delocalized information is available for full CTF restoration. Ideally, this is - % fully represented in every particle, but as long as the particle itself is there, - % don't worry if the high resolution information is not. +% The window is often much larger than the particle. This is to ensure that all of the +% delocalized information is available for full CTF restoration. Ideally, this is +% fully represented in every particle, but as long as the particle itself is there, +% don't worry if the high resolution information is not. - % Should keep track of this and figure into the quality weight somehow. +% Should keep track of this and figure into the quality weight somehow. - availableArea = WINDOW_SIZE - PADVALUES(1,:) - PADVALUES(2,:); - if any(availableArea - minSizeMask < -2) +availableArea = WINDOW_SIZE - PADVALUES(1,:) - PADVALUES(2,:); +if any(availableArea - minSizeMask < -2) fprintf(['\nvs %d %d %d\nws %d %d %d\nmr %2.1f %2.1f %2.1f\n',... 'minArea %d %d %d\navailArea %d %d %d\nc %2.1f %2.1f %2.1f\n'], ... - VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, minSizeMask, availableArea, CENTER); + VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, minSizeMask, availableArea, CENTER); INDICES = 'noUse'; - PADVALUES = [availableArea]; - end - if any(isnan(PADVALUES(:))) - fprintf('center %f %f %f\n',CENTER); - fprintf('min %f %f %f\n',minSizeMask); - fprintf('winLowCorner %f %f %f\n', winLowCorner); - fprintf('top %f %f %f\n',winTopCorner); - fprintf('%f %f %f\n',winCenter); - fprintf('del %f %f %f\n',deltaWinCenter); - fprintf('%f %f %f\n',LOW); - fprintf('%f %f %f\n',TOP); - error('\n\nFound a NaN in the pad values. But Why ben why?\n\n'); - INDICES='noUse'; - PADVALUES = [availableArea]; - end - % padSUM = sum(PADVALUES(:,1)) .* WINDOW_SIZE(2) .* WINDOW_SIZE(3); - % padSUM = padSUM + sum(PADVALUES(:,2)) .* WINDOW_SIZE(1) .* WINDOW_SIZE(3); - % padSUM = padSUM + sum(PADVALUES(:,3)) .* WINDOW_SIZE(1) .* WINDOW_SIZE(2); - % padSUM = padSUM ./ prod(WINDOW_SIZE); - - % minimal sampling - % if (padSUM) > .30 || any(any((PADVALUES - [WINDOW_SIZE;WINDOW_SIZE]) >0)) - % INDICES = 'noUse'; - % PADVALUES = padSUM; - % end + PADVALUES = [availableArea]; +end +if any(isnan(PADVALUES(:))) + fprintf('center %f %f %f\n',CENTER); + fprintf('min %f %f %f\n',minSizeMask); + fprintf('winLowCorner %f %f %f\n', winLowCorner); + fprintf('top %f %f %f\n',winTopCorner); + fprintf('%f %f %f\n',winCenter); + fprintf('del %f %f %f\n',deltaWinCenter); + fprintf('%f %f %f\n',LOW); + fprintf('%f %f %f\n',TOP); + error('\n\nFound a NaN in the pad values. But Why ben why?\n\n'); + INDICES='noUse'; + PADVALUES = [availableArea]; +end +% padSUM = sum(PADVALUES(:,1)) .* WINDOW_SIZE(2) .* WINDOW_SIZE(3); +% padSUM = padSUM + sum(PADVALUES(:,2)) .* WINDOW_SIZE(1) .* WINDOW_SIZE(3); +% padSUM = padSUM + sum(PADVALUES(:,3)) .* WINDOW_SIZE(1) .* WINDOW_SIZE(2); +% padSUM = padSUM ./ prod(WINDOW_SIZE); + +% minimal sampling +% if (padSUM) > .30 || any(any((PADVALUES - [WINDOW_SIZE;WINDOW_SIZE]) >0)) +% INDICES = 'noUse'; +% PADVALUES = padSUM; +% end diff --git a/logicals/BH_multi_maskCheck.m b/logicals/BH_multi_maskCheck.m index 5175ded5..82030006 100755 --- a/logicals/BH_multi_maskCheck.m +++ b/logicals/BH_multi_maskCheck.m @@ -1,18 +1,18 @@ function [ maskType, maskSize, maskRadius, maskCenter ] = BH_multi_maskCheck( emc, PREFIX, pixelSize, varargin ) %Check that any masked sub-regions are in bounds with proper apodization given %the offset. -% +% if nargin > 3 && strcmpi(varargin{1},'FSC') maskCenter = [0,0,0]; maskRadius = round(emc.('particleRadius') ./ pixelSize); else - maskCenter = emc.(sprintf('%s_mCenter',PREFIX)) ./ pixelSize; - maskRadius = round(emc.(sprintf('%s_mRadius',PREFIX)) ./ pixelSize); + maskCenter = emc.(sprintf('%s_mCenter',PREFIX)) ./ pixelSize; + maskRadius = round(emc.(sprintf('%s_mRadius',PREFIX)) ./ pixelSize); end - + maskType = emc.(sprintf('%s_mType',PREFIX)); -% Although the minimum roll-off is set for 6 pixels on each edge, use 14 rather +% Although the minimum roll-off is set for 6 pixels on each edge, use 14 rather % than 12 to make sure a shift in the origin doesn't preculde the volume from % analysis. maskSize = 2.*floor(emc.('particleRadius') ./ pixelSize)+14; diff --git a/logicals/BH_multi_parallelJobs.m b/logicals/BH_multi_parallelJobs.m index 8d07404a..2f4783ea 100644 --- a/logicals/BH_multi_parallelJobs.m +++ b/logicals/BH_multi_parallelJobs.m @@ -1,6 +1,6 @@ function [nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, ... - nGPUs, ... - calcSize,flgAvg) + nGPUs, ... + calcSize,flgAvg) %UNTITLED Summary of this function goes here % Detailed explanation goes here @@ -18,7 +18,7 @@ elseif totMem > 10.9e9 && totMem < 12.2e9 % 1080Ti or Tesla or Titan Xp scaleMem = 1.0; -elseif totMem > 12.8e9 && totMem < 17.2e9 +elseif totMem > 12.8e9 && totMem < 17.2e9 % TitanV V/p100 % seems to crash when it shouldn't, figure out later. scaleMem = 1.3; else @@ -40,7 +40,7 @@ nParProcesses = min(nParProcesses,3*nGPUs); else % Fourier interp, requires more mem. Unoptimized - nParProcesses = min(flgAvg,nParProcesses); + nParProcesses = min(flgAvg,nParProcesses); end end diff --git a/logicals/BH_multi_parallelWorkers.m b/logicals/BH_multi_parallelWorkers.m index 5871deaf..21a5da75 100644 --- a/logicals/BH_multi_parallelWorkers.m +++ b/logicals/BH_multi_parallelWorkers.m @@ -5,8 +5,8 @@ pInfo = parcluster(); if (pInfo.NumWorkers < nWorkers_wanted) fprintf('\nnWorkers requested but only %d are visible, reducing\n', ... - nWorkers_wanted,pInfo.NumWorkers); - nWorkers = pInfo.NumWorkers; + nWorkers_wanted,pInfo.NumWorkers); + nWorkers = pInfo.NumWorkers; else nWorkers = nWorkers_wanted; end diff --git a/logicals/BH_multi_validArea.m b/logicals/BH_multi_validArea.m index d4fa25b2..fc5e2d9a 100755 --- a/logicals/BH_multi_validArea.m +++ b/logicals/BH_multi_validArea.m @@ -1,9 +1,9 @@ function [ sizeWINDOW, sizeCALC, sizeMASK, padWINDOW, padCALC ] = ... - BH_multi_validArea( MASK_SIZE, MASK_RADIUS, scaleCalcSize ) + BH_multi_validArea( MASK_SIZE, MASK_RADIUS, scaleCalcSize ) %Calculate the padding and sizes needed to work with a given dimension. -% +% % Input: -% +% % MASK_RADIUS: radius in pixels defining the size of valid area % % SIZE_REF: size of the reference being used, check that it is large enough to @@ -15,7 +15,7 @@ % % sizeCALC: padded size to avoid wraparound effects in circular convolution. % -% sizeMASK: smallest size to use for mask creation that includes enough room +% sizeMASK: smallest size to use for mask creation that includes enough room % for an appropriate taper. This would be passed to BH_mask3d as the % size, or trim to this size and then pass volume to BH_padZeros3d % with the taper option. @@ -34,7 +34,7 @@ - + sizeMASK = ceil(scaleCalcSize.* max(MASK_SIZE).*[1,1,1]); % Ensure that the reference being used is large enough to be masked @@ -42,7 +42,7 @@ -% I don't remember what I was going for here. This should be simpler. +% I don't remember what I was going for here. This should be simpler. % % % padWINDOW = ceil((sqrt(mPlusTaper*mPlusTaper')-mPlusTaper)); % % % padWINDOW = [ padWINDOW ; padWINDOW ]; % % % sizeWINDOW = 2.* padWINDOW(1,:) + sizeMASK; @@ -50,7 +50,7 @@ padWINDOW = BH_multi_padVal(sizeMASK,sizeWINDOW); -% Calc padding, each image masked to dimesion 2*maskRad+20 +% Calc padding, each image masked to dimesion 2*maskRad+20 % Also make cubic so that weight mask can be calculated once for each tomo and % then interpolated. In the future, it would be better to calculate a mask for % each particle, then this could just be 2.*sizeMask. @@ -68,7 +68,7 @@ if any((sizeCALC - sizeWINDOW) < 0); sizeCALC = sizeWINDOW; % Find the next largest size for fft -[ sizeCALC ] = BH_multi_iterator( sizeCALC, 'fourier' ); + [ sizeCALC ] = BH_multi_iterator( sizeCALC, 'fourier' ); end [ padCALC ] = BH_multi_padVal( sizeMASK, sizeCALC ); diff --git a/logicals/BH_weightAngCheckPeaks.m b/logicals/BH_weightAngCheckPeaks.m index 217b0973..02e0b2fb 100644 --- a/logicals/BH_weightAngCheckPeaks.m +++ b/logicals/BH_weightAngCheckPeaks.m @@ -1,7 +1,7 @@ function [ peakWgt, sortedList ] = BH_weightAngCheckPeaks(positionList, nPeaks, ... - score_sigma, iSubTomo, tomoName,... - track_stats) - + score_sigma, iSubTomo, tomoName,... + track_stats) + %UNTITLED Summary of this function goes here % Detailed explanation goes here @@ -9,7 +9,7 @@ % converege to the same value earlier on and therby save computation. For % testing, set a bit more conservatively. angleTolerance = 5; -compressByFactor = 2; +compressByFactor = 2; % Check that the class id is not set to ignore includedPeaks = reshape(find(positionList(26:26:26*nPeaks) ~= -9999),[],1); @@ -46,7 +46,7 @@ % re-determine sorted included positions includedPeaks = reshape(find(sortedList(26:26:26*nPeaks) ~= -9999),[],1); excludedPeaks = reshape(find(sortedList(26:26:26*nPeaks) == -9999),[],1); - peakWgt(includedPeaks) = 1; + peakWgt(includedPeaks) = 1; peakWgt(excludedPeaks) = -9999; nIncluded = numel(includedPeaks); @@ -55,29 +55,29 @@ % List to loop over to check for unique angles combinations = [nIncluded:-1:2,1]; nCHk = nchoosek(combinations,2); - + for iMax = nIncluded:-1:2 % Rotation matrix for the peak in question - + rNxMx = reshape(sortedList(17+26*(iMax-1):25+26*(iMax-1)),3,3); for iComb = find(nCHk(:,1) == iMax) rComb = reshape(sortedList(17+26*(nCHk(iComb,2)-1):25+26*(nCHk(iComb,2)-1)),3,3); -% distVect = (180/pi)*sqrt(sum((rNxMx-rComb).^2,1)); + % distVect = (180/pi)*sqrt(sum((rNxMx-rComb).^2,1)); % From http://www.boris-belousov.net/2016/12/01/quat-dist/#using-rotation-matrices R = rNxMx * transpose(rComb); angDist = acosd((trace(R)-1)./2); - - + + if all(angDist < angleTolerance) % this is a duplicate rotation, set its class to -9999 to ignore. sortedList(26+26*(iMax-1)) = -9999; peakWgt(iMax) = -9999; break end - + end end - + % % Sort with peak # in col one descending on the CCC % rankedScores = sortrows([includedPeaks, positionList(1 + 26.*(includedPeaks-1))'],-2); @@ -118,7 +118,7 @@ fprintf('%1.3f ', peakWgt(i)); end fprintf(']\n'); - + end diff --git a/masking/BH_bandLimitCenterNormalize.m b/masking/BH_bandLimitCenterNormalize.m index 575237ad..aad7661d 100755 --- a/masking/BH_bandLimitCenterNormalize.m +++ b/masking/BH_bandLimitCenterNormalize.m @@ -1,5 +1,5 @@ function [ IMG ] = BH_bandLimitCenterNormalize( IMG, BANDPASS, MASK, ... - PAD, PRECISION ) + PAD, PRECISION ) %Bandlimit, mask, center, normalize and img. % Masks ROI in image, then bandlimits while also setting the mean to 0 and the % variance. Returns FT, if image is to be recovered must take ifft. @@ -14,7 +14,7 @@ % MASK = binary mask for mean subtraction % % PAD = zero padding -% +% % PRECISION = single, double, singleTaper, doubleTaper % % Note: F(0,0,0) = prod(size(img)).*mean(img(:)) st we might set the img @@ -54,11 +54,11 @@ if (applyBandpass) IMG = IMG .* BANDPASS; end - % set rms to 1 extra parenthesis intentional to save numel(tmpIMG) divisions - % The factor of N * IMG is to deal with matlab scaling the FFT by 1/N on - % the inverse transform. +% set rms to 1 extra parenthesis intentional to save numel(tmpIMG) divisions +% The factor of N * IMG is to deal with matlab scaling the FFT by 1/N on +% the inverse transform. %%%if ~(isnumeric(MASK) || islogical(MASK)) - IMG = IMG ./ ((sqrt(sum(sum(sum(abs(IMG).^2)))) ./ numel(IMG))); +IMG = IMG ./ ((sqrt(sum(sum(sum(abs(IMG).^2)))) ./ numel(IMG))); %%%end clear BANDPASS MASK PAD PRECISION end diff --git a/masking/BH_bandLimitCenterNormalize_2.m b/masking/BH_bandLimitCenterNormalize_2.m index 049a8cb3..4c05ff1a 100644 --- a/masking/BH_bandLimitCenterNormalize_2.m +++ b/masking/BH_bandLimitCenterNormalize_2.m @@ -1,5 +1,5 @@ function [ IMG ] = BH_bandLimitCenterNormalize_2( IMG, BANDPASS, MASK, ... - PAD, PRECISION ) + PAD, PRECISION ) %Bandlimit, mask, center, normalize and img. % Masks ROI in image, then bandlimits while also setting the mean to 0 and the % variance. Returns FT, if image is to be recovered must take ifft. @@ -14,7 +14,7 @@ % MASK = binary mask for mean subtraction % % PAD = zero padding -% +% % PRECISION = single, double, singleTaper, doubleTaper % % Note: F(0,0,0) = prod(size(img)).*mean(img(:)) st we might set the img @@ -54,9 +54,9 @@ if (applyBandpass) IMG = IMG .* BANDPASS; end - % set rms to 1 extra parenthesis intentional to save numel(tmpIMG) divisions +% set rms to 1 extra parenthesis intentional to save numel(tmpIMG) divisions %%%if ~(isnumeric(MASK) || islogical(MASK)) - IMG = IMG ./ ((sqrt(sum(sum(sum(abs(IMG).^2)))) ./ numel(IMG))); +IMG = IMG ./ ((sqrt(sum(sum(sum(abs(IMG).^2)))) ./ numel(IMG))); %%%end clear BANDPASS MASK PAD PRECISION end diff --git a/masking/BH_bandLimitCenterNormalize_cpu.m b/masking/BH_bandLimitCenterNormalize_cpu.m index a340ecc4..f11de4fe 100755 --- a/masking/BH_bandLimitCenterNormalize_cpu.m +++ b/masking/BH_bandLimitCenterNormalize_cpu.m @@ -1,5 +1,5 @@ function [ IMG ] = BH_bandLimitCenterNormalize_cpu( IMG, BANDPASS, MASK, ... - PAD, PRECISION ) + PAD, PRECISION ) %Bandlimit, mask, center, normalize and img. % Masks ROI in image, then bandlimits while also setting the mean to 0 and the % variance. Returns FT, if image is to be recovered must take ifft. @@ -14,7 +14,7 @@ % MASK = binary mask for mean subtraction % % PAD = zero padding -% +% % PRECISION = single, double, singleTaper, doubleTaper % % Note: F(0,0,0) = prod(size(img)).*mean(img(:)) st we might set the img @@ -27,7 +27,7 @@ if isnumeric(MASK) || islogical(MASK) IMG(MASK) = IMG(MASK) - mean(double(IMG(MASK))); - % IMG(MASK) = IMG(MASK) ./rms(double(IMG(MASK))); + % IMG(MASK) = IMG(MASK) ./rms(double(IMG(MASK))); end @@ -37,9 +37,9 @@ IMG = IMG .* BANDPASS; end - % set rms to 1 extra parenthesis intentional to save numel(tmpIMG) divisions +% set rms to 1 extra parenthesis intentional to save numel(tmpIMG) divisions %%%if ~(isnumeric(MASK) || islogical(MASK)) - IMG = IMG ./ ((sqrt(sum(sum(sum(abs(IMG).^2)))) ./ numel(IMG))); +IMG = IMG ./ ((sqrt(sum(sum(sum(abs(IMG).^2)))) ./ numel(IMG))); %%%end clear BANDPASS MASK PAD PRECISION end diff --git a/masking/BH_bandpass3d.m b/masking/BH_bandpass3d.m index 0e0591d2..b5d50573 100755 --- a/masking/BH_bandpass3d.m +++ b/masking/BH_bandpass3d.m @@ -1,7 +1,7 @@ function [ BANDPASS ] = BH_bandpass3d( SIZE, HIGH_THRESH, HIGH_CUT, LOW_CUT, ... - METHOD, PIXEL_SIZE ) + METHOD, PIXEL_SIZE ) %Create a bandpass filter, to apply to fft of real space 3d images. -% +% % Input variables: % SIZE = size of the image filter is to be applied to : vector, float @@ -15,7 +15,7 @@ % METHOD = 'GPU' case specific, create mask on GPU, otherwise on CPU % % PIXEL_SIZE = Sampling frequency : Angstrom/Pixel -% +% % % Output variables: % @@ -30,27 +30,27 @@ % This is accomplished by oversampling the image to be filtered, so that % the fall off can be over a sufficient number of pixels, while still % happening over a small range of frequency. -% +% % Because this is frequency space, the fall off depends on the resolution % where it begins. For sampling of 3A/pixel with a cutoff starting at -% 20A^-1 the spatial frequency drops to ~ 17.3 over six pixels if the -% frequency rectangle is 256 pixel sq. For a cutoff starting at 10A the +% 20A^-1 the spatial frequency drops to ~ 17.3 over six pixels if the +% frequency rectangle is 256 pixel sq. For a cutoff starting at 10A the % drop over 6 pixels is only to ~9.3 -% +% % For the monolayer work, the largest dimension is ~140 pixels, s.t. 256 % provides a substantial padding for cross-correlation, and also a % reasonably tight window for filtering. The memory requirements are ~ % 67/134 mb for single/double precision, compared to another order of -% magnitude for 512. +% magnitude for 512. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % TODO: % -test with gpu flag -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - + if numel(SIZE) == 2 SIZE = [SIZE,1]; end @@ -59,29 +59,29 @@ % regular users to adjust. The value of 2.0 makes for a nice (soft) fall % off over ~ 7 pixels. Larger value results in a steeper cutoff. % Corresponds to the standard deviation in the gaussian roll. -if isnumeric(PIXEL_SIZE) +if isnumeric(PIXEL_SIZE) [bSize, highRoll, lowRoll, highCut, lowCut] = calc_frequencies( ... - SIZE, HIGH_THRESH, HIGH_CUT, LOW_CUT, PIXEL_SIZE ); + SIZE, HIGH_THRESH, HIGH_CUT, LOW_CUT, PIXEL_SIZE ); else bSize = SIZE; if strcmpi(PIXEL_SIZE,'nyquistHigh') - highCut = 7/min(bSize(bSize>1)); - highThresh = 1e-6; - highRoll = sqrt((-1.*highCut.^2)./(2.*log(highThresh))); + highCut = 7/min(bSize(bSize>1)); + highThresh = 1e-6; + highRoll = sqrt((-1.*highCut.^2)./(2.*log(highThresh))); else - highCut = 0; highThresh = 0; highRoll = 0; + highCut = 0; highThresh = 0; highRoll = 0; end lowRoll = 1.5 .* (1.0./min(bSize(bSize>1))); lowCut = 0.485+LOW_CUT; -end +end gaussian = @(x,m,s) exp( -1.*(x-m).^2 ./ (2.*s.^2) ); - + % initialize window of appropriate size if strcmp(METHOD, 'GPU') - mWindow(bSize(1),bSize(2),bSize(3)) = gpuArray(single(0)); + mWindow(bSize(1),bSize(2),bSize(3)) = gpuArray(single(0)); else mWindow(bSize(1),bSize(2),bSize(3)) = single(0); end @@ -92,16 +92,16 @@ [ radius,~,~,~,~,~] = BH_multi_gridCoordinates( bSize, 'Cartesian', METHOD, ... - {'single',... - [1,0,0;0,1,0;0,0,1],... - [0,0,0]','forward',1,1}, ... - 1, 0, 1 ); + {'single',... + [1,0,0;0,1,0;0,0,1],... + [0,0,0]','forward',1,1}, ... + 1, 0, 1 ); % Calc lowpass filter mWindow = ... - ( (radius < lowCut) .* mWindow + ... - (radius >= lowCut) .* gaussian(radius, lowCut, lowRoll) ); + ( (radius < lowCut) .* mWindow + ... + (radius >= lowCut) .* gaussian(radius, lowCut, lowRoll) ); % Breaks Hermitian symmetry % % % Don't randomize the high-pass since the signal it modulates is very @@ -111,18 +111,18 @@ if highCut ~= 0 mWindow = (radius <= highCut) .* gaussian(radius, highCut, highRoll) + ... - (radius > highCut) .* mWindow; -end - -%mWindow((mWindow<= 10^-8)) = 0; + (radius > highCut) .* mWindow; +end + +%mWindow((mWindow<= 10^-8)) = 0; BANDPASS = mWindow; clearvars -except BANDPASS - + end % end of BH_mask3d function. function [bSize, highRoll, lowRoll, highCut, lowCut] = calc_frequencies(... - SIZE, HIGH_THRESH, HIGH_CUT, LOW_CUT, PIXEL_SIZE ) + SIZE, HIGH_THRESH, HIGH_CUT, LOW_CUT, PIXEL_SIZE ) bSize = SIZE; @@ -133,7 +133,7 @@ -% Translate boundries from A^-1 to cycles/pixel. A value of zero means no +% Translate boundries from A^-1 to cycles/pixel. A value of zero means no % high pass filter. if HIGH_CUT ~= 0 highCut = PIXEL_SIZE ./ HIGH_CUT; @@ -165,8 +165,8 @@ highRoll = 0; end - - + + end % end of calc_frequencies function. diff --git a/masking/BH_eraseBeads.m b/masking/BH_eraseBeads.m index 6c03bed7..58505d10 100644 --- a/masking/BH_eraseBeads.m +++ b/masking/BH_eraseBeads.m @@ -10,53 +10,53 @@ [d1,d2,d3] = size(Stack); -modelName = sprintf('fixedStacks/%s_ali%d.erase',fileName,mapBackIter + 1); +modelName = sprintf('fixedStacks/%s_ali%d.erase',fileName,mapBackIter + 1); if (mapBackIter) % The bead model needs to be updated. tiltxf = sprintf('mapBack%d/%s_ali%d_ctf.tltxf',mapBackIter,fileName,mapBackIter); old_model = sprintf('fixedStacks/%s_ali%d.erase',fileName, mapBackIter); - - % Make sure the erase model is set up for the fixed stack. This model - % will be updated if tomoCPR is run. - if isfile(old_model) - [fail] = system(sprintf('imodtrans -2 %s %s fixedStacks/%s_ali%d.erase', ... - tiltxf, old_model , fileName, mapBackIter + 1)); - if (fail) - fprintf('model %s exists : %d\n', old_model,exist(old_model,'file')); - fprintf('xf %s exists : %d\n', tiltxf,exist(tiltxf,'file')); - - error('imodtrans failed to update the bead erase model fixedStacks/%s_ali%d.erase',fileName, mapBackIter) - end - - else - fprintf('WARNING: skipping bead erasing, b/c no file %s is found',old_model); - return - end - - -else - % Make sure the erase model is set up for the fixed stack. This model - % will be updated if tomoCPR is run. - if ~isfile(sprintf('fixedStacks/%s.erase',fileName)) - fprintf('WARNING: skipping bead erasing, b/c no file fixedStacks/%s.erase is found',fileName); - return - end - - [fail] = system(sprintf('imodtrans -i fixedStacks/%s.fixed fixedStacks/%s.erase %s',fileName,fileName,modelName)); - if (fail) - error('imodtrans failed to set the original bead erase model to the fixed stack header'); + % Make sure the erase model is set up for the fixed stack. This model + % will be updated if tomoCPR is run. + if isfile(old_model) + [fail] = system(sprintf('imodtrans -2 %s %s fixedStacks/%s_ali%d.erase', ... + tiltxf, old_model , fileName, mapBackIter + 1)); + if (fail) + fprintf('model %s exists : %d\n', old_model,exist(old_model,'file')); + fprintf('xf %s exists : %d\n', tiltxf,exist(tiltxf,'file')); + + error('imodtrans failed to update the bead erase model fixedStacks/%s_ali%d.erase',fileName, mapBackIter) end + else + fprintf('WARNING: skipping bead erasing, b/c no file %s is found',old_model); + return + end + + +else + + % Make sure the erase model is set up for the fixed stack. This model + % will be updated if tomoCPR is run. + if ~isfile(sprintf('fixedStacks/%s.erase',fileName)) + fprintf('WARNING: skipping bead erasing, b/c no file fixedStacks/%s.erase is found',fileName); + return + end + + [fail] = system(sprintf('imodtrans -i fixedStacks/%s.fixed fixedStacks/%s.erase %s',fileName,fileName,modelName)); + if (fail) + error('imodtrans failed to set the original bead erase model to the fixed stack header'); + end + end [fail] = system(sprintf('model2point %s %s_txt',modelName,modelName)); if (fail) - error('model2point failed to convert the bead erase model to text') -end + error('model2point failed to convert the bead erase model to text') +end @@ -76,69 +76,69 @@ for iPrj = 1:d3 -iProjection = Stack(:,:,iPrj); -iPrj_inFullStack = TLT(iPrj, 23); - -oX = round(beadModel( abs(beadModel(:,3) - iPrj_inFullStack +1) < 10^-1, 1)); -oY = round(beadModel( abs(beadModel(:,3) - iPrj_inFullStack +1) < 10^-1, 2)); - -if (useGPU) - iMask = zeros(size(iProjection),'single','gpuArray'); -else - iMask = zeros(size(iProjection),'single'); -end - + iProjection = Stack(:,:,iPrj); + iPrj_inFullStack = TLT(iPrj, 23); + + oX = round(beadModel( abs(beadModel(:,3) - iPrj_inFullStack +1) < 10^-1, 1)); + oY = round(beadModel( abs(beadModel(:,3) - iPrj_inFullStack +1) < 10^-1, 2)); + + if (useGPU) + iMask = zeros(size(iProjection),'single','gpuArray'); + else + iMask = zeros(size(iProjection),'single'); + end + for iBead = 1:length(oX) - startX = max(1,oX(iBead)-beadRadius-6); - endX = min(oX(iBead)+beadRadius+5, size(iProjection,1)); - startY = max(1,oY(iBead)-beadRadius-6); - endY = min(oY(iBead)+beadRadius+5, size(iProjection,2)); + startX = max(1,oX(iBead)-beadRadius-6); + endX = min(oX(iBead)+beadRadius+5, size(iProjection,1)); + startY = max(1,oY(iBead)-beadRadius-6); + endY = min(oY(iBead)+beadRadius+5, size(iProjection,2)); + + if (endX - startX <=1) || (endY - startY <= 1) + fprintf('Skipping bead from %d-%d X, %d-%d, Y iPrj %d\n', ... + startX, endX, startY, endY, iPrj); + else + % Avoid overlap + currentTile = iMask(startX:endX,startY:endY); + padVal = BH_multi_padVal(size(currentTile),size(gKernel)); + currentTile = BH_padZeros3d(currentTile,'fwd',padVal,deviceFlag,'single'); - if (endX - startX <=1) || (endY - startY <= 1) - fprintf('Skipping bead from %d-%d X, %d-%d, Y iPrj %d\n', ... - startX, endX, startY, endY, iPrj); - else - % Avoid overlap - currentTile = iMask(startX:endX,startY:endY); - padVal = BH_multi_padVal(size(currentTile),size(gKernel)); - currentTile = BH_padZeros3d(currentTile,'fwd',padVal,deviceFlag,'single'); - - beadMask = currentTile > gKernel; - - iMask(startX:endX,startY:endY) = BH_padZeros3d((gKernel.*(~beadMask)+beadMask.*currentTile),'inv',padVal,deviceFlag,'single'); - end + beadMask = currentTile > gKernel; + + iMask(startX:endX,startY:endY) = BH_padZeros3d((gKernel.*(~beadMask)+beadMask.*currentTile),'inv',padVal,deviceFlag,'single'); + end end - -iMaskInv = 1- iMask; -% AVG = real(ifftn(fftn(iProjection).*gKernel)); - -% maxP = max(AVG(:)); -% minP = min(AVG(:)); -% hInc = (maxP-minP)/1000; -% hVal = minP:hInc:maxP-hInc; -% -% h = hist(AVG(:),hVal); -% -% dVal = diff(h); -% -% cutOFF = find(dVal > 0.33*max(dVal(:)),1,'first'); -% -% intCutOFF = hVal(cutOFF); -% -% iMask = AVG < intCutOFF; -% -dataMean = mean(iProjection(iMask<0.01)); -dataRMS = rms(iProjection(iMask<0.01)-dataMean); - -%noiseImage = fftn(iProjection.*iMaskInv); -%noiseImage = real(ifftn(abs(noiseImage).*exp(1i.* (-pi + 2.*pi.*rand(size(noiseImage),'single','gpuArray'))))); -%Stack(:,:,iPrj) = iProjection.*iMaskInv + iMask.*noiseImage; - - Stack(:,:,iPrj) = iProjection.*iMaskInv + iMask.*(randn([d1,d2]).*dataRMS+dataMean); - - + + iMaskInv = 1- iMask; + % AVG = real(ifftn(fftn(iProjection).*gKernel)); + + % maxP = max(AVG(:)); + % minP = min(AVG(:)); + % hInc = (maxP-minP)/1000; + % hVal = minP:hInc:maxP-hInc; + % + % h = hist(AVG(:),hVal); + % + % dVal = diff(h); + % + % cutOFF = find(dVal > 0.33*max(dVal(:)),1,'first'); + % + % intCutOFF = hVal(cutOFF); + % + % iMask = AVG < intCutOFF; + % + dataMean = mean(iProjection(iMask<0.01)); + dataRMS = rms(iProjection(iMask<0.01)-dataMean); + + %noiseImage = fftn(iProjection.*iMaskInv); + %noiseImage = real(ifftn(abs(noiseImage).*exp(1i.* (-pi + 2.*pi.*rand(size(noiseImage),'single','gpuArray'))))); + %Stack(:,:,iPrj) = iProjection.*iMaskInv + iMask.*noiseImage; + + Stack(:,:,iPrj) = iProjection.*iMaskInv + iMask.*(randn([d1,d2]).*dataRMS+dataMean); + + end diff --git a/masking/BH_fftShift.m b/masking/BH_fftShift.m index 260ba15c..170f0f5f 100755 --- a/masking/BH_fftShift.m +++ b/masking/BH_fftShift.m @@ -44,7 +44,7 @@ % if (useGPU) % shiftMask = zeros(windowRadius.*shiftVect+1,'single','gpuArray'); % else -% shiftMask = zeros(windowRadius.*shiftVect+1,'single'); +% shiftMask = zeros(windowRadius.*shiftVect+1,'single'); % end @@ -56,28 +56,28 @@ % to a "centered" frame. if (doHalfGrid) [X,Y] = BH_multi_gridCoordinates([imgSize,1],'Cartesian',METHOD,{'none'},0,-1,0,{'halfgrid'}); - + % TODO fix the -2 option in gridCoordinates to do this sY = size(Y,2); oY = floor(sY/2) + 1; isOddY = mod(sY,2); flgIfft = flgIfft * isOddY; - + tmpY = Y(:,oY+isOddY-flgIfft:end); Y(:,oY+flgIfft:end) = Y(:,1:oY-1+isOddY-flgIfft); Y(:,1:oY-1+flgIfft) = tmpY; clear tmpY - + else [X,Y] = BH_multi_gridCoordinates([imgSize,1],'Cartesian',METHOD,{'none'},0,-2,0); end - + else if (doHalfGrid) imgSize [X,Y,Z] = BH_multi_gridCoordinates([imgSize],'Cartesian',METHOD,{'none'},0,-1,0,{'halfgrid'}); - + % TODO fix the -2 option in gridCoordinates to do this sZ = size(Z,3); oZ = floor(sZ/2) +1; @@ -88,7 +88,7 @@ oY = floor(sY/2) + 1; isOddY = mod(sY,2); flgIfftY = flgIfft * isOddY; - + tmpY2 = Y(:,oY+isOddY-flgIfftY:end,oZ+isOddZ-flgIfftZ:end); tmpY1 = Y(:,oY+isOddY-flgIfftY:end,1:oZ-1+isOddZ-flgIfftZ); % Put in 2 from 0 @@ -96,10 +96,10 @@ % Put in 1 from 3 Y(:,oY:end,1:oZ-1) = Y(:,1:oY-1+isOddY,oZ+isOddZ:end); % Put in 0 from tmpY2 - Y(:,1:oY-1+flgIfftY,1:oZ-1+flgIfftZ) = tmpY2; + Y(:,1:oY-1+flgIfftY,1:oZ-1+flgIfftZ) = tmpY2; % Put in 3 from 1 Y(:,1:oY-1+flgIfftY,oZ+flgIfftZ:end) = tmpY1; clear tmpY2 tmpY1 - + tmpZ2 = Z(:,oY+isOddY-flgIfftY:end,oZ+isOddZ-flgIfftZ:end); tmpZ1 = Z(:,oY+isOddY-flgIfftY:end,1:oZ-1+isOddZ-flgIfftZ); % Put in 2 from 0 @@ -107,12 +107,12 @@ % Put in 1 from 3 Z(:,oY:end,1:oZ-1) = Z(:,1:oY-1+isOddY,oZ+isOddZ:end); % Put in 0 from tmpY2 - Z(:,1:oY-1+flgIfftY,1:oZ-1+flgIfftZ) = tmpZ2; + Z(:,1:oY-1+flgIfftY,1:oZ-1+flgIfftZ) = tmpZ2; % Put in 3 from 1 Z(:,1:oY-1+flgIfftY,oZ+flgIfftZ:end) = tmpZ1; clear tmpY2 tmpY1 else - [X,Y,Z] = BH_multi_gridCoordinates(imgSize,'Cartesian',METHOD,{'none'},0,-2,0); + [X,Y,Z] = BH_multi_gridCoordinates(imgSize,'Cartesian',METHOD,{'none'},0,-2,0); end end @@ -121,24 +121,24 @@ % Cut out the grid coordinates. if ( gather(windowRadius(1)) ) % Set the extraval to zero, will it break? - if windowRadius(1) < 0 + if windowRadius(1) < 0 % Tread as a diameter padVal = BH_multi_padVal(imgSize, abs(windowRadius)); else padVal = BH_multi_padVal(imgSize, windowRadius.*shiftVect+0); end - - - X = BH_padZeros3d(X,padVal(1,:),padVal(2,:),METHOD,'single'); - Y = BH_padZeros3d(Y,padVal(1,:),padVal(2,:),METHOD,'single'); - + + + X = BH_padZeros3d(X,padVal(1,:),padVal(2,:),METHOD,'single'); + Y = BH_padZeros3d(Y,padVal(1,:),padVal(2,:),METHOD,'single'); + end if (flg2D) shiftMask = sub2ind(size(X),X,Y); else if ( windowRadius ) - Z = BH_padZeros3d(Z,padVal(1,:),padVal(2,:),METHOD,'single'); + Z = BH_padZeros3d(Z,padVal(1,:),padVal(2,:),METHOD,'single'); end shiftMask = sub2ind(size(X),X,Y,Z); end diff --git a/masking/BH_mask3d.m b/masking/BH_mask3d.m index 35b74351..e9981d27 100755 --- a/masking/BH_mask3d.m +++ b/masking/BH_mask3d.m @@ -1,12 +1,12 @@ function [ MASK, volCOM ] = BH_mask3d( SHAPE, SIZE, RADIUS, CENTER, varargin) %Create a mask for real space 3d images. -% +% % Input variables: % % SHAPE = 'sphere' or 'cylinder' : string % SIZE = [x, y, z] dimension of mask : vector, int % RADIUS= [rx, ry, rz] radius in each dimension : vector, float -% +% % CENTER= [cx, cy, cz] center of the mask : vector, float % % Output variables: @@ -21,7 +21,7 @@ % apodization in real space, or similarly the fall off in reciprocal % space. Any sharp transitions in either can influence alignment % negatively, and even result in falsely inflating reslution estimations. -% +% % This is avoided by -- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % @@ -34,46 +34,46 @@ flgCOM = 0; flg3d = 1; - global bh_global_binary_mask_low_pass - global bh_global_binary_mask_threshold - global bh_global_vol_est_scaling - if isempty(bh_global_binary_mask_low_pass) - bh_global_binary_mask_low_pass = 14; - end - if isempty(bh_global_binary_mask_threshold) - bh_global_binary_mask_threshold = 2.5; - end - if isempty(bh_global_vol_est_scaling) - % The low pass version of the map used for the estimate overestimates - % the molecular volume at the hydration radius of the underlying atoms. - % This flag will override the value I've calculated which depends on - % the masking resolution. TODO when the map resolution is lower than - % the masking resolution, this will again underestimate the scaling, - % artificialy *de*pressing the FSC - bh_global_vol_est_scaling = 0.0; - end - - if (bh_global_vol_est_scaling == 0) - localParticleScaling = (-2.8e-3) .* bh_global_binary_mask_low_pass.^2 + ... - 0.14 .* bh_global_binary_mask_low_pass + 1.5; - else - localParticleScaling = bh_global_vol_est_scaling; - end - - +global bh_global_binary_mask_low_pass +global bh_global_binary_mask_threshold +global bh_global_vol_est_scaling +if isempty(bh_global_binary_mask_low_pass) + bh_global_binary_mask_low_pass = 14; +end +if isempty(bh_global_binary_mask_threshold) + bh_global_binary_mask_threshold = 2.5; +end +if isempty(bh_global_vol_est_scaling) + % The low pass version of the map used for the estimate overestimates + % the molecular volume at the hydration radius of the underlying atoms. + % This flag will override the value I've calculated which depends on + % the masking resolution. TODO when the map resolution is lower than + % the masking resolution, this will again underestimate the scaling, + % artificialy *de*pressing the FSC + bh_global_vol_est_scaling = 0.0; +end + +if (bh_global_vol_est_scaling == 0) + localParticleScaling = (-2.8e-3) .* bh_global_binary_mask_low_pass.^2 + ... + 0.14 .* bh_global_binary_mask_low_pass + 1.5; +else + localParticleScaling = bh_global_vol_est_scaling; +end + + asymmetricRestriction = 0; if nargin > 5 if strcmp(varargin{1},'2d') flg3d = 0; end asymmetricRestriction = varargin{2}; - + elseif nargin > 4 if strcmp(varargin{1},'2d') flg3d = 0; else - flgCOM = 1; + flgCOM = 1; end end @@ -89,76 +89,76 @@ % Check that input is approprate. [mShape, mSize, mRadius, mCenter, binaryMask,fscMask] = ... - parseVariables( SHAPE, SIZE, RADIUS, CENTER, flg3d); + parseVariables( SHAPE, SIZE, RADIUS, CENTER, flg3d); if (asymmetricRestriction ~= 0 && ~strcmpi(mShape,'Cylinder')) error('Experimental symmetry restricting mask is only available for cylinders right now.'); end - METHOD = 'GPU'; +METHOD = 'GPU'; - clear mWindow - if strcmpi(mShape, 'rectangle') - if (flg3d) - mWindow(2*mRadius(1)+14-1, 2*mRadius(2)+14-1, 2*mRadius(3)+14-1) = ... - gpuArray(single(0)); - else - mWindow(2*mRadius(1)+14-1, 2*mRadius(2)+14-1) = gpuArray(single(0)); - end - mWindow = mWindow + 1; - else - mWindow = zeros(mSize,'single','gpuArray'); - mWindow = mWindow + 1; - end - +clear mWindow +if strcmpi(mShape, 'rectangle') if (flg3d) - [ gaussKernel ] = gpuArray(BH_multi_gaussian3d(5, 0.5 )); + mWindow(2*mRadius(1)+14-1, 2*mRadius(2)+14-1, 2*mRadius(3)+14-1) = ... + gpuArray(single(0)); else - [ gaussKernel ] = gpuArray(BH_multi_gaussian2d(5, 0.5 )); + mWindow(2*mRadius(1)+14-1, 2*mRadius(2)+14-1) = gpuArray(single(0)); end + mWindow = mWindow + 1; +else + mWindow = zeros(mSize,'single','gpuArray'); + mWindow = mWindow + 1; +end +if (flg3d) + [ gaussKernel ] = gpuArray(BH_multi_gaussian3d(5, 0.5 )); +else + [ gaussKernel ] = gpuArray(BH_multi_gaussian2d(5, 0.5 )); +end -if strcmpi(mShape, 'sphere') +if strcmpi(mShape, 'sphere') + if (flg3d) [ G1,G2,G3,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cartesian',METHOD,... - {'single',... - [1,0,0;0,1,0;0,0,1],... - mCenter','forward',1,1}, ... - 0, 1, 0 ); + {'single',... + [1,0,0;0,1,0;0,0,1],... + mCenter','forward',1,1}, ... + 0, 1, 0 ); else [ G1,G2,~,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cartesian',METHOD, ... - {'single',... - [1,0,0;0,1,0;0,0,1],... - mCenter','forward',1,1}, ... - 0, 1, 0 ); + {'single',... + [1,0,0;0,1,0;0,0,1],... + mCenter','forward',1,1}, ... + 0, 1, 0 ); end - - ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2; + + ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2; if (flg3d) ellipsoid = ellipsoid + (G3./mRadius(3)).^2; end -fullMask = (ellipsoid <= 1) ; -mWindow = mWindow .* fullMask; -for iShell = 1:pixelFallOff - - ellipsoid = (G1./(mRadius(1)+iShell)).^2 + (G2./(mRadius(2)+iShell)).^2; - if (flg3d) - ellipsoid = ellipsoid + (G3./(mRadius(3)+iShell)).^2 ; + fullMask = (ellipsoid <= 1) ; + mWindow = mWindow .* fullMask; + for iShell = 1:pixelFallOff + + ellipsoid = (G1./(mRadius(1)+iShell)).^2 + (G2./(mRadius(2)+iShell)).^2; + if (flg3d) + ellipsoid = ellipsoid + (G3./(mRadius(3)+iShell)).^2 ; + end + ellipsoid = (ellipsoid <= 1) - mWindow; + mWindow = mWindow + (ellipsoid .* taper(iShell)); end - ellipsoid = (ellipsoid <= 1) - mWindow; - mWindow = mWindow + (ellipsoid .* taper(iShell)); -end -clear borderMask G1 G2 G3 - mWindow = convn(mWindow, gaussKernel, 'same') ; - mWindow = mWindow ./ max(mWindow(:)); - % Set the edges at mRadius back to 1 after convolution - mWindow(fullMask) = 1; - mWindow(mWindow < convCutLow) = 0; + clear borderMask G1 G2 G3 + mWindow = convn(mWindow, gaussKernel, 'same') ; + mWindow = mWindow ./ max(mWindow(:)); + % Set the edges at mRadius back to 1 after convolution + mWindow(fullMask) = 1; + mWindow(mWindow < convCutLow) = 0; end % z filtering for cylinder @@ -166,92 +166,92 @@ if (flg3d) [ G1,G2,G3,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cartesian',METHOD,... - {'single',... - [1,0,0;0,1,0;0,0,1],... - mCenter','forward',1,1}, ... - 0, 1, 0 ); - G3 = abs(G3); + {'single',... + [1,0,0;0,1,0;0,0,1],... + mCenter','forward',1,1}, ... + 0, 1, 0 ); + G3 = abs(G3); else [ G1,G2,~,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cartesian',METHOD,... - {'single',... - [1,0,0;0,1,0;0,0,1],... - mCenter','forward',1,1}, ... - 0, 1, 0 ); + {'single',... + [1,0,0;0,1,0;0,0,1],... + mCenter','forward',1,1}, ... + 0, 1, 0 ); end - - -ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2; - -if (flg3d) - fullMask = (ellipsoid <= 1) & (G3 <= mRadius(3)); -else - fullMask = (ellipsoid <= 1); -end - -if (asymmetricRestriction) + + + ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2; + if (flg3d) - [ ~,angles,~,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cylindrical',METHOD,... - {'single',... - [1,0,0;0,1,0;0,0,1],... - mCenter','forward',1,1}, ... - 0, 1, 0 ); - G3 = abs(G3); + fullMask = (ellipsoid <= 1) & (G3 <= mRadius(3)); else - [ ~,angles,~,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cylindrical',METHOD,... - {'single',... - [1,0,0;0,1,0;0,0,1],... - mCenter','forward',1,1}, ... - 0, 1, 0 ); + fullMask = (ellipsoid <= 1); end - sectorMax = 2*pi/asymmetricRestriction * 1.025; - angles = (angles > (2*pi-sectorMax/2) | angles < sectorMax/2); -% % % gc = BH_multi_gaussian3d(-1.*size(angles),1.5); -% % % mWindow = real(ifftn(fftn(angles.*fullMask).*gc)); - KERNEL = EMC_gaussianKernel([1,5], 1.5, 'gpu', {}); - mWindow = EMC_convn(single(angles.*fullMask), KERNEL); - clear KERNEL - mWindow = mWindow ./ max(angles(:)); - -else - - mWindow = mWindow .* fullMask; - -for iShell = 1:pixelFallOff - ellipsoid = (G1./(mRadius(1)+iShell)).^2 + ... - (G2./(mRadius(2)+iShell)).^2; - if (flg3d) - ellipsoid = ((ellipsoid <= 1).*(G3 <= mRadius(3)+iShell)) - mWindow; + if (asymmetricRestriction) + if (flg3d) + [ ~,angles,~,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cylindrical',METHOD,... + {'single',... + [1,0,0;0,1,0;0,0,1],... + mCenter','forward',1,1}, ... + 0, 1, 0 ); + G3 = abs(G3); + else + [ ~,angles,~,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cylindrical',METHOD,... + {'single',... + [1,0,0;0,1,0;0,0,1],... + mCenter','forward',1,1}, ... + 0, 1, 0 ); + end + + sectorMax = 2*pi/asymmetricRestriction * 1.025; + angles = (angles > (2*pi-sectorMax/2) | angles < sectorMax/2); + % % % gc = BH_multi_gaussian3d(-1.*size(angles),1.5); + % % % mWindow = real(ifftn(fftn(angles.*fullMask).*gc)); + KERNEL = EMC_gaussianKernel([1,5], 1.5, 'gpu', {}); + mWindow = EMC_convn(single(angles.*fullMask), KERNEL); + clear KERNEL + mWindow = mWindow ./ max(angles(:)); + else - ellipsoid = ((ellipsoid <= 1) - mWindow); + + mWindow = mWindow .* fullMask; + + for iShell = 1:pixelFallOff + ellipsoid = (G1./(mRadius(1)+iShell)).^2 + ... + (G2./(mRadius(2)+iShell)).^2; + if (flg3d) + ellipsoid = ((ellipsoid <= 1).*(G3 <= mRadius(3)+iShell)) - mWindow; + else + ellipsoid = ((ellipsoid <= 1) - mWindow); + end + + mWindow = mWindow + (ellipsoid .* taper(iShell)); + + mWindow = convn(mWindow, gaussKernel, 'same') ; + mWindow = mWindow ./ max(mWindow(:)); + % Set the edges at mRadius back to 1 after convolution + mWindow(fullMask) = 1; + mWindow(mWindow < convCutLow) = 0; + end end - - mWindow = mWindow + (ellipsoid .* taper(iShell)); - - mWindow = convn(mWindow, gaussKernel, 'same') ; - mWindow = mWindow ./ max(mWindow(:)); - % Set the edges at mRadius back to 1 after convolution - mWindow(fullMask) = 1; - mWindow(mWindow < convCutLow) = 0; -end -end -clear borderMask G1 G2 G3 - + clear borderMask G1 G2 G3 + end % rectangular filtering if strcmpi(mShape, 'rectangle') - - - + + + [ padVal ] = BH_multi_padVal(size(mWindow), mSize); prePad = padVal(1,:); postPad= padVal(2,:); - + % Still doesn't really address an off-centered mask, but for this particular % case, this works with the new definition of the origin. - + shiftOrigin = ~mod(mSize,2); prePad = prePad + shiftOrigin ; @@ -262,123 +262,123 @@ postTrim = abs(postPad) .* (postPad < 0); prePad = prePad .* (prePad >= 0); postPad = postPad .* (postPad >= 0); - + % UPDATE TO USE padZeros3d TO TRIM if (flg3d) mWindow = mWindow(preTrim(1) + 1: end - postTrim(1), ... - preTrim(2) + 1: end - postTrim(2), ... - preTrim(3) + 1: end - postTrim(3)); + preTrim(2) + 1: end - postTrim(2), ... + preTrim(3) + 1: end - postTrim(3)); else mWindow = mWindow(preTrim(1) + 1: end - postTrim(1), ... - preTrim(2) + 1: end - postTrim(2)); + preTrim(2) + 1: end - postTrim(2)); end - + mWindow = BH_padZeros3d(mWindow,prePad,postPad,METHOD,'singleTaper'); - + end clear X Y Z x y z if strcmpi(mShape, 'BINARY') % Select a pretty aggressive cutoff, then dilate by at 10A or at least 2 % pixels. -% % % binaryMask = binaryMask - mean(binaryMask(:)); -% % % binaryMask = binaryMask ./ rms(binaryMask(:)); + % % % binaryMask = binaryMask - mean(binaryMask(:)); + % % % binaryMask = binaryMask ./ rms(binaryMask(:)); pixelSize = SIZE; - + if (flg3d) rectMask = BH_mask3d('rectangle',size(binaryMask), ... - (size(binaryMask)./2 - 7),[0,0,0]); + (size(binaryMask)./2 - 7),[0,0,0]); else rectMask = BH_mask3d('rectangle',size(binaryMask), ... - (size(binaryMask)./2 - 7),[0,0,0],'2d'); + (size(binaryMask)./2 - 7),[0,0,0],'2d'); end - - - + + + if (flg3d) binaryMask = BH_bandLimitCenterNormalize(medfilt3(gather(binaryMask),[3,3,3]).*rectMask, ... - BH_bandpass3d(size(binaryMask), ... - 0,0, ... - bh_global_binary_mask_low_pass, ... - 'GPU',pixelSize),... - (rectMask > .01),... - [0,0,0;0,0,0], 'single'); + BH_bandpass3d(size(binaryMask), ... + 0,0, ... + bh_global_binary_mask_low_pass, ... + 'GPU',pixelSize),... + (rectMask > .01),... + [0,0,0;0,0,0], 'single'); else binaryMask = BH_bandLimitCenterNormalize(medfilt2(gather(binaryMask),[3,3]).*rectMask, ... - BH_bandpass3d([size(binaryMask),1], ... - 0,0,... - bh_global_binary_mask_low_pass, ... - 'GPU',pixelSize),... - (rectMask > .01),... - [0,0;0,0], 'single'); + BH_bandpass3d([size(binaryMask),1], ... + 0,0,... + bh_global_binary_mask_low_pass, ... + 'GPU',pixelSize),... + (rectMask > .01),... + [0,0;0,0], 'single'); end - + binaryMask = real(ifftn(binaryMask)).*rectMask; -% % % -% % % binaryOutside = (binaryVol > 0.5); -% % % if (fscMask) -% % % maxThreshold = mean(binaryVol(binaryOutside)) + ... -% % % 1.5.* std(binaryVol(binaryOutside)) -% % % else -% % % -% % % maxThreshold = mean(binaryVol(binaryOutside)) + ... -% % % 3.* std(binaryVol(binaryOutside)) -% % % end -% % % % maxThreshold = (kurtosis(binaryVol(:))-3).^0.5; -% % % binaryMask = (binaryVol > maxThreshold ); - -% % Just to test a hunch -% if (flg3d) -% binarySmooth = (medfilt3(gather(binaryMask),[3,3,3])); -% else -% binarySmooth = (medfilt2(gather(binaryMask),[3,3])); -% end + % % % + % % % binaryOutside = (binaryVol > 0.5); + % % % if (fscMask) + % % % maxThreshold = mean(binaryVol(binaryOutside)) + ... + % % % 1.5.* std(binaryVol(binaryOutside)) + % % % else + % % % + % % % maxThreshold = mean(binaryVol(binaryOutside)) + ... + % % % 3.* std(binaryVol(binaryOutside)) + % % % end + % % % % maxThreshold = (kurtosis(binaryVol(:))-3).^0.5; + % % % binaryMask = (binaryVol > maxThreshold ); + + % % Just to test a hunch + % if (flg3d) + % binarySmooth = (medfilt3(gather(binaryMask),[3,3,3])); + % else + % binarySmooth = (medfilt2(gather(binaryMask),[3,3])); + % end + - maxThreshold = bh_global_binary_mask_threshold.*(std(binaryMask(binaryMask(:)>0))); binaryVol = binaryMask > maxThreshold; clear binarySmooth - + if (fscMask) dilationThresholds = [ 0.9 0.85 0.75 0.7 0.65 0.5 0.35 0.2 0.1 ] ; else dilationThresholds = [ 1.0000 0.9 ] ; end for threshold = dilationThresholds.*maxThreshold - - % figure, imshow3D(binaryMask) 0.6923 0.6154 - if threshold >= 0 - currentMask = single(gpuArray(binaryVol)); - - if (flg3d) - dilationKernel = gpuArray(BH_multi_gaussian3d(3.*[1,1,1],3.0)); - else - dilationKernel = gpuArray(BH_multi_gaussian2d(3.*[1,1],3.0)); - end - %figure, imshow3D(gather(b)) - - if (fscMask) - dilationIter = ceil(threshold.^2./3); - else - dilationIter = ceil(threshold.^2./3); - end - - % Grow - for i = 1:dilationIter - currentMask = single(~currentMask.*convn(currentMask,dilationKernel,'same') > 0.00)+currentMask; - end - + % figure, imshow3D(binaryMask) 0.6923 0.6154 + if threshold >= 0 + currentMask = single(gpuArray(binaryVol)); + + if (flg3d) + dilationKernel = gpuArray(BH_multi_gaussian3d(3.*[1,1,1],3.0)); + else + dilationKernel = gpuArray(BH_multi_gaussian2d(3.*[1,1],3.0)); + end + %figure, imshow3D(gather(b)) + + if (fscMask) + dilationIter = ceil(threshold.^2./3); + else + dilationIter = ceil(threshold.^2./3); + end + + % Grow + for i = 1:dilationIter + currentMask = single(~currentMask.*convn(currentMask,dilationKernel,'same') > 0.00)+currentMask; + end + + binaryVol = (binaryMask.*currentMask > threshold); - -% figure, imshow3D(gather(binaryVol)) - - - end + + % figure, imshow3D(gather(binaryVol)) + + + end end % It should already be binary here right? - + particleVolEstimate = sum(binaryVol(:) == 1); % Return volume estimate for visualization/ analysis. Zero out first pixel in FSC. volCOM = single(binaryVol); @@ -387,15 +387,15 @@ bw = bwdist(binaryVol(:,:,i)); binaryVol(:,:,i) = binaryVol(:,:,i) + bw < mRadius / 2 ; %%% Comment out for tight mask demo end - clear bw -% a = (binaryMask.*binaryVol > threshold); + clear bw + % a = (binaryMask.*binaryVol > threshold); % Bringback the edges %particleVolEstimate = sum(binaryVol(:)); - + if (fscMask) - % figure, imshow3D(gather(a)) + % figure, imshow3D(gather(a)) currentMask = single(gpuArray(binaryVol)); - + if (flg3d) taperKernel = gpuArray(BH_multi_gaussian3d(4.*[1,1,1],1.75)); else @@ -409,41 +409,41 @@ if (flg3d) smoothKernel = gpuArray(BH_multi_gaussian3d(mRadius.*[1,1,1],mRadius./2)); else - smoothKernel = gpuArray(BH_multi_gaussian2d(mRadius.*[1,1],mRadius./2)); + smoothKernel = gpuArray(BH_multi_gaussian2d(mRadius.*[1,1],mRadius./2)); end for i = 1:2 %%% set to one for tight mask demo currentMask = convn(currentMask,smoothKernel,'same'); - % figure, imshow3D(gather(currentMask)) + % figure, imshow3D(gather(currentMask)) end currentMask = currentMask ./ max(currentMask(:)); - + % We assume the particle envelope only cuts through solvent, which % is not always the case in cryoSTAC of extended assemblies. To estimate % The power reduction in the signal is (I think) a good idea, because - % Otherwise the noise reduction term is too strong. + % Otherwise the noise reduction term is too strong. % Get the reduction in power in the taper region due to masking powerReduction = sum(abs(binaryMask(:)).^2.*(currentMask(:)>0))./ ... - sum(abs(binaryMask(:)).^2.* currentMask(:)); + sum(abs(binaryMask(:)).^2.* currentMask(:)); maskVolume = sum(currentMask(:)>0); particleVolEstimate = particleVolEstimate ./ localParticleScaling; particleFraction = particleVolEstimate ./ maskVolume .* powerReduction; fprintf('Estimated partVol, %d voxels\nmaskVol %d voxels\npwrReduction %2.3f\npartFract %2.3f\n',... - particleVolEstimate, maskVolume, powerReduction,particleFraction); - + particleVolEstimate, maskVolume, powerReduction,particleFraction); + % Should probably use varargout, but for now, returning the center of % mass is not done at the same stage as a particle volume estimate. - + volCOM(1) = 1/particleFraction; - - % figure, imshow3D(gather(b)) + + % figure, imshow3D(gather(b)) mWindow = gather(currentMask); else mWindow = gather(binaryVol); end - %figure, imshow3D(gather(mWindow)) - clear a b avgM - + %figure, imshow3D(gather(mWindow)) + clear a b avgM + end %mWindow(mWindow < .5.*fallOffVal) = 0; @@ -456,23 +456,23 @@ if (flgCOM) % get the center of mass of the low-pass filtered volume and return. [X,Y,Z,~,~,~] = BH_multi_gridCoordinates(size(binaryVol),'Cartesian',... - 'GPU',{'none'},0,1,0); - - binaryVol = (binaryVol - min(binaryVol(MASK > 0.01)) ).*MASK; + 'GPU',{'none'},0,1,0); + + binaryVol = (binaryVol - min(binaryVol(MASK > 0.01)) ).*MASK; volCOM = [sum(sum(sum(binaryVol.*X))), ... - sum(sum(sum(binaryVol.*Y))), ... - sum(sum(sum(binaryVol.*Z)))] ./ sum(binaryVol(:)); + sum(sum(sum(binaryVol.*Y))), ... + sum(sum(sum(binaryVol.*Z)))] ./ sum(binaryVol(:)); clear binaryVol X Y Z binaryMask + +end -end - - clearvars -except MASK volCOM +clearvars -except MASK volCOM end % end of BH_mask3d function. function [mShape, mSize, mRadius, mCenter, binaryMask,fscMask] = ... - parseVariables( SHAPE, SIZE, RADIUS, CENTER, flg3d) + parseVariables( SHAPE, SIZE, RADIUS, CENTER, flg3d) %Check for correct input parameters: % If SHAPE is an image, default to rectangular size of image and apply mask, @@ -506,14 +506,14 @@ mRadius = max(3,floor(10./pixelSize)); else - - if strcmpi(SHAPE, 'sphere') + + if strcmpi(SHAPE, 'sphere') mShape = SHAPE; - + dim = 1; elseif strcmpi(SHAPE, 'cylinder') mShape = SHAPE; - + dim = 2; elseif strcmpi(SHAPE, 'rectangle') mShape = SHAPE; @@ -521,26 +521,26 @@ else error('SHAPE must be sphere, cylinder, or rectangle not %s', SHAPE) end - - if ~isnumeric(SIZE) || ~((length(SIZE) == 3) || (length(SIZE) == 2)) + if ~isnumeric(SIZE) || ~((length(SIZE) == 3) || (length(SIZE) == 2)) + error('SIZE must be a vector in R3') else mSize = SIZE; end - + if ~isnumeric(RADIUS) || ~((length(RADIUS) == 3) || (length(SIZE) == 2)) error('RADIUS must be radius radius [height], dimension %d', dim) else mRadius = RADIUS; - + end - + if ~isnumeric(CENTER) || ~((length(CENTER) == 3) || (length(SIZE) == 2)) - error('CENTER must be a vector in R3') + error('CENTER must be a vector in R3') else - mCenter = CENTER; - + mCenter = CENTER; + end end diff --git a/masking/BH_mask3d_cpu.m b/masking/BH_mask3d_cpu.m index 43a26198..71d93086 100755 --- a/masking/BH_mask3d_cpu.m +++ b/masking/BH_mask3d_cpu.m @@ -1,12 +1,12 @@ function [ MASK ] = BH_mask3d( SHAPE, SIZE, RADIUS, CENTER ) %Create a mask for real space 3d images. -% +% % Input variables: % % SHAPE = 'sphere' or 'cylinder' : string % SIZE = [x, y, z] dimension of mask : vector, int % RADIUS= [rx, ry, rz] radius in each dimension : vector, float -% +% % CENTER= [cx, cy, cz] center of the mask : vector, float % % Output variables: @@ -21,7 +21,7 @@ % apodization in real space, or similarly the fall off in reciprocal % space. Any sharp transitions in either can influence alignment % negatively, and even result in falsely inflating reslution estimations. -% +% % This is avoided by -- %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % @@ -42,94 +42,94 @@ % Check that input is approprate. [mShape, mSize, mRadius, mCenter, binaryMask] = ... - parseVariables( SHAPE, SIZE, RADIUS, CENTER); + parseVariables( SHAPE, SIZE, RADIUS, CENTER); - - METHOD = 'cpu'; - clear mWindow - if strcmpi(mShape, 'rectangle') - mWindow(2*mRadius(1)+14-1, 2*mRadius(2)+14-1, 2*mRadius(3)+14-1) = (single(0)); - mWindow = mWindow + 1; - else - mWindow(mSize(1), mSize(2), mSize(3)) = (single(0)); - mWindow = mWindow + 1; - end - - [ gaussKernel ] = (BH_multi_gaussian3d(5, 0.5 )); +METHOD = 'cpu'; +clear mWindow +if strcmpi(mShape, 'rectangle') + mWindow(2*mRadius(1)+14-1, 2*mRadius(2)+14-1, 2*mRadius(3)+14-1) = (single(0)); + mWindow = mWindow + 1; +else + mWindow(mSize(1), mSize(2), mSize(3)) = (single(0)); + mWindow = mWindow + 1; +end +[ gaussKernel ] = (BH_multi_gaussian3d(5, 0.5 )); -if strcmpi(mShape, 'sphere') -[ G1,G2,G3,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cartesian', METHOD, ... - {'single',... - [1,0,0;0,1,0;0,0,1],... - mCenter','forward',1,1}, ... - 0, 1, 0 ); -ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2 + (G3./mRadius(3)).^2 ; -fullMask = (ellipsoid <= 1) ; -mWindow = mWindow .* fullMask; -for iShell = 1:pixelFallOff - ellipsoid = (G1./(mRadius(1)+iShell)).^2 + ... - (G2./(mRadius(2)+iShell)).^2 + ... - (G3./(mRadius(3)+iShell)).^2 ; - ellipsoid = (ellipsoid <= 1) - mWindow; - mWindow = mWindow + (ellipsoid .* taper(iShell)); -end -clear borderMask G1 G2 G3 - mWindow = convn(mWindow, gaussKernel, 'same') ; - mWindow = mWindow ./ max(mWindow(:)); - % Set the edges at mRadius back to 1 after convolution - mWindow(fullMask) = 1; - mWindow(mWindow < convCutLow) = 0; +if strcmpi(mShape, 'sphere') + + [ G1,G2,G3,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cartesian', METHOD, ... + {'single',... + [1,0,0;0,1,0;0,0,1],... + mCenter','forward',1,1}, ... + 0, 1, 0 ); + + ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2 + (G3./mRadius(3)).^2 ; + fullMask = (ellipsoid <= 1) ; + mWindow = mWindow .* fullMask; + for iShell = 1:pixelFallOff + ellipsoid = (G1./(mRadius(1)+iShell)).^2 + ... + (G2./(mRadius(2)+iShell)).^2 + ... + (G3./(mRadius(3)+iShell)).^2 ; + ellipsoid = (ellipsoid <= 1) - mWindow; + mWindow = mWindow + (ellipsoid .* taper(iShell)); + end + clear borderMask G1 G2 G3 + mWindow = convn(mWindow, gaussKernel, 'same') ; + mWindow = mWindow ./ max(mWindow(:)); + % Set the edges at mRadius back to 1 after convolution + mWindow(fullMask) = 1; + mWindow(mWindow < convCutLow) = 0; end % z filtering for cylinder if strcmpi(mShape, 'cylinder') -[ G1,G2,G3,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cartesian', METHOD, ... - {'single',... - [1,0,0;0,1,0;0,0,1],... - mCenter','forward',1,1}, ... - 0, 1, 0 ); -G3 = abs(G3); -ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2; -fullMask = (ellipsoid <= 1) & (G3 <= mRadius(3)); -mWindow = mWindow .* fullMask; -for iShell = 1:pixelFallOff - ellipsoid = (G1./(mRadius(1)+iShell)).^2 + ... - (G2./(mRadius(2)+iShell)).^2; - - ellipsoid = ((ellipsoid <= 1).*(G3 <= mRadius(3)+iShell)) - mWindow; - + [ G1,G2,G3,~,~,~ ] = BH_multi_gridCoordinates( mSize, 'Cartesian', METHOD, ... + {'single',... + [1,0,0;0,1,0;0,0,1],... + mCenter','forward',1,1}, ... + 0, 1, 0 ); + G3 = abs(G3); + ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2; + fullMask = (ellipsoid <= 1) & (G3 <= mRadius(3)); + mWindow = mWindow .* fullMask; + for iShell = 1:pixelFallOff + ellipsoid = (G1./(mRadius(1)+iShell)).^2 + ... + (G2./(mRadius(2)+iShell)).^2; + + ellipsoid = ((ellipsoid <= 1).*(G3 <= mRadius(3)+iShell)) - mWindow; + mWindow = mWindow + (ellipsoid .* taper(iShell)); - + mWindow = convn(mWindow, gaussKernel, 'same') ; mWindow = mWindow ./ max(mWindow(:)); % Set the edges at mRadius back to 1 after convolution - mWindow(fullMask) = 1; + mWindow(fullMask) = 1; mWindow(mWindow < convCutLow) = 0; -end -clear borderMask G1 G2 G3 - + end + clear borderMask G1 G2 G3 + end % rectangular filtering if strcmpi(mShape, 'rectangle') - - + + [ padVal ] = BH_multi_padVal(size(mWindow), mSize); prePad = padVal(1,:); postPad= padVal(2,:); % Still doesn't really address an off-centered mask, but for this particular % case, this works with the new definition of the origin. - + shiftOrigin = ~mod(mSize,2); prePad = prePad + shiftOrigin ; @@ -140,85 +140,85 @@ postTrim = abs(postPad) .* (postPad < 0); prePad = prePad .* (prePad >= 0); postPad = postPad .* (postPad >= 0); - + mWindow = mWindow(preTrim(1) + 1: end - postTrim(1), ... - preTrim(2) + 1: end - postTrim(2), ... - preTrim(3) + 1: end - postTrim(3)); - + preTrim(2) + 1: end - postTrim(2), ... + preTrim(3) + 1: end - postTrim(3)); + mWindow = BH_padZeros3d(mWindow,prePad,postPad,METHOD,'singleTaper'); - + end clear X Y Z x y z if strcmpi(mShape, 'BINARY') % Select a pretty aggressive cutoff, then dilate by at 10A or at least 2 % pixels. -% % % binaryMask = binaryMask - mean(binaryMask(:)); -% % % binaryMask = binaryMask ./ rms(binaryMask(:)); + % % % binaryMask = binaryMask - mean(binaryMask(:)); + % % % binaryMask = binaryMask ./ rms(binaryMask(:)); pixelSize = SIZE rectMask = BH_mask3d_cpu('rectangle',size(binaryMask),(size(binaryMask)./2 - 7),[0,0,0]); binaryVol = BH_bandLimitCenterNormalize_cpu(binaryMask.*rectMask, ... - BH_bandpass3d(size(binaryMask),0,0,24,METHOD,pixelSize),... - (rectMask > .01),... - [0,0,0;0,0,0], 'double'); + BH_bandpass3d(size(binaryMask),0,0,24,METHOD,pixelSize),... + (rectMask > .01),... + [0,0,0;0,0,0], 'double'); binaryVol = real(ifftn(binaryVol)); - -% figure, imshow3D(binaryMask) + + % figure, imshow3D(binaryMask) binaryMask = (binaryVol > 3.0);%(log(kurtosis(binaryMask(:)./3)))); a = single((binaryMask)); dilationKernel = (BH_multi_gaussian3d([3,3,3],3.0)); b = a; -% figure, imshow3D(gather(b)) + % figure, imshow3D(gather(b)) for i = 1:mRadius + 5 - b = single(~a.*convn(b,dilationKernel,'same') > 0.2)+a; - end + b = single(~a.*convn(b,dilationKernel,'same') > 0.2)+a; + end a = b; binaryMask = (binaryVol.*(a > 0.5) > 0.3);%(log(kurtosis(binaryMask(:)./3)))); a = single((binaryMask)); b = a; -% figure, imshow3D(gather(b)) - for i = 1:mRadius - b = single(~a.*convn(b,dilationKernel,'same') > 0.2)+a; - end + % figure, imshow3D(gather(b)) + for i = 1:mRadius + b = single(~a.*convn(b,dilationKernel,'same') > 0.2)+a; + end a = b; - -% figure, imshow3D(gather(a)) + + % figure, imshow3D(gather(a)) taperKernel = (BH_multi_gaussian3d([3,3,3],1.5)); for i = 1:5 b = ~(a).*convn(b,taperKernel,'same')+a; end -% figure, imshow3D(gather(b)) + % figure, imshow3D(gather(b)) smoothKernel = (BH_multi_gaussian3d([5,5,5],0.65)); - for i = 1:2 + for i = 1:2 b = convn(b,smoothKernel,'same')+b; end - % figure, imshow3D(gather(b)) + % figure, imshow3D(gather(b)) mWindow = gather(b ./ max(b(:))); - % figure, imshow3D(gather(mWindow)) - clear a b avgM - + % figure, imshow3D(gather(mWindow)) + clear a b avgM + end %mWindow(mWindow < .5.*fallOffVal) = 0; MASK = mWindow; clear mWindow - - clearvars -except MASK + +clearvars -except MASK end % end of BH_mask3d function. function [mShape, mSize, mRadius, mCenter, binaryMask] = ... - parseVariables( SHAPE, SIZE, RADIUS, CENTER) + parseVariables( SHAPE, SIZE, RADIUS, CENTER) %Check for correct input parameters: % If SHAPE is an image, default to rectangular size of image and apply mask, @@ -227,7 +227,7 @@ binaryMask = ''; if isnumeric(SHAPE) - binaryMask = SHAPE; + binaryMask = SHAPE; clear SHAPE mShape = 'BINARY'; mSize = size(binaryMask); @@ -238,14 +238,14 @@ pixelSize = SIZE; mRadius = max(3,floor(10./pixelSize)) else - - if strcmpi(SHAPE, 'sphere') + + if strcmpi(SHAPE, 'sphere') mShape = SHAPE; - + dim = 1; elseif strcmpi(SHAPE, 'cylinder') mShape = SHAPE; - + dim = 2; elseif strcmpi(SHAPE, 'rectangle') mShape = SHAPE; @@ -253,29 +253,29 @@ else error('SHAPE must be sphere, cylinder, or rectangle not %s', SHAPE) end - + if ~isnumeric(SIZE) || ~(length(SIZE) == 3) error('SIZE must be a vector in R3') else mSize = SIZE; end - + if ~isnumeric(RADIUS) || ~(length(RADIUS) == 3) error('RADIUS must be radius radius [height], dimension %d', dim) else mRadius = RADIUS; -% % % % For cylindircal/spherical reduce the radius by one (instead of 1 at the edge -% % % % then the value will be ~.995) -% % % if strcmpi(SHAPE, 'sphere') || strcmpi(SHAPE, 'cylinder') -% % % mRadius = mRadius - 1; -% % % end + % % % % For cylindircal/spherical reduce the radius by one (instead of 1 at the edge + % % % % then the value will be ~.995) + % % % if strcmpi(SHAPE, 'sphere') || strcmpi(SHAPE, 'cylinder') + % % % mRadius = mRadius - 1; + % % % end end - + if ~isnumeric(CENTER) || ~(length(CENTER) == 3) - error('CENTER must be a vector in R3') + error('CENTER must be a vector in R3') else - mCenter = CENTER; - + mCenter = CENTER; + end end diff --git a/masking/BH_multi_calcTaper.m b/masking/BH_multi_calcTaper.m index c7aa4813..9361400c 100644 --- a/masking/BH_multi_calcTaper.m +++ b/masking/BH_multi_calcTaper.m @@ -1,6 +1,6 @@ function [ taper ] = BH_multi_calcTaper(pixelFallOff) % Calculate a cosine edge -% This doesn't include the 1 that precedes it or the 0 that would follow, +% This doesn't include the 1 that precedes it or the 0 that would follow, % just the transition region. taper = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); diff --git a/masking/BH_multi_gaussian2d.m b/masking/BH_multi_gaussian2d.m index 685c7cd1..9084af65 100644 --- a/masking/BH_multi_gaussian2d.m +++ b/masking/BH_multi_gaussian2d.m @@ -31,11 +31,11 @@ end % If real space, shift origin to center, if fourier leave origin at corner. - [ X,Y,~,~,~,~ ] = BH_multi_gridCoordinates( hood, 'Cartesian', 'cpu', ... - {'none'}, flgFourier, ... - 1-flgFourier, 0 ); - - +[ X,Y,~,~,~,~ ] = BH_multi_gridCoordinates( hood, 'Cartesian', 'cpu', ... + {'none'}, flgFourier, ... + 1-flgFourier, 0 ); + + if (flgFourier) @@ -43,15 +43,15 @@ gfunc = @(x,y,s)(exp(-0.5*(x.^2+y.^2).*(s.*2.*pi).^(2))); normFactor = @(x,y,r,Ord)((2i.*pi).^Ord); else - gfunc = @(x,y,s)(exp(-0.5*(x.^2+y.^2)./s.^2)); + gfunc = @(x,y,s)(exp(-0.5*(x.^2+y.^2)./s.^2)); normFactor = @(x,y,s,Ord)((2.*pi).^-1.*(s^2).^(-(Ord+1))); end - + switch ORDER - case 0 - + case 0 + if ( flgFourier ) Kx = normFactor(X,Y,1,ORDER).*gfunc(X,Y,StdDev); else @@ -59,7 +59,7 @@ end case 1 - + if ( flgFourier ) Kx = -X.*normFactor(X,Y,X,ORDER).*gfunc(X,Y,StdDev); Ky = -Y.*normFactor(X,Y,Y,ORDER).*gfunc(X,Y,StdDev); @@ -84,7 +84,7 @@ % if (flgFourier) % KERNELX = gfunc(X,Y,StdDev); %KERNELX = KERNELX ./ max(KERNELX(:)); -% %KERNELY = gfunc(Y,StdDev); %KERNELY = KERNELY ./ max(KERNELY(:)); +% %KERNELY = gfunc(Y,StdDev); %KERNELY = KERNELY ./ max(KERNELY(:)); % else % KERNELX = gfunc(X,Y,StdDev); % end diff --git a/masking/BH_multi_gaussian3d.m b/masking/BH_multi_gaussian3d.m index 1e17ab82..469cae97 100755 --- a/masking/BH_multi_gaussian3d.m +++ b/masking/BH_multi_gaussian3d.m @@ -16,13 +16,13 @@ else fourthPow = 1; end - [ X,Y,Z,~,~,~ ] = BH_multi_gridCoordinates( hood, 'Cartesian', 'cpu', ... - {'none'}, 0, 1, 0 ); +[ X,Y,Z,~,~,~ ] = BH_multi_gridCoordinates( hood, 'Cartesian', 'cpu', ... + {'none'}, 0, 1, 0 ); % Zeroth order gaussian derivative gfunc = @(x,y,z,s)((2.*pi.*s.^2*fourthPow).^(-3/2).*... - exp(-0.5*(x./s).^(2*fourthPow)).*... - exp(-0.5*(y./s).^(2*fourthPow)).* ... - exp(-0.5*(z./s).^(2*fourthPow))); + exp(-0.5*(x./s).^(2*fourthPow)).*... + exp(-0.5*(y./s).^(2*fourthPow)).* ... + exp(-0.5*(z./s).^(2*fourthPow))); KERNEL = gfunc(X,Y,Z,StdDev); diff --git a/masking/BH_multi_makeHermitian.m b/masking/BH_multi_makeHermitian.m index bdab8c9c..5d30cefe 100644 --- a/masking/BH_multi_makeHermitian.m +++ b/masking/BH_multi_makeHermitian.m @@ -15,14 +15,14 @@ flg2d = false; end - + % pull the halfXform to the gpu incase it is not already there halfXform = gpuArray(halfXform); if (padConv) [nX,nY,nZ] = size(halfXform); nX = nX + originalSize(1); - + oX = originalSize(1)+1; isOdd = 1; fullXform = zeros([nX,nY,nZ],'single','gpuArray'); @@ -33,14 +33,14 @@ nZ = originalSize(3); oX = floor(nX/2)+1; isOdd = mod(nX,2); - - fullXform = zeros(originalSize,'single','gpuArray'); + + fullXform = zeros(originalSize,'single','gpuArray'); end hermitianMask = EMC_maskIndex('c2c', originalSize, 'GPU',{}); fullXform(1:size(halfXform,1), :, :) = halfXform; fullXform = fullXform(hermitianMask); - + % % % % NOTE this is now used for things that are abs()^2 but if used more % % % % generally an option to multiply the hermitian conjugate side by -1 will diff --git a/masking/BH_multi_padVal.m b/masking/BH_multi_padVal.m index b7b8188e..f6fb2e19 100755 --- a/masking/BH_multi_padVal.m +++ b/masking/BH_multi_padVal.m @@ -4,22 +4,22 @@ % "extra" position should preced the origin for an odd dimension and follow % the origin for an even volume. - % Forward padding - sizeDiff = size2 - size1; - oddDiff = mod(sizeDiff, 2); - oddInput= mod(size1, 2); - - padVal = [ floor(sizeDiff./2) + (oddInput) .* oddDiff ; ... - floor(sizeDiff./2) + ~(oddInput) .* oddDiff]; - - % InversePadding - sizeDiff = size1 - size2; - oddDiff = mod(sizeDiff, 2); - oddInput= mod(size2, 2); - - padVal = [padVal ; - floor(sizeDiff./2) + (oddInput) .* oddDiff ; ... - floor(sizeDiff./2) + ~(oddInput) .* oddDiff]; - +% Forward padding +sizeDiff = size2 - size1; +oddDiff = mod(sizeDiff, 2); +oddInput= mod(size1, 2); + +padVal = [ floor(sizeDiff./2) + (oddInput) .* oddDiff ; ... + floor(sizeDiff./2) + ~(oddInput) .* oddDiff]; + +% InversePadding +sizeDiff = size1 - size2; +oddDiff = mod(sizeDiff, 2); +oddInput= mod(size2, 2); + +padVal = [padVal ; + floor(sizeDiff./2) + (oddInput) .* oddDiff ; ... + floor(sizeDiff./2) + ~(oddInput) .* oddDiff]; + end diff --git a/masking/BH_multi_projectionMask.m b/masking/BH_multi_projectionMask.m index d48704d7..c48557df 100755 --- a/masking/BH_multi_projectionMask.m +++ b/masking/BH_multi_projectionMask.m @@ -19,8 +19,8 @@ % Do calc on GPU either way, and if cpu pull results [X1,Y1,~,~,~,~] = BH_multi_gridCoordinates([r1,r2],'Cartesian', ... - 'GPU',{'none'},0,1,0); - + 'GPU',{'none'},0,1,0); + clear deltaZ @@ -39,12 +39,12 @@ for iPrj = 1:d3 - + R = BH_defineMatrix([TLT(iPrj,6),TLT(iPrj,4),TLT(iPrj,6)], ... - 'Bah','forwardVector'); - + 'Bah','forwardVector'); + rInv = R'; - + % Z-starts at zero so it doesn't contribute X2 = X1.*R(1) + Y1.*R(4); Y2 = X1.*R(2) + Y1.*R(5); @@ -52,20 +52,20 @@ % To account for thickness, the X-dimension must be expanded Xmin = X2 + (Z1 - R(7)*r3); Xmax = X2 + (Z1 + R(7)*r3); - - + + % Now the gridvectors have values on a centered grid, shift back to % index coordinates, where the origin is at lower left = 1,1 x3min = (round(Xmin + d1/2 )); x3max = (round(Xmax + d1/2)); y3 = (round(Y2 + d2/2 )); - + % Remove any coordinates that are out of the original frame, the values here % are the x,y pairs that are sampled in the projection frame. x3max( x3max < 1 | x3max > d1) = 1; x3min( x3min < 1 | x3min > d1) = 1; y3( y3 < 1 | y3 > d2) = 1; - + % uniqI/J are coordinate indices in the unshifted frame that are sampled linInd = unique([sub2ind([d1,d2], x3min(:),y3(:));sub2ind([d1,d2], x3max(:),y3(:))]); @@ -73,9 +73,9 @@ xP = uniqI - d1/2; yP = uniqJ - d2/2; - + zP = -1/rInv(9).*(xP.*rInv(3)+yP.*rInv(6)); - + clear interpZ maskZ if strcmp(METHOD,'GPU') maskZ = false([d1,d2],'gpuArray'); @@ -91,22 +91,22 @@ if ( any(zShift) ) if strcmp(METHOD,'GPU') maskZ(linInd) = (zShift(1) - zShift(2) < interpZ(linInd) & ... - zShift(1) + zShift(2) > interpZ(linInd)); + zShift(1) + zShift(2) > interpZ(linInd)); else maskZ(linInd) = gather((zShift(1) - zShift(2) < interpZ(linInd) & ... - zShift(1) + zShift(2) > interpZ(linInd))); + zShift(1) + zShift(2) > interpZ(linInd))); end else maskZ(linInd) = true; end - + evalMask(:,:,TLT(iPrj,1)) = maskZ; deltaZ(:,:,TLT(iPrj,1)) = interpZ; - + end clear X1 Y1 X2 Y2 Z2 rSample rElevation rTilt rInPlane R l x1 y1 xShift yShift clear interpZ x3 y3 xNew yNew zNew linInd - + end diff --git a/masking/BH_padZeros3d.m b/masking/BH_padZeros3d.m index b538f591..ed96d174 100755 --- a/masking/BH_padZeros3d.m +++ b/masking/BH_padZeros3d.m @@ -1,7 +1,7 @@ function [ PADDED_IMG ] = BH_padZeros3d( IMAGE, PADLOW, PADTOP, ... - METHOD, PRECISION, varargin ) + METHOD, PRECISION, varargin ) %Pad an image volume with zeros. -% +% % % Input variables: % @@ -16,7 +16,7 @@ % METHOD = case sensitive 'GPU' otherwise cpu % % Output variables:BH_bandpass3d.m -% +% % PADDED_IMG = the padded image. % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -84,14 +84,14 @@ % If any pad values are negative, first trim the image. try -IMAGE = IMAGE(1+trimLOW(1):end-trimTOP(1),... - 1+trimLOW(2):end-trimTOP(2),... - 1+trimLOW(3):end-trimTOP(3)); + IMAGE = IMAGE(1+trimLOW(1):end-trimTOP(1),... + 1+trimLOW(2):end-trimTOP(2),... + 1+trimLOW(3):end-trimTOP(3)); catch -fprintf('%f %f %f\n,%f %f %f\n',PADLOW,PADTOP); -fprintf('%f %f %f\n, %f %f %f\n',trimLOW,trimTOP); + fprintf('%f %f %f\n,%f %f %f\n',PADLOW,PADTOP); + fprintf('%f %f %f\n, %f %f %f\n',trimLOW,trimTOP); end - + if ismatrix(IMAGE) imgSize = [size(IMAGE),1]; else @@ -143,7 +143,7 @@ if (doRand) PADDED_IMG = randn(padSize,'double').*extrapVal+extrapMean; else - PADDED_IMG = zeros(padSize,'double'); + PADDED_IMG = zeros(padSize,'double'); end else error('PRECISION must be single or double, not %s', PRECISION) @@ -157,20 +157,20 @@ if (twoD) if (taper) [d1,d2,d3] = size(IMAGE); - + IMAGE(:,1:7) = IMAGE(:,1:7) .* repmat(flip(taper),d1,1,d3) + ... - repmat(flip(extrapVal.*(1-taper)),d1,1,d3); + repmat(flip(extrapVal.*(1-taper)),d1,1,d3); IMAGE(1:7,:) = IMAGE(1:7,:) .* repmat(flip(taper)',1,d2,d3) + ... - repmat(flip(extrapVal.*(1-taper))',1,d2,d3); + repmat(flip(extrapVal.*(1-taper))',1,d2,d3); + - IMAGE(:,end-6:end) = IMAGE(:,end-6:end) .* repmat(taper,d1,1,d3) + ... - repmat(extrapVal.*(1-taper),d1,1,d3); + repmat(extrapVal.*(1-taper),d1,1,d3); IMAGE(end-6:end,:) = IMAGE(end-6:end,:) .* repmat(taper',1,d2,d3) + ... - repmat(extrapVal.*(1-taper'),1,d2,d3); - + repmat(extrapVal.*(1-taper'),1,d2,d3); + end - + if ( fourierOverSample ) sX1 = ceil((size(IMAGE,1)+1)./2); sX2 = size(IMAGE,1)-sX1-1; @@ -183,25 +183,25 @@ PADDED_IMG(end-sX2:end,end-sY2:end) = IMAGE(end-sX2:end,end-sY2:end); else PADDED_IMG(padLOW(1)+1: end - padTOP(1), ... - padLOW(2)+1: end - padTOP(2)) = IMAGE; + padLOW(2)+1: end - padTOP(2)) = IMAGE; end else if (taper) [d1,d2,d3] = size(IMAGE); - + IMAGE(:,1:7,:) = IMAGE(:,1:7,:) .* repmat(flip(taper),d1,1,d3) + ... - repmat(flip(extrapVal.*(1-taper)),d1,1,d3); + repmat(flip(extrapVal.*(1-taper)),d1,1,d3); IMAGE(1:7,:,:) = IMAGE(1:7,:,:) .* repmat(flip(taper)',1,d2,d3) + ... - repmat(flip(extrapVal.*(1-taper')),1,d2,d3); + repmat(flip(extrapVal.*(1-taper')),1,d2,d3); IMAGE(:,:,1:7) = IMAGE(:,:,1:7) .* repmat(permute(flip(taper),[3,1,2]),d1,d2,1) + ... - repmat(permute(flip(extrapVal.*(1-taper)),[3,1,2]),d1,d2,1); - + repmat(permute(flip(extrapVal.*(1-taper)),[3,1,2]),d1,d2,1); + IMAGE(:,end-6:end,:) = IMAGE(:,end-6:end,:) .* repmat(taper,d1,1,d3) + ... - repmat(extrapVal.*(1-taper),d1,1,d3); + repmat(extrapVal.*(1-taper),d1,1,d3); IMAGE(end-6:end,:,:) = IMAGE(end-6:end,:,:) .* repmat(taper',1,d2,d3) + ... - repmat(extrapVal.*(1-taper'),1,d2,d3); + repmat(extrapVal.*(1-taper'),1,d2,d3); IMAGE(:,:,end-6:end) = IMAGE(:,:,end-6:end) .* repmat(permute(taper,[3,1,2]),d1,d2,1) + ... - repmat(permute(extrapVal.*(1-taper),[3,1,2]),d1,d2,1); + repmat(permute(extrapVal.*(1-taper),[3,1,2]),d1,d2,1); end if ( fourierOverSample ) @@ -220,17 +220,17 @@ PADDED_IMG(1:sX1,1:sY1,end-sZ2:end) = IMAGE(1:sX1,1:sY1,end-sZ2:end); PADDED_IMG(end-sX2:end,1:sY1,end-sZ2:end) = IMAGE(end-sX2:end,1:sY1,end-sZ2:end); PADDED_IMG(1:sX1,end-sY2:end,end-sZ2:end) = IMAGE(1:sX1,end-sY2:end,end-sZ2:end); - PADDED_IMG(end-sX2:end,end-sY2:end,end-sZ2:end) = IMAGE(end-sX2:end,end-sY2:end,end-sZ2:end); + PADDED_IMG(end-sX2:end,end-sY2:end,end-sZ2:end) = IMAGE(end-sX2:end,end-sY2:end,end-sZ2:end); else - PADDED_IMG(padLOW(1)+1: end - padTOP(1), ... - padLOW(2)+1: end - padTOP(2), ... - padLOW(3)+1: end - padTOP(3)) = IMAGE; + PADDED_IMG(padLOW(1)+1: end - padTOP(1), ... + padLOW(2)+1: end - padTOP(2), ... + padLOW(3)+1: end - padTOP(3)) = IMAGE; end - + end -clear IMAGE +clear IMAGE end % end of the padZeros3d functions diff --git a/masking/BH_weightMask3d.m b/masking/BH_weightMask3d.m index af8a9757..71596467 100755 --- a/masking/BH_weightMask3d.m +++ b/masking/BH_weightMask3d.m @@ -1,6 +1,6 @@ function [ WEDGE_MASK, padValue] = BH_weightMask3d(SIZE, ORIENTATION, METHOD, ... - particleRadius, flgIncCtf, ... - SYMMETRY, samplingRate) + particleRadius, flgIncCtf, ... + SYMMETRY, samplingRate) %Create a missing wedge mask. % % Input variables: @@ -25,11 +25,11 @@ % % Goals & Limitations: % -% As a test case, use an asymmetric wedge -50,70 in order to visualize +% As a test case, use an asymmetric wedge -50,70 in order to visualize % any ambiguities in angles. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % -% TODO: +% TODO: % - test gpu option for function and return value. (template search requires % return of gpu) % @@ -41,15 +41,15 @@ if isvector(ORIENTATION) flgCTF = 0; - tiltColumn=1; - tiltAzimuth=90; - + tiltColumn=1; + tiltAzimuth=90; + else - % Consider amplitude modulation by CTF (NOT CTF envelope though) + % Consider amplitude modulation by CTF (NOT CTF envelope though) flgCTF = 1; tiltColumn=4; tiltAzimuth=ORIENTATION(1,6); - + end flgTiltWeight = 1; @@ -68,7 +68,7 @@ flgCTF = 0; end -if strcmp(METHOD, 'GPU') +if strcmp(METHOD, 'GPU') useGPU = true; else useGPU = false; @@ -77,9 +77,9 @@ %SIZE = [512,512,512]; if all(SIZE > 0) outputScaling = SIZE(1)/512; - SIZE = [512,512,512] -% outputScaling = SIZE(1)/256; -% SIZE = [256,256,256] + SIZE = [512,512,512] + % outputScaling = SIZE(1)/256; + % SIZE = [256,256,256] else %optional override for template matching which may not always be cubic, wich %trades a little accuracy in the ctf mask for speed. @@ -105,7 +105,7 @@ error('binary wedge uncertain if cpu or gpu.') end -% Move binary wedge to a separate function + % Move binary wedge to a separate function maxTilt = 90-abs(min(ORIENTATION(:,tiltColumn))); minTilt = 90-abs(max(ORIENTATION(:,tiltColumn))); tiltAxis= BH_defineMatrix(1.*[90-tiltAzimuth,0,0], 'Bah', 'forwardVector'); @@ -113,12 +113,12 @@ theta = atan2d(bX,bZ); clear bX bZ WEDGE_MASK = single(~( (180-minTilt<=theta | theta<=maxTilt-180) | ... - (-1.*minTilt<=theta & theta<=maxTilt) )); + (-1.*minTilt<=theta & theta<=maxTilt) )); % Set the value at the origin = 0.2 origMask = ceil((size(WEDGE_MASK)+1)./2); WEDGE_MASK(origMask(1)-2:origMask(1)+2,:,origMask(3)-2:origMask(3)+2) = 1; - + clear theta maxTilt minTilt tiltAxis @@ -130,10 +130,10 @@ WEDGE_MASK = convn(WEDGE_MASK, gaussKernel, 'same'); -% -% WEDGE_MASK = convn(WEDGE_MASK, gaussKernel, 'same'); + % + % WEDGE_MASK = convn(WEDGE_MASK, gaussKernel, 'same'); -% [ WEDGE_MASK ] = BH_multi_randomizeTaper(WEDGE_MASK); + % [ WEDGE_MASK ] = BH_multi_randomizeTaper(WEDGE_MASK); rad = fftshift(BH_bandpass3d(size(WEDGE_MASK),0,0,0,METHOD,'nyquist')); WEDGE_MASK = WEDGE_MASK .* rad; @@ -143,7 +143,7 @@ return else filterMask = false; - + end @@ -170,7 +170,7 @@ % % % zeros(paddedSize(2),1), ... % % % window(@hamming,2.*zoneOfInfluence+1)); -% Use the shape transform to estimate the extent of the projections influence +% Use the shape transform to estimate the extent of the projections influence % Even though we end up multiplying, because the projection is only a single % pixel thick to start with, this is equivalent to convolution, st this is % essentially accounting for creating a finite backprojection volume by @@ -185,9 +185,9 @@ BLOB = fftshift(BLOB ./ max(BLOB(:))); oB = ceil((SIZE+1)./2); blobKernel = BLOB(oB(1)-6:oB(1)+6,... - oB(2)-6:oB(2)+6,... - oB(3)-6:oB(3)+6); - + oB(2)-6:oB(2)+6,... + oB(3)-6:oB(3)+6); + BLOB = squeeze(gather(BLOB(oB(1), oB(2), :)))'; zWeight = repmat(BLOB, SIZE(2),1,SIZE(1)); @@ -201,77 +201,77 @@ [ rWeight ] = calc_rWeight( SIZE, 'single', METHOD); -% % % rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); -% % % rOrig = ceil((SIZE(1)+1)./2); -% % % % % % % imod tilt zero freq = 0.2 * first non zero component -% % % rWeight(rOrig ) = 1; -% % % -% % % -% % % % rWeight = rWeight + 1; -% % % -% % % -% % % % rWeight = (rWeight ./ max(rWeight)).^0.5; -% % % rWeight = rWeight + 1./rWeight.^2; -% % % % resample2d only handles scaling right now, so pad to z=3 -% % % rWeight = repmat((rWeight), 1, SIZE(2),3); -% % % -% % % rWeight = BH_resample3d( rWeight, [90-tiltAzimuth,0,0], ... -% % % [0,0,0],'Bah','GPU','forwardVector'); -% % % rWeight = rWeight(:,:,2); - + % % % rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); + % % % rOrig = ceil((SIZE(1)+1)./2); + % % % % % % % imod tilt zero freq = 0.2 * first non zero component + % % % rWeight(rOrig ) = 1; + % % % + % % % + % % % % rWeight = rWeight + 1; + % % % + % % % + % % % % rWeight = (rWeight ./ max(rWeight)).^0.5; + % % % rWeight = rWeight + 1./rWeight.^2; + % % % % resample2d only handles scaling right now, so pad to z=3 + % % % rWeight = repmat((rWeight), 1, SIZE(2),3); + % % % + % % % rWeight = BH_resample3d( rWeight, [90-tiltAzimuth,0,0], ... + % % % [0,0,0],'Bah','GPU','forwardVector'); + % % % rWeight = rWeight(:,:,2); + else - rWeight = 1; + rWeight = 1; end [mtf,~,~,~,~,~] = BH_multi_gridCoordinates(SIZE(1:2), ... - 'Cartesian','cpu',... - {'none'},1,1,1); - + 'Cartesian','cpu',... + {'none'},1,1,1); + if (useGPU) [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(SIZE(1:2), ... - 'Cartesian','GPU',... - {'none'},1,0,1); + 'Cartesian','GPU',... + {'none'},1,0,1); else [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(SIZE(1:2), ... - 'Cartesian','cpu',... - {'none'},1,0,1); - -end + 'Cartesian','cpu',... + {'none'},1,0,1); + +end + + +pixelSize = ORIENTATION(1,16).*ORIENTATION(1,14).*samplingRate; % scaled pixel size +radialGrid = radialGrid./(pixelSize.*10.^10); - pixelSize = ORIENTATION(1,16).*ORIENTATION(1,14).*samplingRate; % scaled pixel size - - radialGrid = radialGrid./(pixelSize.*10.^10); - -% exposureFilter = ones([SIZE(1:2),size(ORIENTATION,1)],'single'); -if (flgCTF) +% exposureFilter = ones([SIZE(1:2),size(ORIENTATION,1)],'single'); +if (flgCTF) if (flgNoExposure) exposureFilter = zeros(1,1,size(ORIENTATION,1)) + 1; else - [ exposureFilter ] = BH_exposureFilter( SIZE(1:2), ORIENTATION, 'cpu',samplingRate,0 ); + [ exposureFilter ] = BH_exposureFilter( SIZE(1:2), ORIENTATION, 'cpu',samplingRate,0 ); end % Calc the downweighting due to CTF --> here assuming phases were flipped by % multiplying by the CTF past the first zero iPrj = 1; defocus = [ORIENTATION(iPrj,15) - ORIENTATION(iPrj,12), ... - ORIENTATION(iPrj,15) + ORIENTATION(iPrj,12), ... - ORIENTATION(iPrj,13)]; + ORIENTATION(iPrj,15) + ORIENTATION(iPrj,12), ... + ORIENTATION(iPrj,13)]; Cs = ORIENTATION(iPrj,17); wavelength = ORIENTATION(iPrj,18); ampContrast = ORIENTATION(iPrj,19); - - % assuming ampContrast = 0.1, using -0.15 results in a weight with + + % assuming ampContrast = 0.1, using -0.15 results in a weight with % (0.1^0.15)^2~ 0.5 at zero freqency. Allows some recovery of low freq without % creating too severe a blur - [Hqz, HqzUnMod] = BH_ctfCalc(pixelSize,Cs,wavelength,defocus,SIZE(1:2),ampContrast,-1,-1); - + [Hqz, HqzUnMod] = BH_ctfCalc(pixelSize,Cs,wavelength,defocus,SIZE(1:2),ampContrast,-1,-1); + Hqz = single(abs(Hqz.*HqzUnMod)); -% Hqz = conv2(Hqz,fspecial('gaussian',[5,5],1.0),'same'); -% SAVE_IMG(MRCImage(gather(single(Hqz))), 'tmp.mrc'); + % Hqz = conv2(Hqz,fspecial('gaussian',[5,5],1.0),'same'); + % SAVE_IMG(MRCImage(gather(single(Hqz))), 'tmp.mrc'); else Hqz = zeros(size(radialGrid),'single')+1; @@ -297,7 +297,7 @@ % centralSection(:,:,ceil((SIZE(3)+1)/2) - zoneOfInfluence: ... % ceil((SIZE(3)+1)/2) + zoneOfInfluence) = Z; -% % % centralSection = centralSection .* zWeight; +% % % centralSection = centralSection .* zWeight; % % % clear zWeight @@ -309,42 +309,42 @@ for iPrj = 1:nPrjs - + %exposureFilter = exp(-1.*ORIENTATION(iPrj,11).*(fftshift(radialGrid)).^2); -% Applying a re-weighting to the ctfTiles - remove this -% Optimal exposure filter from Grant,Grigorieff 2015 eLife -% Assuming either 200 or 300 KV -% % % WAVELENGTH = ORIENTATION(iPrj,18); -% % % if WAVELENGTH > 2.1*10^-12 -% % % kvScale = 0.8; -% % % else -% % % kvScale = 1.0; -% % % end -% % % CUMeDOSE = ORIENTATION(iPrj,11); -% % % exposureFilter = exp(-0.5*CUMeDOSE .* (kvScale.*0.245.*fftshift(radialGrid) .^ -1.665 + 2.81).^-1); - -%expF = 1; -if (flgCTF) - expF = exposureFilter(:,:,ORIENTATION(iPrj,1)); -else - expF = 1; -end -%expF = (expF./sqrt((sum(sum(sum(abs(expF).^2))))./numel(expF))); + % Applying a re-weighting to the ctfTiles - remove this + % Optimal exposure filter from Grant,Grigorieff 2015 eLife + % Assuming either 200 or 300 KV + % % % WAVELENGTH = ORIENTATION(iPrj,18); + % % % if WAVELENGTH > 2.1*10^-12 + % % % kvScale = 0.8; + % % % else + % % % kvScale = 1.0; + % % % end + % % % CUMeDOSE = ORIENTATION(iPrj,11); + % % % exposureFilter = exp(-0.5*CUMeDOSE .* (kvScale.*0.245.*fftshift(radialGrid) .^ -1.665 + 2.81).^-1); + + %expF = 1; + if (flgCTF) + expF = exposureFilter(:,:,ORIENTATION(iPrj,1)); + else + expF = 1; + end + %expF = (expF./sqrt((sum(sum(sum(abs(expF).^2))))./numel(expF))); if (flgRadial) centralSection = repmat(rWeight.* fftshift(expF.* ... - Hqz ),1,1, SIZE(3)) .* ... - zWeight; -% centralSection = repmat(rWeight.* ... -% Hqz .* mtf,1,1, SIZE(3)) .* ... -% zWeight; + Hqz ),1,1, SIZE(3)) .* ... + zWeight; + % centralSection = repmat(rWeight.* ... + % Hqz .* mtf,1,1, SIZE(3)) .* ... + % zWeight; else centralSection = repmat(fftshift(exposureFilter(:,:,ORIENTATION(iPrj,1)).*Hqz ),1,1, SIZE(3)) .* ... - zWeight; - end + zWeight; + end + + - - symInc = 360/SYMMETRY; for iSym = 1:SYMMETRY % Since the angles here are used bring the rotated projection back to standard @@ -353,44 +353,44 @@ if any(size(ORIENTATION) == 1) R = BH_defineMatrix([90, ORIENTATION(iPrj), -90], 'Bah', 'inv'); else -% rSample = BH_defineMatrix((ORIENTATION(iPrj,8:10)), 'Bah', 'inv'); -% rElevation = BH_defineMatrix([90, ORIENTATION(iPrj, 7), -90], 'Bah', 'inv'); - rTilt = BH_defineMatrix([1.*ORIENTATION(iPrj,6),1.*ORIENTATION(iPrj,4),-1*ORIENTATION(iPrj,6)],'Bah','inv'); + % rSample = BH_defineMatrix((ORIENTATION(iPrj,8:10)), 'Bah', 'inv'); + % rElevation = BH_defineMatrix([90, ORIENTATION(iPrj, 7), -90], 'Bah', 'inv'); + rTilt = BH_defineMatrix([1.*ORIENTATION(iPrj,6),1.*ORIENTATION(iPrj,4),-1*ORIENTATION(iPrj,6)],'Bah','inv'); % Switch to imod means that the inPlane rotation has been applied to the % projections already %%%rInPlane = BH_defineMatrix(1.*[ORIENTATION(iPrj,5),0,0], 'Bah', 'inv'); -% rInPlane = eye(3); - - -% R = rElevation*rTilt*rSample*rInPlane; - R = rTilt; + % rInPlane = eye(3); + + + % R = rElevation*rTilt*rSample*rInPlane; + R = rTilt; end R = R*BH_defineMatrix([(1-iSym)*symInc,0,0],'Bah','forward'); - % - if (flgTiltWeight) -% tiltDiff = 1-cosd(ORIENTATION(iPrj,tiltColumn)).^1; -% tiltGrad = 1-(tiltDiff.*exp(-15.*fftshift(radialGrid).^2)); -% tiltWeight = repmat(tiltGrad,1,1,size(centralSection,3)); - iAng = ORIENTATION(iPrj,tiltColumn); - tiltWeight = ((exp(-10.*mtf.^(0.5+cosd(iAng).^2.5))+(0.6))./(1.6)).^sind(abs(iAng)); - tiltWeight = repmat(tiltWeight,1,1,size(centralSection,3)); - - % tiltWeight = cosd(ORIENTATION(iPrj,tiltColumn)); - else - tiltWeight = 1; - end - - - - wedgeMask = wedgeMask + ... - BH_resample3d(centralSection.*tiltWeight, R, ... - [0,0,0], 'Bah','GPU','inv'); - - - -% % % BH_resample3d(centralSection.*tiltWeight, R, [0,0,0], 'Bah','GPU','inv'); + % + if (flgTiltWeight) + % tiltDiff = 1-cosd(ORIENTATION(iPrj,tiltColumn)).^1; + % tiltGrad = 1-(tiltDiff.*exp(-15.*fftshift(radialGrid).^2)); + % tiltWeight = repmat(tiltGrad,1,1,size(centralSection,3)); + iAng = ORIENTATION(iPrj,tiltColumn); + tiltWeight = ((exp(-10.*mtf.^(0.5+cosd(iAng).^2.5))+(0.6))./(1.6)).^sind(abs(iAng)); + tiltWeight = repmat(tiltWeight,1,1,size(centralSection,3)); + + % tiltWeight = cosd(ORIENTATION(iPrj,tiltColumn)); + else + tiltWeight = 1; + end + + + + wedgeMask = wedgeMask + ... + BH_resample3d(centralSection.*tiltWeight, R, ... + [0,0,0], 'Bah','GPU','inv'); + + + + % % % BH_resample3d(centralSection.*tiltWeight, R, [0,0,0], 'Bah','GPU','inv'); end end @@ -400,9 +400,9 @@ if (flgSmooth) [ gaussKernel ] = gpuArray(BH_multi_gaussian3d(5, 0.75 )); - + wedgeMask = convn(wedgeMask, gaussKernel, 'same'); -% % % % % wedgeMask = convn(wedgeMask, gaussKernel, 'same'); + % % % % % wedgeMask = convn(wedgeMask, gaussKernel, 'same'); end @@ -410,22 +410,22 @@ % wedgeMask = wedgeMask(padValue(1)+1 : end - padValue(1), ... % padValue(2)+1 : end - padValue(2), ... % padValue(3)+1 : end - padValue(3)); - + % wedgeMask = single(wedgeMask ./ max(wedgeMask(:))); % wedgeMask = convn(wedgeMask, gaussKernel, 'same'); -% +% % wedgeMask = wedgeMask ./ max(wedgeMask(:)); % [rad,~,~,~,~,~] = BH_multi_gridCoordinates(SIZE,'Cylindrical', ... % 'GPU',{BH_defineMatrix([0,90,90-tiltAzimuth],'Bah','forwardVector'),[0,0,0]','forward'},1,1,0); -% % % +% % % % %rad = (rad < 0.5); % wedgeMask = wedgeMask .* rad; - - WEDGE_MASK = BH_reScale3d( gather(single(wedgeMask ./ max(wedgeMask(:)))), ... - '', sprintf('%f',outputScaling), METHOD); + +WEDGE_MASK = BH_reScale3d( gather(single(wedgeMask ./ max(wedgeMask(:)))), ... + '', sprintf('%f',outputScaling), METHOD); rad = fftshift(BH_bandpass3d(size(WEDGE_MASK),0,0,0,'GPU','nyquist')); WEDGE_MASK = WEDGE_MASK .* rad; @@ -437,7 +437,7 @@ % BINARY_WEDGE = BINARY_WEDGE .* (rad <0.5) ; % BINARY_WEDGE = convn(single(BINARY_WEDGE), BH_multi_gaussian3d(5, 1.0 ),'same'); % % % BINARY_WEDGE = BINARY_WEDGE ./ max(BINARY_WEDGE(:)) .*rad; -% clear wedgeMask centralSection radialGrid rad zWeight +% clear wedgeMask centralSection radialGrid rad zWeight clearvars -except WEDGE_MASK padValues @@ -446,30 +446,30 @@ function [ rWeight ] = calc_rWeight( SIZE, PRECISION, METHOD) - rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); - if strcmp(METHOD,'GPU') - rWeight = gpuArray(rWeight); - end - rOrig = ceil((SIZE(1)+1)./2); - % % % % imod tilt zero freq = 0.2 * first non zero component - rWeight(rOrig ) = 0.2; - [rCut] = find(rWeight == floor(0.45*SIZE(1))); - pixelFallOff = rCut(1) ; - taperLow = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); - - pixelFallOff = SIZE(1)-rCut(2)+1 ; - taperTop = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); - rWeight(1:rCut(1)) = rWeight(1:rCut(1)).*flip(taperLow)'; - rWeight(rCut(2):end) = rWeight(rCut(2):end).*taperTop'; - - - %rWeight = rWeight + 1./rWeight.^2; - % resample2d only handles scaling right now, so pad to z=3 - rWeight = repmat((rWeight), 1, SIZE(2),1); - if strcmpi(PRECISION,'single') - rWeight = single(rWeight); - else - % This should be the default. - rWeight = double(rWeight); - end +rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); +if strcmp(METHOD,'GPU') + rWeight = gpuArray(rWeight); +end +rOrig = ceil((SIZE(1)+1)./2); +% % % % imod tilt zero freq = 0.2 * first non zero component +rWeight(rOrig ) = 0.2; +[rCut] = find(rWeight == floor(0.45*SIZE(1))); +pixelFallOff = rCut(1) ; +taperLow = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); + +pixelFallOff = SIZE(1)-rCut(2)+1 ; +taperTop = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); +rWeight(1:rCut(1)) = rWeight(1:rCut(1)).*flip(taperLow)'; +rWeight(rCut(2):end) = rWeight(rCut(2):end).*taperTop'; + + +%rWeight = rWeight + 1./rWeight.^2; +% resample2d only handles scaling right now, so pad to z=3 +rWeight = repmat((rWeight), 1, SIZE(2),1); +if strcmpi(PRECISION,'single') + rWeight = single(rWeight); +else + % This should be the default. + rWeight = double(rWeight); +end end diff --git a/masking/BH_weightMaskMex.m b/masking/BH_weightMaskMex.m index 87df8d17..5a327a0b 100644 --- a/masking/BH_weightMaskMex.m +++ b/masking/BH_weightMaskMex.m @@ -1,5 +1,5 @@ function [ SF3D ] = BH_weightMaskMex(SIZE, SAMPLING, TLT, ... - xyzSubTomo,reconGeometry, wiener_constant) + xyzSubTomo,reconGeometry, wiener_constant) %UNTITLED Summary of this function goes here % Detailed explanation goes here @@ -18,7 +18,7 @@ originVol = ceil((reconGeometry(1,1:3)+1)./2); prjVector = xyzSubTomo - originVol + reconShift; - + iCs = single(TLT(:,17).*10^3); iWavelength = single(TLT(:,18).*10^10); iPhaseShift = TLT(:,19) ; @@ -28,14 +28,14 @@ nTilts = uint32(size(TLT,1)); exposure = gather(single(TLT(:,11))); - + % Need a defocus offset based on XYZ position in the tomogram for iPrj = 1:nTilts rTilt = BH_defineMatrix(TLT(iPrj,4),'TILT','forwardVector') ; prjCoords = rTilt * prjVector'; iDefocus(iPrj) = iDefocus(iPrj)-(prjCoords(3).*pixelSize(iPrj)); end - + iDefocus = gather(single(iDefocus)); % TODO figure out how to get this from the data iThickness = 75; % nm @@ -44,12 +44,12 @@ fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics(:)); - [SF3D] = mexSF3D(doHalfMask,doSqCTF,SIZE,pixelSize,iWavelength,iCs, ... - gather(single(iDefocus + iddF)), ... - gather(single(iDefocus - iddF)), ... - idPHI,iPhaseShift,nTilts,tiltAngles, ... - exposure,fractionOfElastics.*fractionOfDose,int16(1), ... - gather(single(wiener_constant))); +[SF3D] = mexSF3D(doHalfMask,doSqCTF,SIZE,pixelSize,iWavelength,iCs, ... + gather(single(iDefocus + iddF)), ... + gather(single(iDefocus - iddF)), ... + idPHI,iPhaseShift,nTilts,tiltAngles, ... + exposure,fractionOfElastics.*fractionOfDose,int16(1), ... + gather(single(wiener_constant))); % SF3D = SF3D ./ (WGT+0.01); diff --git a/masking/BH_weightMask_dp.m b/masking/BH_weightMask_dp.m index 4931726d..c1287281 100755 --- a/masking/BH_weightMask_dp.m +++ b/masking/BH_weightMask_dp.m @@ -1,6 +1,6 @@ function [ ctfWeights ] = BH_weightMask_dp(subTomoMeta, SIZE, SAMPLING,... - GEOMETRY, PRECISION, METHOD,... - varargin) + GEOMETRY, PRECISION, METHOD,... + varargin) % Calculate the sampling of a number of subtomograms. % % If tiltGeometry, reconGeometry, and sTgeometry are structs, then calculate the @@ -52,9 +52,9 @@ % % % [radialCTFCalc, phiCTFCalc, ~, ~, ~, ~ ] = ... % % % BH_multi_gridCoordinates(SIZE(1:2),'Cylindrical', ... -% % % METHOD, {'none'},1,1,0); -% % % -% % % +% % % METHOD, {'none'},1,1,0); +% % % +% % % % % % % This should be the default. % % % radialCTFCalc = {radialCTFCalc ./ pixelSize,1,phiCTFCalc}; @@ -78,69 +78,69 @@ error('METHOD must be GPU or %s\n', 'cpu'); end for iTomo = 1:1+(nTomos-1)*(calcAllWeights) - - -% % % reconGeometry = subTomoMeta.('reconGeometry').(tomoList{iTomo})./SAMPLING; + + + % % % reconGeometry = subTomoMeta.('reconGeometry').(tomoList{iTomo})./SAMPLING; reconGeometry = subTomoMeta.('reconGeometry').(tomoList{iTomo}); - + reconShift = reconGeometry(2,:); % Also there are no offsets here, but this should be considered as in % syntheticMapback - also need to update things to save the tilt header. % Possibly just calculate the respective origins and lowerLeft vol as part of % the initialization. - -% % originPrj = subTomoMeta.('tiltGeometry').(tomoList{iTomo})(1,20:22)./SAMPLING; -% % % originPrj = ceil((originPrj+1)./2); + + % % originPrj = subTomoMeta.('tiltGeometry').(tomoList{iTomo})(1,20:22)./SAMPLING; + % % % originPrj = ceil((originPrj+1)./2); originPrj = subTomoMeta.('tiltGeometry').(tomoList{iTomo})(1,20:22); - -% % % originPrj(3) = 1; + + % % % originPrj(3) = 1; originVol = ceil((reconGeometry(1,1:3)+1)./2); lowerLeftVol = originPrj+reconShift-originVol; - + positionList = geometryFull.(tomoList{iTomo}); positionList = positionList(positionList(:,26)~=-9999,:); TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); nPrjs = size(TLT,1); - nSubTomos = size(positionList,1) ; - + nSubTomos = size(positionList,1) ; + zCoords = sort(positionList(:,13)); - + % The thickness in the tomogram is NOT the thickness in the projection % if the specimen is tilted at zero-tilt angle. Need to think of a % better way to calculate this. -% % if (length(zCoords) < 5) -% % iThickness = 100 -% % else -% % iThickness = iqr(zCoords).*pixelSize*10^9 -% % % meanMax = mean(zCoords(1:5)).*pixelSize*10^9; -% % % meanMin = mean(zCoords(end-5:end)).*pixelSize*10^9; -% % end -% % -% % if iThickness > 400 -% % fprintf('capping thickness to 400\n'); -% % iThickness = 400; -% % end + % % if (length(zCoords) < 5) + % % iThickness = 100 + % % else + % % iThickness = iqr(zCoords).*pixelSize*10^9 + % % % meanMax = mean(zCoords(1:5)).*pixelSize*10^9; + % % % meanMin = mean(zCoords(end-5:end)).*pixelSize*10^9; + % % end + % % + % % if iThickness > 400 + % % fprintf('capping thickness to 400\n'); + % % iThickness = 400; + % % end iThickness = 75; - - -% % % [ exposureFilter ] = BH_exposureFilter( SIZE(1:2), TLT, METHOD, SAMPLING, 1 ); - + + + % % % [ exposureFilter ] = BH_exposureFilter( SIZE(1:2), TLT, METHOD, SAMPLING, 1 ); + if strcmp(PRECISION,'double') exposureFilter = double(exposureFilter); end for iSubTomo = 1:1+(nSubTomos-1)*(calcAllWeights) fprintf('%d/%d tomo %d/%d subtomo\n',iTomo,nTomos,iSubTomo,nSubTomos); - % xyzSubTomo = (positionList(iSubTomo,11:13)./1 + lowerLeftVol); - % prjVector = (xyzSubTomo- originPrj); + % xyzSubTomo = (positionList(iSubTomo,11:13)./1 + lowerLeftVol); + % prjVector = (xyzSubTomo- originPrj); if (calcAllWeights) - xyzSubTomo = positionList(iSubTomo,11:13)./SAMPLING; + xyzSubTomo = positionList(iSubTomo,11:13)./SAMPLING; prjVector = xyzSubTomo - originVol + reconShift; else ctfGroupSize = subTomoMeta.('ctfGroupSize').(tomoList{iTomo})(2); @@ -149,115 +149,115 @@ prjVector = xyzSubTomo - originVol + reconShift; prjVector(2:3) = 0; end - - - if ( calcAllWeights ) + + + if ( calcAllWeights ) iGold = positionList(iSubTomo,7); else iGold = 1; end -% % % for iPrj = 1:nPrjs -% % % -% % % -% % % iCs = TLT(iPrj,17); -% % % iWavelength = TLT(iPrj,18); -% % % iPhaseShift = TLT(iPrj,19); -% % % iDefocus = TLT(iPrj,15); -% % % iddF = TLT(iPrj,12); -% % % idPHI = TLT(iPrj,13); -% % % -% % % data = rWeight; -% % % -% % % % The central section is located at -1.*tiltAngle in the 3dFT since we -% % % % rotate the specimen back by this amount in reconstruction. -% % % rTilt = BH_defineMatrix([90,TLT(iPrj,4),-90],'Bah','invVector'); - if (calcAllWeights) - rSubTomo = reshape(positionList(iSubTomo,17:25),3,3); - else - rSubTomo = eye(3); - end -% % % % To calculate the defocus, we need to rotate to where the projection is -% % % % in real space TLT(iPrj,4) -% % % -% % % rProjection = BH_defineMatrix([90,TLT(iPrj,4),-90],'Bah','forwardVector'); -% % % prjCoords = rProjection*prjVector'; -% % % -% % % iDf = prjCoords(3).*pixelSize + iDefocus; -% % % defVect = [iDf - iddF, iDf + iddF, idPHI]; -% % % % Note the transpose (=inverse since the rotation matrix is orthogonal) is -% % % % taken because the stored matrix is for interpolation -% % % r = rSubTomo'*rTilt; -% % % -% % % % By default these are single PRECISION, but we truncate them anyhow, so -% % % % leave as single. -% % % [X,Y,Z,~,~,~] = BH_multi_gridCoordinates([SIZE(1:2),1],'Cartesian',METHOD,... -% % % {'single',r,[0,0,0]','invVector',1,1},0,1,0); -% % % -% % % -% % % % assuming ampContrast = 0.1, using -0.15 results in a weight with -% % % % (0.1^0.15)^2~ 0.5 at zero freqency. Allows some recovery of low freq without -% % % % creating too severe a blur -% % % [Hqz, ~] = BH_ctfCalc(radialCTFCalc,iCs,iWavelength,defVect,SIZE(1:2),iPhaseShift,-1.0); -% % % -% % % -% % % -% % % % %Default is double -% % % % if strcmpi(PRECISION,'single') -% % % % Hqz = single(abs(Hqz.*HqzUnMod)).^ctfScaleFactor; -% % % % elseif strcmpi(PRECISION, 'double') -% % % % Hqz = double(abs(Hqz.*HqzUnMod)).^ctfScaleFactor; -% % % % end -% % % -% % % %Default is double -% % % if strcmpi(PRECISION,'single') -% % % Hqz = single(abs(Hqz).^2); -% % % elseif strcmpi(PRECISION, 'double') -% % % Hqz = double(abs(Hqz).^2); -% % % end -% % % if SAMPLING > 1 -% % % Hqz = Hqz .^ (1*(SAMPLING - 1)^-3); -% % % end - - -% % % fractionOfDose = TLT(iPrj,14)/mean(TLT(:,14)); -% % % fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(iPrj,4))*400 )); -% -% fprintf('fractionOfDose %2.2f fractionOfElastic %2.2f at angle %2.2f\n',... -% fractionOfDose, fractionOfElastics,TLT(iPrj,4)); - - % TODO does this make sense to run as power of one or two? - exposureFilterPower = 1; -% % % data = data.*Hqz.*exposureFilter(:,:,TLT(iPrj,1)).^exposureFilterPower.*(fractionOfDose*fractionOfElastics); - - -% % % data = data(:); -% % % -% % % % Shift from image to array coordinates and set any out of bounds values to -% % % % the origin. -% % % originXYZ = ceil(((SIZE + 1)./2)); -% % % X = X(:)+originXYZ(1); -% % % Y = Y(:)+originXYZ(2); -% % % Z = Z(:)+originXYZ(3); -% % % -% % % -% % % outOfBounds = logical(( X < 1 | X > SIZE(1) ) + (Y < 1 | Y > SIZE(2)) + (Z < 1 | Z > SIZE(3))); -% % % X(outOfBounds) = 1;%originXYZ(1); -% % % Y(outOfBounds) = 1;%originXYZ(2); -% % % Z(outOfBounds) = 1;%originXYZ(3); -% % % -% % % -% % % -% % % l = sub2ind(SIZE,round(X),round(Y),round(Z)); -% % % clear X Y Z -% % % % much faster and effecitive for this simple sort. -% % % [B,I] = sort(l); -% % % idu = I(logical(B(1:end-1) - B(2:end))); -% % % -% % % tiltScale = 1;%- ( abs(sind(TLT(iPrj,4))).*0.2 ); -% % % rec{iGold}(l(idu)) = rec{iGold}(l(idu)) + tiltScale.*data(idu); -% % % clear l idu -% % % -% % % end % end loop over projections + % % % for iPrj = 1:nPrjs + % % % + % % % + % % % iCs = TLT(iPrj,17); + % % % iWavelength = TLT(iPrj,18); + % % % iPhaseShift = TLT(iPrj,19); + % % % iDefocus = TLT(iPrj,15); + % % % iddF = TLT(iPrj,12); + % % % idPHI = TLT(iPrj,13); + % % % + % % % data = rWeight; + % % % + % % % % The central section is located at -1.*tiltAngle in the 3dFT since we + % % % % rotate the specimen back by this amount in reconstruction. + % % % rTilt = BH_defineMatrix([90,TLT(iPrj,4),-90],'Bah','invVector'); + if (calcAllWeights) + rSubTomo = reshape(positionList(iSubTomo,17:25),3,3); + else + rSubTomo = eye(3); + end + % % % % To calculate the defocus, we need to rotate to where the projection is + % % % % in real space TLT(iPrj,4) + % % % + % % % rProjection = BH_defineMatrix([90,TLT(iPrj,4),-90],'Bah','forwardVector'); + % % % prjCoords = rProjection*prjVector'; + % % % + % % % iDf = prjCoords(3).*pixelSize + iDefocus; + % % % defVect = [iDf - iddF, iDf + iddF, idPHI]; + % % % % Note the transpose (=inverse since the rotation matrix is orthogonal) is + % % % % taken because the stored matrix is for interpolation + % % % r = rSubTomo'*rTilt; + % % % + % % % % By default these are single PRECISION, but we truncate them anyhow, so + % % % % leave as single. + % % % [X,Y,Z,~,~,~] = BH_multi_gridCoordinates([SIZE(1:2),1],'Cartesian',METHOD,... + % % % {'single',r,[0,0,0]','invVector',1,1},0,1,0); + % % % + % % % + % % % % assuming ampContrast = 0.1, using -0.15 results in a weight with + % % % % (0.1^0.15)^2~ 0.5 at zero freqency. Allows some recovery of low freq without + % % % % creating too severe a blur + % % % [Hqz, ~] = BH_ctfCalc(radialCTFCalc,iCs,iWavelength,defVect,SIZE(1:2),iPhaseShift,-1.0); + % % % + % % % + % % % + % % % % %Default is double + % % % % if strcmpi(PRECISION,'single') + % % % % Hqz = single(abs(Hqz.*HqzUnMod)).^ctfScaleFactor; + % % % % elseif strcmpi(PRECISION, 'double') + % % % % Hqz = double(abs(Hqz.*HqzUnMod)).^ctfScaleFactor; + % % % % end + % % % + % % % %Default is double + % % % if strcmpi(PRECISION,'single') + % % % Hqz = single(abs(Hqz).^2); + % % % elseif strcmpi(PRECISION, 'double') + % % % Hqz = double(abs(Hqz).^2); + % % % end + % % % if SAMPLING > 1 + % % % Hqz = Hqz .^ (1*(SAMPLING - 1)^-3); + % % % end + + + % % % fractionOfDose = TLT(iPrj,14)/mean(TLT(:,14)); + % % % fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(iPrj,4))*400 )); + % + % fprintf('fractionOfDose %2.2f fractionOfElastic %2.2f at angle %2.2f\n',... + % fractionOfDose, fractionOfElastics,TLT(iPrj,4)); + + % TODO does this make sense to run as power of one or two? + exposureFilterPower = 1; + % % % data = data.*Hqz.*exposureFilter(:,:,TLT(iPrj,1)).^exposureFilterPower.*(fractionOfDose*fractionOfElastics); + + + % % % data = data(:); + % % % + % % % % Shift from image to array coordinates and set any out of bounds values to + % % % % the origin. + % % % originXYZ = ceil(((SIZE + 1)./2)); + % % % X = X(:)+originXYZ(1); + % % % Y = Y(:)+originXYZ(2); + % % % Z = Z(:)+originXYZ(3); + % % % + % % % + % % % outOfBounds = logical(( X < 1 | X > SIZE(1) ) + (Y < 1 | Y > SIZE(2)) + (Z < 1 | Z > SIZE(3))); + % % % X(outOfBounds) = 1;%originXYZ(1); + % % % Y(outOfBounds) = 1;%originXYZ(2); + % % % Z(outOfBounds) = 1;%originXYZ(3); + % % % + % % % + % % % + % % % l = sub2ind(SIZE,round(X),round(Y),round(Z)); + % % % clear X Y Z + % % % % much faster and effecitive for this simple sort. + % % % [B,I] = sort(l); + % % % idu = I(logical(B(1:end-1) - B(2:end))); + % % % + % % % tiltScale = 1;%- ( abs(sind(TLT(iPrj,4))).*0.2 ); + % % % rec{iGold}(l(idu)) = rec{iGold}(l(idu)) + tiltScale.*data(idu); + % % % clear l idu + % % % + % % % end % end loop over projections [maxSize,maxCoord] = max(SIZE); if (2.*maxSize > 512) padScaling = maxSize ./ 512; @@ -266,157 +266,157 @@ paddedSize = ceil(2.*maxSize).*[1,1,1]; padScaling = 1/2; end - + [ SF3D ] = fourierCtfRecTex(paddedSize, [positionList(iSubTomo,1:10),... - xyzSubTomo.*SAMPLING,... - positionList(iSubTomo,14:16),... - rSubTomo(1:9),positionList(iSubTomo,26)],... - TLT, reconGeometry, originPrj, ... - SAMPLING); - - - -% % % -% % % % For now just recreate the Hermitian pair -% % % nX = paddedSize(1); -% % % nZ = paddedSize(3); -% % % oX = floor(nX/2)+1; -% % % isOdd = mod(nX,2); + xyzSubTomo.*SAMPLING,... + positionList(iSubTomo,14:16),... + rSubTomo(1:9),positionList(iSubTomo,26)],... + TLT, reconGeometry, originPrj, ... + SAMPLING); + + + + % % % + % % % % For now just recreate the Hermitian pair + % % % nX = paddedSize(1); + % % % nZ = paddedSize(3); + % % % oX = floor(nX/2)+1; + % % % isOdd = mod(nX,2); [ rec{iGold} ] = BH_multi_makeHermitian(SF3D, paddedSize, padScaling); - -% % % try -% % % tmpArray = zeros(paddedSize,'single','gpuArray'); -% % % tmpArray(oX-1+isOdd:end,:,:) = SF3D; -% % % tmpArray(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); -% % % clear SF3D -% % % -% % % rec{iGold} = BH_reScale3d(tmpArray,'',padScaling,'GPU'); -% % % clear tmpArray -% % % catch -% % % fprintf('SIZE %d paddedSize %d scaleFactor %d\n',SIZE(1),paddedSize(1),padScale); -% % % error('I broke in generating the Hermitian mates!'); -% % % end -% % % % rec{iGold}(oX-1+isOdd:end,:,:) = SF3D; -% % % % rec{iGold}(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); -% % % % clear SF3D + % % % try + % % % tmpArray = zeros(paddedSize,'single','gpuArray'); + % % % tmpArray(oX-1+isOdd:end,:,:) = SF3D; + % % % tmpArray(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); + % % % clear SF3D + % % % + % % % rec{iGold} = BH_reScale3d(tmpArray,'',padScaling,'GPU'); + % % % clear tmpArray + % % % catch + % % % fprintf('SIZE %d paddedSize %d scaleFactor %d\n',SIZE(1),paddedSize(1),padScale); + % % % error('I broke in generating the Hermitian mates!'); + % % % end + % % % % rec{iGold}(oX-1+isOdd:end,:,:) = SF3D; + % % % % rec{iGold}(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); + % % % % clear SF3D - -% % For now just recreate the Hermitian pair -% nX = size(rec{iGold} ,1); -% nZ = size(rec{iGold} ,3); -% oX = floor(nX/2)+1; -% isOdd = mod(nX,2); - -% rec{iGold}(oX-1+isOdd:end,:,:) = SF3D; -% rec{iGold}(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); -% clear SF3D - -% % % for iGold = 1:1 + calcAllWeights -% % % % Zero out the large value from out of bounds conditions -% % % rec{iGold}(1) = 0; -% % % end - + + % % For now just recreate the Hermitian pair + % nX = size(rec{iGold} ,1); + % nZ = size(rec{iGold} ,3); + % oX = floor(nX/2)+1; + % isOdd = mod(nX,2); + + % rec{iGold}(oX-1+isOdd:end,:,:) = SF3D; + % rec{iGold}(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); + % clear SF3D + + + % % % for iGold = 1:1 + calcAllWeights + % % % % Zero out the large value from out of bounds conditions + % % % rec{iGold}(1) = 0; + % % % end + end % end loop over subTomos end %end loop over Tomos - - - -% % % g = BH_multi_gaussian3d(16.*[1,1,1],1.25); -% % % gf=fftshift(fftn(ifftshift(g))); -% % % g=real(fftshift(ifftn(ifftshift(gf.^3)))); clear gf -% % % g = g ./ sum(g(:)); -% % % -% % % if (useGPU) -% % % g = gpuArray(g); -% % % end - - %SAVE_IMG(MRCImage(gather(rec{1})),'tmpR1.mrc'); - %SAVE_IMG(MRCImage(gather(rec{2})),'tmpR2.mrc'); - + + + % % % g = BH_multi_gaussian3d(16.*[1,1,1],1.25); + % % % gf=fftshift(fftn(ifftshift(g))); + % % % g=real(fftshift(ifftn(ifftshift(gf.^3)))); clear gf + % % % g = g ./ sum(g(:)); + % % % + % % % if (useGPU) + % % % g = gpuArray(g); + % % % end + + + %SAVE_IMG(MRCImage(gather(rec{1})),'tmpR1.mrc'); + %SAVE_IMG(MRCImage(gather(rec{2})),'tmpR2.mrc'); + for iGold = 1:1+calcAllWeights - + if (outputScaling) ctfWeights{iGold,iCtfGroup} = gather(BH_reScale3d( rec{iGold}... - ,'', sprintf('%f',outputScaling), METHOD)); rec{iGold} = []; + ,'', sprintf('%f',outputScaling), METHOD)); rec{iGold} = []; else ctfWeights{iGold,iCtfGroup} = gather(rec{iGold}); rec{iGold} = []; end - - -% % % % % ctfWeights{iGold,iCtfGroup} = convn(single(rec{iGold}),g,'same'); rec{iGold} = []; - - -% % % % if (outputScaling ~= 1) -% % % % ctfWeights{iGold,iCtfGroup} = BH_reScale3d( ctfWeights{iGold,iCtfGroup}... -% % % % ,'', sprintf('%f',outputScaling), METHOD); -% % % % end -% % % % if (flgFirstPass) -% % % % % Only calc this once. -% % % % rad = fftshift(BH_bandpass3d(size(ctfWeights{iGold,iCtfGroup}),... -% % % % 0,0,0,'cpu','nyquist')); -% % % % flgFirstPass = 0; -% % % % end -% % % % -% % % % ctfWeights{iGold,iCtfGroup} = gather(ctfWeights{iGold,iCtfGroup}) .* rad; -% % % % -% % % % -% % % % ctfWeights{iGold,iCtfGroup} = (ctfWeights{iGold,iCtfGroup} - ... -% % % % min(ctfWeights{iGold,iCtfGroup}(rad > 0.0)))./((nPrjs)./2-0.5); -% % % % -% % % % % The mean value should probably be < 0.5 due to exposure filtering and -% % % % % inelastic losses. Figure out how to put this on an absolute scale. -% % % % %FIXME -% % % % -% % % % m = BH_movingAverage(ctfWeights{iGold,iCtfGroup},[7,7,7]); -% % % % -% % % % ctfWeights{iGold,iCtfGroup}(m < (mean(m(m>0)) + 0.5*std(m(m>0)))) = 0; -% % % % -% % % % meanPositiveValues = mean(mean(mean(ctfWeights{iGold,iCtfGroup}(ctfWeights{iGold,iCtfGroup} > 1e-2)))); -% % % % -% % % % ctfWeights{iGold,iCtfGroup} = gather(ctfWeights{iGold,iCtfGroup} .* (0.5/meanPositiveValues)); - - + + % % % % % ctfWeights{iGold,iCtfGroup} = convn(single(rec{iGold}),g,'same'); rec{iGold} = []; + + + + % % % % if (outputScaling ~= 1) + % % % % ctfWeights{iGold,iCtfGroup} = BH_reScale3d( ctfWeights{iGold,iCtfGroup}... + % % % % ,'', sprintf('%f',outputScaling), METHOD); + % % % % end + % % % % if (flgFirstPass) + % % % % % Only calc this once. + % % % % rad = fftshift(BH_bandpass3d(size(ctfWeights{iGold,iCtfGroup}),... + % % % % 0,0,0,'cpu','nyquist')); + % % % % flgFirstPass = 0; + % % % % end + % % % % + % % % % ctfWeights{iGold,iCtfGroup} = gather(ctfWeights{iGold,iCtfGroup}) .* rad; + % % % % + % % % % + % % % % ctfWeights{iGold,iCtfGroup} = (ctfWeights{iGold,iCtfGroup} - ... + % % % % min(ctfWeights{iGold,iCtfGroup}(rad > 0.0)))./((nPrjs)./2-0.5); + % % % % + % % % % % The mean value should probably be < 0.5 due to exposure filtering and + % % % % % inelastic losses. Figure out how to put this on an absolute scale. + % % % % %FIXME + % % % % + % % % % m = BH_movingAverage(ctfWeights{iGold,iCtfGroup},[7,7,7]); + % % % % + % % % % ctfWeights{iGold,iCtfGroup}(m < (mean(m(m>0)) + 0.5*std(m(m>0)))) = 0; + % % % % + % % % % meanPositiveValues = mean(mean(mean(ctfWeights{iGold,iCtfGroup}(ctfWeights{iGold,iCtfGroup} > 1e-2)))); + % % % % + % % % % ctfWeights{iGold,iCtfGroup} = gather(ctfWeights{iGold,iCtfGroup} .* (0.5/meanPositiveValues)); + + end - - + + end % end loop over ctfGroups clear rad g rec Hqz HqzUnMod B I data outOfBounds end function [ rWeight ] = calc_rWeight( SIZE, PRECISION, METHOD) - rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); - if strcmp(METHOD,'GPU') - rWeight = gpuArray(rWeight); - end - rOrig = ceil((SIZE(1)+1)./2); - % % % % imod tilt zero freq = 0.2 * first non zero component - rWeight(rOrig ) = 0.2; - [rCut] = find(rWeight == floor(0.45*SIZE(1))); - pixelFallOff = rCut(1) ; - taperLow = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); - - pixelFallOff = SIZE(1)-rCut(2)+1 ; - taperTop = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); - rWeight(1:rCut(1)) = rWeight(1:rCut(1)).*flip(taperLow)'; - rWeight(rCut(2):end) = rWeight(rCut(2):end).*taperTop'; - - - %rWeight = rWeight + 1./rWeight.^2; - % resample2d only handles scaling right now, so pad to z=3 - rWeight = repmat((rWeight), 1, SIZE(2),1); - if strcmpi(PRECISION,'single') - rWeight = single(rWeight); - else - % This should be the default. - rWeight = double(rWeight); - end +rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); +if strcmp(METHOD,'GPU') + rWeight = gpuArray(rWeight); +end +rOrig = ceil((SIZE(1)+1)./2); +% % % % imod tilt zero freq = 0.2 * first non zero component +rWeight(rOrig ) = 0.2; +[rCut] = find(rWeight == floor(0.45*SIZE(1))); +pixelFallOff = rCut(1) ; +taperLow = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); + +pixelFallOff = SIZE(1)-rCut(2)+1 ; +taperTop = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); +rWeight(1:rCut(1)) = rWeight(1:rCut(1)).*flip(taperLow)'; +rWeight(rCut(2):end) = rWeight(rCut(2):end).*taperTop'; + + +%rWeight = rWeight + 1./rWeight.^2; +% resample2d only handles scaling right now, so pad to z=3 +rWeight = repmat((rWeight), 1, SIZE(2),1); +if strcmpi(PRECISION,'single') + rWeight = single(rWeight); +else + % This should be the default. + rWeight = double(rWeight); +end end diff --git a/masking/BH_weightMask_dpRUN.m b/masking/BH_weightMask_dpRUN.m index 839bd0b3..7bfa22d1 100644 --- a/masking/BH_weightMask_dpRUN.m +++ b/masking/BH_weightMask_dpRUN.m @@ -13,12 +13,12 @@ if length(varargin{1}) > 1 tiltStart = varargin{1}(2); %ctfScaleFactor = varargin{1}(2)\ tiltStop = varargin{1}(3); - else + else tiltStart = 0; tiltStop = 0; end useGPU = varargin{1}(1); -else +else useGPU = -1; end tiltStart @@ -48,14 +48,14 @@ end [ctfWeights] = BH_weightMask_dp(subTomoMeta,wgtSize,samplingRate, ... - {tomoList,geom},'double','GPU', ... - ctfScaleFactor); - + {tomoList,geom},'double','GPU', ... + ctfScaleFactor); + if SYMMETRY > 1 for iGold = 1:2 ctfWeights{iGold} = gather(BH_resample3d(ctfWeights{iGold},[0,0,0], ... - [0,0,0],{'Bah',SYMMETRY,'linear',1}, ... - 'GPU','forward')); + [0,0,0],{'Bah',SYMMETRY,'linear',1}, ... + 'GPU','forward')); end end diff --git a/masking/EMC_applyBandpass.m b/masking/EMC_applyBandpass.m index e339262a..51892cbc 100644 --- a/masking/EMC_applyBandpass.m +++ b/masking/EMC_applyBandpass.m @@ -83,38 +83,38 @@ % full bandpass if isequal(bandSize, imgSize) IMAGE = fftn(IMAGE) .* BANDPASS; - + if OPTION.standardize IMAGE(1) = 0; IMAGE = IMAGE ./ (sqrt(sum(abs(IMAGE).^2, 'all')) / numel(IMAGE)); end - + if OPTION.ifft IMAGE = ifftn(IMAGE, 'symmetric'); end - -% half bandpass + + % half bandpass elseif isequal([floor(imgSize(1)/2)+1, imgSize(2:end)], bandSize) IMAGE = EMC_rfftn(IMAGE) .* BANDPASS; - + if OPTION.standardize IMAGE(1) = 0; - + % Capture every independant chunk (same for 2d or 3d) cD = ceil(imgSize(1)/2); % center donor factor = sum(abs(IMAGE(1,:,:)).^2, 'all'); % unique row/plane factor = factor + 2*sum(abs(IMAGE(2:cD,:,:)).^2, 'all'); % common chunk if ~mod(imgSize(1),2); factor = factor + sum(abs(IMAGE(cD+1,:,:)).^2, 'all'); end % unique row/plane - + IMAGE = IMAGE ./ (sqrt(factor) / prod(imgSize, 'native')); end - + if OPTION.ifft IMAGE = EMC_irfftn(IMAGE, imgSize); end else error('EMC:IMAGE', 'IMAGE (size:%s) and BANDPASS (size:%s) should have the same size', ... - mat2str(imgSize), mat2str(size(BANDPASS))) + mat2str(imgSize), mat2str(size(BANDPASS))) end end % EMC_applyBandpass diff --git a/masking/EMC_getBandpass.m b/masking/EMC_getBandpass.m index bf52bb64..3ec94f36 100644 --- a/masking/EMC_getBandpass.m +++ b/masking/EMC_getBandpass.m @@ -119,9 +119,9 @@ end [vX, vY, vZ] = EMC_coordVectors(SIZE, METHOD, {'origin', OPTION.origin; ... - 'half', OPTION.half; ... - 'normalize', true; ... - 'precision', OPTION.precision}); + 'half', OPTION.half; ... + 'normalize', true; ... + 'precision', OPTION.precision}); % The radial grid is the same for both highpass and lowpass filters. if flg.is3d @@ -136,7 +136,7 @@ %% High pass filter if flg.highpass highpassCut = PIXEL / HIGHPASS; % [1/pix] - + % Gaussian roll if strcmpi(OPTION.highpassRoll, 'extended') % roll, up to the zero frequency sigma = sqrt(-1 * highpassCut^2 / (2 * log(OPTION.highpassThresh))); @@ -147,7 +147,7 @@ else % classic gaussian roll sigma = OPTION.highpassRoll; end - + BANDPASS = (radius > highpassCut) + (radius <= highpassCut) .* gaussian(radius, highpassCut, sigma); end @@ -169,13 +169,13 @@ else % classic gaussian roll sigma = OPTION.lowpassRoll; end - + if flg.highpass BANDPASS = (radius < lowpassCut) .* BANDPASS + ... - (radius >= lowpassCut) .* gaussian(radius, lowpassCut, sigma); + (radius >= lowpassCut) .* gaussian(radius, lowpassCut, sigma); else BANDPASS = (radius < lowpassCut) + ... - (radius >= lowpassCut) .* gaussian(radius, lowpassCut, sigma); + (radius >= lowpassCut) .* gaussian(radius, lowpassCut, sigma); end end @@ -190,7 +190,7 @@ end if strcmpi(METHOD, 'gpu') - flg.gpu = true; + flg.gpu = true; elseif strcmpi(METHOD, 'cpu') flg.gpu = false; else @@ -203,7 +203,7 @@ end OPTION = EMC_getOption(OPTION, {'origin', 'half', 'highpassRoll', 'highpassThresh', ... - 'lowpassRoll', 'lowpassThresh', 'precision'}, false); + 'lowpassRoll', 'lowpassThresh', 'precision'}, false); % let EMC_coordVectors do the checkIN, just set the default. if ~isfield(OPTION, 'origin') @@ -219,13 +219,13 @@ % HIGHPASS cutoff if ~isscalar(HIGHPASS) error('EMC:HIGHPASS', 'HIGHPASS should be a scalar, got a %s of size %s', ... - class(HIGHPASS), mat2str(size(HIGHPASS))) + class(HIGHPASS), mat2str(size(HIGHPASS))) elseif isnan(HIGHPASS) || HIGHPASS == 0 flg.highpass = false; elseif isnumeric(HIGHPASS) && ~isinf(HIGHPASS) && HIGHPASS > 0 if HIGHPASS < PIXEL * 2 error('EMC:HIGHPASS', ... - 'HIGHPASS should be greater (lower resolution) or equal to Nyquist (%.3f)', PIXEL * 2) + 'HIGHPASS should be greater (lower resolution) or equal to Nyquist (%.3f)', PIXEL * 2) end flg.highpass = true; else @@ -238,7 +238,7 @@ flg.highpass = false; elseif LOWPASS < PIXEL * 2 error('EMC:LOWPASS', 'LOWPASS should be greater (lower resolution) or equal to Nyquist (%.3f)', ... - PIXEL * 2) + PIXEL * 2) elseif flg.highpass && HIGHPASS < LOWPASS error('EMC:LOWPASS', 'LOWPASS should be smaller (high frequency) than HIGHPASS') end @@ -252,10 +252,10 @@ % highpassRoll if isfield(OPTION, 'highpassRoll') if ~(isscalar(OPTION.highpassRoll) && isnumeric(OPTION.highpassRoll) ... - && OPTION.highpassRoll >= 0 && ~isinf(OPTION.highpassRoll)) ... - && ... - ~((isstring(OPTION.highpassRoll) || ischar(OPTION.highpassRoll)) && ... - strcmpi(OPTION.highpassRoll, 'extended')) + && OPTION.highpassRoll >= 0 && ~isinf(OPTION.highpassRoll)) ... + && ... + ~((isstring(OPTION.highpassRoll) || ischar(OPTION.highpassRoll)) && ... + strcmpi(OPTION.highpassRoll, 'extended')) error('EMC:highpassRoll', "OPTION.highpassRoll should be a positive float|int or 'extended'") end else @@ -266,10 +266,10 @@ if isfield(OPTION, 'highpassThresh') if ~isscalar(OPTION.highpassThresh) && ~isnumeric(OPTION.highpassThresh) error('EMC:highpassThresh', 'OPTION.highpassThresh should be a float|int, got %s of size %s', ... - class(OPTION.highpassThresh), mat2str(size(OPTION.highpassThresh))) + class(OPTION.highpassThresh), mat2str(size(OPTION.highpassThresh))) elseif OPTION.highpassThresh < 0 || OPTION.highpassThresh >= 1 || isnan(OPTION.highpassThresh) error('EMC:highpassThresh', 'OPTION.highpassThresh should be between 0 and 1, got %.3f', ... - OPTION.highpassThresh) + OPTION.highpassThresh) end else OPTION.highpassThresh = 1e-3; % default @@ -278,10 +278,10 @@ % lowpassRoll if isfield(OPTION, 'lowpassRoll') if ~(isscalar(OPTION.lowpassRoll) && isnumeric(OPTION.lowpassRoll) ... - && OPTION.lowpassRoll >= 0 && ~isinf(OPTION.lowpassRoll)) ... - && ... - ~((isstring(OPTION.lowpassRoll) || ischar(OPTION.lowpassRoll)) && ... - strcmpi(OPTION.lowpassRoll, 'extended')) + && OPTION.lowpassRoll >= 0 && ~isinf(OPTION.lowpassRoll)) ... + && ... + ~((isstring(OPTION.lowpassRoll) || ischar(OPTION.lowpassRoll)) && ... + strcmpi(OPTION.lowpassRoll, 'extended')) error('EMC:lowpassRoll', "OPTION.lowpassRoll should be a positive float|int or 'extended'") end else @@ -292,10 +292,10 @@ if isfield(OPTION, 'lowpassThresh') if ~isscalar(OPTION.lowpassThresh) && ~isnumeric(OPTION.lowpassThresh) error('EMC:lowpassThresh', 'OPTION.lowpassThresh should be a float|int, got %s of size %s', ... - class(OPTION.lowpassThresh), mat2str(size(OPTION.lowpassThresh))) + class(OPTION.lowpassThresh), mat2str(size(OPTION.lowpassThresh))) elseif OPTION.lowpassThresh < 0 || OPTION.lowpassThresh >= 1 || isnan(OPTION.lowpassThresh) error('EMC:lowpassThresh', 'OPTION.lowpassThresh should be between 0 and 1, got %.3f', ... - OPTION.lowpassThresh) + OPTION.lowpassThresh) end else OPTION.lowpassThresh = 1e-3; % default diff --git a/masking/EMC_limits.m b/masking/EMC_limits.m index b1991cbc..14eacb2c 100644 --- a/masking/EMC_limits.m +++ b/masking/EMC_limits.m @@ -52,7 +52,7 @@ % - Pad img to the desired size: % img = randn(128,128); % limits = EMC_limits(size(img), [150,155], {}); -% imgPadded = EMC_resize(img, limits, {}); +% imgPadded = EMC_resize(img, limits, {}); % % See also EMC_resize % @@ -64,20 +64,20 @@ % %% checkIN -if ~isnumeric(CURRENT) || ~isrow(CURRENT) || any(isinf(CURRENT)) || ~all(CURRENT > 0) || any(rem(CURRENT, 1)) +if ~isnumeric(CURRENT) || ~isrow(CURRENT) || any(isinf(CURRENT)) || ~all(CURRENT > 0) || any(rem(CURRENT, 1)) error('EMC:LIMITS', 'CURRENT should be a row vector of positive integers') -elseif ~isnumeric(DESIRED) || ~isrow(DESIRED) || any(isinf(DESIRED)) || ~all(DESIRED > 0) || any(rem(DESIRED, 1)) +elseif ~isnumeric(DESIRED) || ~isrow(DESIRED) || any(isinf(DESIRED)) || ~all(DESIRED > 0) || any(rem(DESIRED, 1)) error('EMC:LIMITS', 'DESIRED should be a row vector of positive integers') elseif ~isequal(size(CURRENT), size(DESIRED)) error('EMC:LIMITS', 'CURRENT and DESIRED should have the same size, got %s ~= %s', ... - mat2str(size(CURRENT)), mat2str(size(DESIRED))) + mat2str(size(CURRENT)), mat2str(size(DESIRED))) end OPTION = EMC_getOption(OPTION, {'shift', 'origin'}, false); if isfield(OPTION, 'origin') if ~isscalar(OPTION.origin) || ~isnumeric(OPTION.origin) || ... - ~(OPTION.origin == -1 || OPTION.origin == 1 || OPTION.origin == 2) + ~(OPTION.origin == -1 || OPTION.origin == 1 || OPTION.origin == 2) error('EMC:origin', "OPTION.origin should be 1, 2, or -1") end else @@ -91,10 +91,10 @@ error('EMC:shift', 'OPTION.shift should only contain integers, got %s', mat2str(OPTION.shift, 2)) elseif ~isequal(size(OPTION.shift), size(CURRENT)) error('EMC:shift', 'CURRENT and OPTION.shift should have the same size, got %s - %s', ... - mat2str(size(CURRENT)), mat2str(size(OPTION.shift))) + mat2str(size(CURRENT)), mat2str(size(OPTION.shift))) elseif OPTION.origin == -1 && any(OPTION.shift) - error('EMC:shift', 'OPTION.shifts are not allowed with half=true or origin=-1, got %s', ... - mat2str(OPTION.shift, 2)) + error('EMC:shift', 'OPTION.shifts are not allowed with half=true or origin=-1, got %s', ... + mat2str(OPTION.shift, 2)) end else OPTION.shift = zeros(1, numel(CURRENT)); % default diff --git a/masking/EMC_maskIndex.m b/masking/EMC_maskIndex.m index a8ec405c..4e272309 100644 --- a/masking/EMC_maskIndex.m +++ b/masking/EMC_maskIndex.m @@ -125,7 +125,7 @@ error('EMC:precision', "OPTION.precision should be 'single', 'double', 'int' or 'uint', got %s", OPTION.precision) end else - error('EMC:precision', "OPTION.precision should be a string|char vector, got %s", class(OPTION.precision)) + error('EMC:precision', "OPTION.precision should be a string|char vector, got %s", class(OPTION.precision)) end elseif prod(SIZE) <= 2^16 - 1 OPTION.precision = 'uint16'; % default @@ -139,10 +139,10 @@ if strcmpi(TYPE, 'nc2nc') || strcmpi(TYPE, 'c2nc') cR = floor(SIZE/2) + 1; % center receiver cD = ceil(SIZE/2); % center donor - + SIZE = EMC_setMethod(cast(SIZE, OPTION.precision), METHOD); INDEX = reshape(1:prod(SIZE, 'native'), SIZE); % linear indexes of a grid with desired size - + if is3d INDEX(cR(1)+1:end, 1, 1) = INDEX(cD(1):-1:2, 1, 1); INDEX(cR(1)+1:end, 2:cR(2), 1) = INDEX(cD(1):-1:2, end:-1:cD(2)+1, 1); @@ -158,18 +158,18 @@ INDEX(cR(1)+1:end, 2:cR(2)) = INDEX(cD(1):-1:2, end:-1:cD(2)+1); INDEX(cR(1)+1:end, cR(2)+1:end) = INDEX(cD(1):-1:2, cD(2):-1:2); end - + elseif strcmpi(TYPE, 'c2c') c = floor(SIZE/2) + 1; % center e = 1 + ~mod(SIZE, 2); % left edge o = ceil(SIZE(1)/2); % lenght common chunk - + SIZE = EMC_setMethod(cast(SIZE, OPTION.precision), METHOD); INDEX = reshape(1:prod(SIZE, 'native'), SIZE); % linear indexes of a grid with desired size if e(1) == 2 % X is even extra = INDEX(c(1), :, :); % save the extra line/plane end - + % Shift half to end of X, then flip the common chuck and then deal with % extra line/plane for even dimensions. if is3d @@ -197,8 +197,8 @@ INDEX(e(1):c(1)-1, 1) = INDEX(end:-1:c(1)+1, 1); end end - -% fftshift and ifftshift + + % fftshift and ifftshift else if strcmpi(TYPE, 'fftshift') half = ceil(SIZE/2); @@ -207,14 +207,14 @@ else error('EMC:TYPE', "TYPE should be 'fftshift', 'ifftshift', 'nc2nc' or 'c2c'") end - + SIZE = EMC_setMethod(cast(SIZE, OPTION.precision), METHOD); % convert after the division if OPTION.half; vX = (1:SIZE(1))'; else; vX = [half(1)+1:SIZE(1), 1:half(1)]'; end - + % Concatenation appears to be faster than fftshift and circshift. if is3d INDEX = vX + [half(2):SIZE(2)-1, 0:half(2)-1] * SIZE(1) + ... - reshape([half(3):SIZE(3)-1, 0:half(3)-1],1,1,[]) * (SIZE(1) * SIZE(2)); + reshape([half(3):SIZE(3)-1, 0:half(3)-1],1,1,[]) * (SIZE(1) * SIZE(2)); else INDEX = vX + [half(2):SIZE(2)-1, 0:half(2)-1] * SIZE(1); end diff --git a/masking/EMC_maskReference.m b/masking/EMC_maskReference.m index 0ee9b679..fe505557 100644 --- a/masking/EMC_maskReference.m +++ b/masking/EMC_maskReference.m @@ -110,20 +110,20 @@ if flg.is3d if flg.isOnGpu IMAGE = EMC_applyBandpass(... - gpuArray(medfilt3(gather(IMAGE), [3,3,3])) .* minusEdges, ... - EMC_getBandpass(SIZE, PIXEL, nan, OPTION.lowpass, flg.method, optBandpass), ... - {}); + gpuArray(medfilt3(gather(IMAGE), [3,3,3])) .* minusEdges, ... + EMC_getBandpass(SIZE, PIXEL, nan, OPTION.lowpass, flg.method, optBandpass), ... + {}); else IMAGE = EMC_applyBandpass(... - medfilt3(IMAGE, [3,3,3]) .* minusEdges, ... - EMC_getBandpass(SIZE, PIXEL, nan, OPTION.lowpass, flg.method, optBandpass), ... - {}); + medfilt3(IMAGE, [3,3,3]) .* minusEdges, ... + EMC_getBandpass(SIZE, PIXEL, nan, OPTION.lowpass, flg.method, optBandpass), ... + {}); end else IMAGE = EMC_applyBandpass(... - medfilt2(IMAGE, [3,3]) .* minusEdges, ... - EMC_getBandpass(SIZE, PIXEL, nan, OPTION.lowpass, flg.method, optBandpass), ... - {}); + medfilt2(IMAGE, [3,3]) .* minusEdges, ... + EMC_getBandpass(SIZE, PIXEL, nan, OPTION.lowpass, flg.method, optBandpass), ... + {}); end % Make sure no wrap-around artifacts. @@ -152,7 +152,7 @@ for i = 1:ceil(threshold.^2 ./ 3) binaryMask = (~binaryMask .* EMC_convn(tofloat(binaryMask), dilationKernel) > 0.00) + binaryMask; end - + % This part is crucial as it restricts the expansion to the close pixel higher than the current % threshold. This is where the 'connectivity' really happens. binaryMask = (IMAGE .* binaryMask) > threshold; @@ -162,7 +162,7 @@ % Save this volume to compute the particle fraction. if OPTION.fsc particleVolEstimate = sum(binaryMask, 'all'); - + taperKernel = EMC_gaussianKernel([1,4], 1.75, flg.method, {'precision', flg.precision}); MASK_CORE = EMC_convn(tofloat(binaryMask), taperKernel); MASK_CORE = EMC_convn(sqrt(MASK_CORE),taperKernel); @@ -182,57 +182,57 @@ % one can arbitrarily decrease the measured SNR. Even though this expansion is almost certainly % containing surrounding solvent, estimate the signal reduction in the taper as the mask could % cut through densities. -if OPTION.fsc +if OPTION.fsc smoothKernel = EMC_gaussianKernel([1, rad], rad/2, flg.method, {'precision', flg.precision}); fscMask = EMC_convn(tofloat(binaryMask), convn(smoothKernel, smoothKernel)); fscMask = fscMask ./ max(fscMask(:)); maskVolume = sum(fscMask>0, 'all'); - + powerReduction = sum(IMAGE.^2 .* (fscMask>0), 'all') ./ sum(IMAGE.^2 .* fscMask, 'all'); - + % Scale the particle volume; 'remove' its hydration volume. % TODO: so we assume particleVolEstimate contains some solvent then? particleVolEstimate = particleVolEstimate ./ OPTION.hydration_scaling; - + % Finally, estimate the fraction of the mask taken by the particle, by % comparing the particle volume (not-hydrated) to the mask volume (scalled down % to take into account the power reduction due to the taper). FRACTION = maskVolume ./ (particleVolEstimate .* powerReduction); - + if EMC_gp_verbose fprintf(['FSC mask: Estimated particule volume : %d voxels\n', ... - ' Estimated mask volume : %d voxels\n', ... - ' Power reduction : %2.3f\n', ... - ' Particle fraction : %2.3f\n'], ... - particleVolEstimate, maskVolume, powerReduction, 1/FRACTION); + ' Estimated mask volume : %d voxels\n', ... + ' Power reduction : %2.3f\n', ... + ' Particle fraction : %2.3f\n'], ... + particleVolEstimate, maskVolume, powerReduction, 1/FRACTION); end - + % Make sure no wrap-around artifacts. MASK = fscMask .* minusEdges; - + % To compute the COM, restrict to the region most likely to contain only the protein. if OPTION.com; COM = EMC_centerOfMass(MASK .* binaryMask, OPTION.origin); else; COM = nan; end - + % Setting varargout. varargout = {MASK_CORE, FRACTION, COM}; elseif OPTION.pca smoothKernel = EMC_gaussianKernel([1, rad], rad/2, flg.method, {'precision', flg.precision}); MASK = EMC_convn(tofloat(binaryMask), convn(smoothKernel, smoothKernel)); - MASK = MASK ./ max(MASK(:)); + MASK = MASK ./ max(MASK(:)); % Make sure no wrap-around artifacts. MASK = MASK .* minusEdges; - + if OPTION.com; COM = EMC_centerOfMass(MASK, OPTION.origin); else; COM = nan; end - + % Setting varargout. varargout = {COM}; else % Make sure no wrap-around artifacts. MASK = binaryMask .* minusEdges; - + if OPTION.com; COM = EMC_centerOfMass(MASK, OPTION.origin); else; COM = nan; end - + % Setting varargout. varargout = {COM}; end @@ -256,13 +256,13 @@ end OPTION = EMC_getOption(OPTION, {'origin', 'fsc', 'pca', 'com', 'lowpass', 'threshold', ... - 'hydration_scaling', 'precision', 'method'}, false); + 'hydration_scaling', 'precision', 'method'}, false); % origin if isfield(OPTION, 'origin') if ~isscalar(OPTION.origin) || ~isnumeric(OPTION.origin) error('EMC:origin', 'OPTION.origin should be an integer, got %s of size: %s', ... - class(OPTION.origin), mat2str(size(OPTION.origin))) + class(OPTION.origin), mat2str(size(OPTION.origin))) elseif OPTION.origin ~= 1 && OPTION.origin ~= 0 && OPTION.origin ~= 2 error('EMC:origin', 'OPTION.origin should be 0, 1 or 2, got %d', OPTION.origin) end @@ -303,7 +303,7 @@ % lowpass if isfield(OPTION, 'lowpass') if ~isscalar(OPTION.lowpass) || ~isnumeric(OPTION.lowpass) || ... - isinf(OPTION.lowpass) || ~(OPTION.lowpass >= 0) + isinf(OPTION.lowpass) || ~(OPTION.lowpass >= 0) error('EMC:LOWPASS', 'OPTION.lowpass should be a nonnegative float|int') elseif OPTION.lowpass < PIXEL * 2 OPTION.lowpass = PIXEL * 2; @@ -316,7 +316,7 @@ if isfield(OPTION, 'threshold') if ~isscalar(OPTION.threshold) || ~isnumeric(OPTION.threshold) error('EMC:threshold', 'OPTION.threshold should be a numeric scalar, got %s of size: %s', ... - class(OPTION.threshold), mat2str(size(OPTION.threshold))) + class(OPTION.threshold), mat2str(size(OPTION.threshold))) end else OPTION.threshold = 2.5; % default @@ -325,7 +325,7 @@ % hydration_scaling if isfield(OPTION, 'hydration_scaling') if ~isscalar(OPTION.hydration_scaling) || ~isnumeric(OPTION.hydration_scaling) || ... - isinf(OPTION.hydration_scaling) || ~(OPTION.hydration_scaling >= 0) + isinf(OPTION.hydration_scaling) || ~(OPTION.hydration_scaling >= 0) error('EMC:hydration_scaling', 'OPTION.hydration_scaling should be a nonnegative float|int') end else diff --git a/masking/EMC_maskShape.m b/masking/EMC_maskShape.m index b8d6b4b3..c0ec5091 100644 --- a/masking/EMC_maskShape.m +++ b/masking/EMC_maskShape.m @@ -91,7 +91,7 @@ % are casted to the nearest integers (9Mar2020). % -%% +%% [SIZE, OPTION, flg] = checkIN(SIZE, RADIUS, METHOD, OPTION); cutoffLow = 0.001; % everything below this value is set to 0. @@ -101,7 +101,7 @@ % Compute the size of the kernel in pixel. kernelSize = round((min(SIZE) * OPTION.kernel - 1) / 2) * 2 + 1; % closest odd int if kernelSize < 9; kernelSize = 9; end % at least 9 pixels - + % Keep the gaussian at ~0.5 at the middle of the roll off. middle = ceil(kernelSize / 2) / 2; sigma = sqrt(-1 * middle^2 / (2 * log(0.5))); @@ -109,12 +109,12 @@ else kernelSize = length(OPTION.kernel); end - + % To make the shape go to zeros at the edges of the mask, the function tape the mask % with zeros. This create a minimum size on the mask. if any(SIZE < ceil(kernelSize/2) * 2 + 1) error('EMC:kernel', 'with a kernel size of %d, the minimum SIZE is %d, got %s', ... - kernelSize, ceil(kernelSize/2) * 2 + 1, mat2str(SIZE)) + kernelSize, ceil(kernelSize/2) * 2 + 1, mat2str(SIZE)) end else kernelSize = 0; @@ -128,9 +128,9 @@ % Define the center of the sphere/circle using the specified origin and shifts. [vX, vY, vZ] = EMC_coordVectors(SIZE, METHOD, {'origin', OPTION.origin; ... - 'shift', OPTION.shift; ... - 'precision', OPTION.precision}); - + 'shift', OPTION.shift; ... + 'precision', OPTION.precision}); + % Compute a radial cartesian grid and apply the equation of the sphere/ellipsoid. % The surface of the ellipsoid is at 1. At this point, the binary ellipsoid mask is computed. if (flg.is3d) @@ -142,18 +142,18 @@ % Cast from logical to float. MASK = cast(MASK, OPTION.precision); -elseif strcmpi(SHAPE, 'cylinder') +elseif strcmpi(SHAPE, 'cylinder') % Adjust the radius for the same reason as explain with sphere/ellipsoids. RADIUS(1:2) = RADIUS(1:2) + ceil(kernelSize/2); - + % To compute a (3d) cylinder, this algorithm first compute a 2d sphere/ellipsoid and then % broadcast it along the Z (depth) axis. if flg.is3d % First compute the 2d ellipsoid with the x and y shifts. [vX, vY] = EMC_coordVectors(SIZE(1:2), METHOD, {'origin', OPTION.origin; ... - 'precision', OPTION.precision; - 'shift', OPTION.shift(1:2)}); - + 'precision', OPTION.precision; + 'shift', OPTION.shift(1:2)}); + % Broadcaste in Z (the ellipsoid is invariant in Z <=> cylinder). RADIUS(3) = round(RADIUS(3)) + floor(kernelSize/2); % adjust Z for blurring. if flg.gpu @@ -162,30 +162,30 @@ vZ = zeros([1, 1, RADIUS(3) .*2 + 1], OPTION.precision); end MASK = (vX'./RADIUS(1)).^2 + (vY./RADIUS(2)).^2 + vZ <= 1; % broadcast - + % Cast from logical to float. MASK = cast(MASK, OPTION.precision); - + % Resize the cylinder to the desired SIZE in Z, taking into account the z shift. OPTION.shift(3) = round(OPTION.shift(3)); limits = EMC_limits(size(MASK), SIZE, {'origin', OPTION.origin; 'shift', [0, 0, OPTION.shift(3)]}); MASK = EMC_resize(MASK, limits, {'taper', false}); - + else % 2d Cylinder; this block is equivalent to SHAPE='sphere'. [vX, vY] = EMC_coordVectors(SIZE, METHOD, {'origin', OPTION.origin; ... - 'precision', OPTION.precision; ... - 'shift', OPTION.shift}); + 'precision', OPTION.precision; ... + 'shift', OPTION.shift}); MASK = (vX'./RADIUS(1)).^2 + (vY./RADIUS(2)).^2 <= 1; - + % Cast from logical to float. MASK = cast(MASK, OPTION.precision); end - + elseif strcmpi(SHAPE, 'rectangle') % Adjust the radius for the convolution. RADIUS = round(RADIUS) + floor(kernelSize/2); OPTION.shift = round(OPTION.shift); - + if flg.gpu MASK = ones(RADIUS .*2 + 1, OPTION.precision, 'gpuArray'); else @@ -193,7 +193,7 @@ end limits = EMC_limits(size(MASK), SIZE, {'origin', OPTION.origin; 'shift', OPTION.shift}); MASK = EMC_resize(MASK, limits, {'taper', false}); - + else if ~(ischar(SHAPE) || isstring(SHAPE)) error('EMC:SHAPE', "SHAPE should be string or char array, got %s", class(SHAPE)) @@ -205,8 +205,8 @@ %% Restrict the mask to the first symmetry pair. if flg.sym [~, angles, ~] = EMC_coordGrids('cylindrical', SIZE, METHOD, {'shift', OPTION.shift; ... - 'origin', OPTION.origin; ... - 'precision', OPTION.precision}); + 'origin', OPTION.origin; ... + 'precision', OPTION.precision}); sectorMax = 2*pi / OPTION.sym * 1.025; % small overlap angles = (angles > (2*pi-sectorMax/2) | angles < sectorMax/2); MASK = MASK .* angles; @@ -234,7 +234,7 @@ end if ~isnumeric(RADIUS) || ~isvector(RADIUS) || ~all(RADIUS > 1) || ... - ~isequal(size(SIZE), size(RADIUS)) || any(isinf(RADIUS)) + ~isequal(size(SIZE), size(RADIUS)) || any(isinf(RADIUS)) error('EMC:RADIUS', 'RADIUS should be a numeric 1x%d vector with every element greater than 1', ndim) end @@ -254,9 +254,9 @@ error('EMC:shift', 'OPTION.shift should be a vector of float|int, got %s', class(OPTION.shift)) elseif any(isnan(OPTION.shift)) || any(isinf(OPTION.shift)) error('EMC:shift', 'OPTION.shift should not contain NaNs or Inf, got %s', mat2str(OPTION.shift, 2)) - elseif numel(OPTION.shift) ~= ndim + elseif numel(OPTION.shift) ~= ndim error('EMC:shift', 'For a %dd SIZE, OPTION.shift should be a vector of %d float|int, got %s', ... - ndim, ndim, mat2str(OPTION.shift, 2)) + ndim, ndim, mat2str(OPTION.shift, 2)) end else OPTION.shift = zeros(1, ndim); % default @@ -265,7 +265,7 @@ % origin if isfield(OPTION, 'origin') if ~isscalar(OPTION.origin) || ~isnumeric(OPTION.origin) || ... - ~(OPTION.origin == 0 || OPTION.origin == 1 || OPTION.origin == 2) + ~(OPTION.origin == 0 || OPTION.origin == 1 || OPTION.origin == 2) % EMC_resize (used with 'rectangle' and 3d 'cylinders') will raise an error if origin=0 error('EMC:origin', "OPTION.origin should be 0, 1, or 2, got %.04f", OPTION.origin) end @@ -309,17 +309,17 @@ flg.ownKernel = true; else error('EMC:kernel', ['OPTION.kernel should be a boolean, a positive float between 0 and 1,', ... - 'or a row numeric vector, got %s', class(OPTION.taper)]) + 'or a row numeric vector, got %s', class(OPTION.taper)]) end else - OPTION.kernel = 0.04; % default - flg.kernel = true; + OPTION.kernel = 0.04; % default + flg.kernel = true; end % sym if isfield(OPTION, 'sym') if isnumeric(OPTION.sym) && isscalar(OPTION.sym) && OPTION.sym > 0 && ... - ~isinf(OPTION.sym) && ~rem(OPTION.sym, 1) + ~isinf(OPTION.sym) && ~rem(OPTION.sym, 1) if OPTION.sym == 1 flg.sym = false; else diff --git a/masking/EMC_resize.m b/masking/EMC_resize.m index 19f43e46..60bca896 100644 --- a/masking/EMC_resize.m +++ b/masking/EMC_resize.m @@ -2,7 +2,7 @@ % % [OUT] = EMC_resize(IMAGE, LIMITS, OPTION) % Pad and/or crop an IMAGE. -% +% % Inputs: % IMAGE (single|double): 2d/3d IMAGE to pad and/or crop. % @@ -161,19 +161,19 @@ else % centered IMAGE: 'origin' = 0|1|2 crop = abs(LIMITS .* (LIMITS < 0)); pad = LIMITS .* (LIMITS > 0); - + % Taper if flg.taper && (any(pad) || OPTION.force_taper) IMAGE = applyTaper_real3d(IMAGE, LIMITS, OPTION, pad, crop, val); end - + % Pad/Crop if flg.pad && flg.crop OUT(1+pad(1):end-pad(2), ... 1+pad(3):end-pad(4), ... 1+pad(5):end-pad(6)) = IMAGE(1+crop(1):end-crop(2), ... - 1+crop(3):end-crop(4), ... - 1+crop(5):end-crop(6)); + 1+crop(3):end-crop(4), ... + 1+crop(5):end-crop(6)); elseif flg.pad OUT(1+pad(1):end-pad(2), 1+pad(3):end-pad(4), 1+pad(5):end-pad(6)) = IMAGE; elseif flg.crop @@ -187,10 +187,10 @@ crop = reshape(LIMITS .* (LIMITS < 0), 2, []); l = floor((inSize+1) / 2) + crop(1, :); % left side r = floor((inSize-2) / 2) + crop(2, :); % right side - + % Taper if flg.taper; IMAGE = applyTaper_fft2d(IMAGE, LIMITS, OPTION, l, r, val); end - + % Pad/Crop if flg.pad || flg.crop OUT(1:l(x), 1:l(y)) = IMAGE(1:l(x), 1:l(y)); @@ -203,12 +203,12 @@ else % centered IMAGE: 'origin' = 0|1|2 crop = abs(LIMITS .* (LIMITS < 0)); pad = LIMITS .* (LIMITS > 0); - + % Taper if flg.taper && (any(pad) || OPTION.force_taper) IMAGE = applyTaper_real2d(IMAGE, LIMITS, OPTION, pad, crop, val); end - + % Pad/Crop if flg.pad && flg.crop OUT(1+pad(1):end-pad(2), 1+pad(3):end-pad(4)) = IMAGE(1+crop(1):end-crop(2), 1+crop(3):end-crop(4)); @@ -357,11 +357,11 @@ flg.crop = false; elseif ~isnumeric(LIMITS) error('EMC:LIMITS', 'For a %dd IMAGE, LIMITS should be a row vector of %d integers, got %s', ... - ndim, ndim*2, class(LIMITS)); + ndim, ndim*2, class(LIMITS)); elseif ~isrow(LIMITS) || any(isnan(LIMITS)) || any(isinf(LIMITS)) || ... - any(rem(LIMITS,1)) || numel(LIMITS) ~= ndim * 2 + any(rem(LIMITS,1)) || numel(LIMITS) ~= ndim * 2 error('EMC:LIMITS', 'For a %dd IMAGE, LIMITS should be a row vector of %d integers, got %s', ... - ndim, ndim*2, mat2str(LIMITS)); + ndim, ndim*2, mat2str(LIMITS)); else if any(LIMITS > 0); flg.pad = true; else; flg.pad = false; end if any(LIMITS < 0); flg.crop = true; else; flg.crop = false; end @@ -392,8 +392,8 @@ [precision, flg.gpu, ~] = EMC_getClass(IMAGE); if isfield(OPTION, 'precision') if ~(ischar(OPTION.precision) || isstring(OPTION.precision)) || ... - ~(strcmpi('single', OPTION.precision) || strcmpi('double', OPTION.precision)) - error('EMC:precision', "OPTION.precision should be 'single' or 'double'") + ~(strcmpi('single', OPTION.precision) || strcmpi('double', OPTION.precision)) + error('EMC:precision', "OPTION.precision should be 'single' or 'double'") end else OPTION.precision = precision; @@ -431,13 +431,13 @@ else flg.taper = false; end - % cell | struct + % cell | struct elseif iscell(OPTION.taper) || isstruct(OPTION.taper) OPTION.taper = EMC_getOption(OPTION.taper, {'type', 'numel', 'percent'}, false); % type if isfield(OPTION.taper, 'type') if ~(ischar(OPTION.taper.type) || isstring(OPTION.taper.type)) || ... - ~strcmpi(OPTION.taper.type, 'cosine') && ~strcmpi(OPTION.taper.type, 'linear') + ~strcmpi(OPTION.taper.type, 'cosine') && ~strcmpi(OPTION.taper.type, 'linear') error('EMC:taper', "OPTION.taper.type should be 'linear' or 'cosine'") end else @@ -448,10 +448,10 @@ if isfield(OPTION.taper, 'percent') error('EMC:taper', 'OPTION.taper.numel and OPTION.taper.percent are mutually exclusive') end - % percent + % percent elseif isfield(OPTION.taper, 'percent') if ~isscalar(OPTION.taper.percent) || ~isnumeric(OPTION.taper.percent) || ... - isinf(OPTION.taper.percent) || ~(OPTION.taper.percent >= 0) || OPTION.taper.percent >= 1 + isinf(OPTION.taper.percent) || ~(OPTION.taper.percent >= 0) || OPTION.taper.percent >= 1 error('EMC:taper', 'OPTION.taper.percent should be a scalar, with: 0 <= scalar < 1') else outSize = SIZE + sum(reshape(LIMITS, 2, [])); @@ -464,8 +464,8 @@ end OPTION.taper = EMC_taper(OPTION.taper.type, OPTION.taper.numel, {}); flg.taper = true; - - % vector|int|float: own taper + + % vector|int|float: own taper elseif isnumeric(OPTION.taper) && isrow(OPTION.taper) if ~flg.gpu && EMC_isOnGpu(OPTION.taper) OPTION.taper = gather(OPTION.taper); @@ -484,7 +484,7 @@ if isfield(OPTION, 'force_taper') if ~islogical(OPTION.force_taper) || ~isscalar(OPTION.force_taper) error('EMC:force_taper', ... - 'OPTION.force_taper should be a boolean, got %s', class(OPTION.force_taper)) + 'OPTION.force_taper should be a boolean, got %s', class(OPTION.force_taper)) end else OPTION.force_taper = false; % default @@ -504,10 +504,10 @@ function raiseError(IMAGE, LIMITS, OPTION) % CASE 1: the taper is too large given the input IMAGE. if OPTION.origin == -1 && any(numel(OPTION.taper) > floor(inSize/2)) error('EMC:taper', 'For a size of %s, the maximum taper for a fft IMAGE is %s, got %d', ... - mat2str(inSize), mat2str(floor(inSize/2)), numel(OPTION.taper)); + mat2str(inSize), mat2str(floor(inSize/2)), numel(OPTION.taper)); elseif any(numel(OPTION.taper) > inSize) error('EMC:taper', 'For a size of %s, the maximum taper for an IMAGE is %s, got %d', ... - mat2str(inSize), mat2str(inSize), numel(OPTION.taper)); + mat2str(inSize), mat2str(inSize), numel(OPTION.taper)); end extendedSize = reshape(ones(2, ndim) .* inSize, 1, []); @@ -530,15 +530,15 @@ function raiseError(IMAGE, LIMITS, OPTION) halfSize(1, :) = halfSize(1, :) + mod(inSize, 2); % count extra pixel if odd; halfSize = reshape(halfSize, 1, []); maxCrop = -1 .* (halfSize - taperToApply); - -% CASE 3: For a real space IMAGE, if the size of the IMAGE is smaller than size (cropping + tapter), -% applying the taper will raise an index error. + + % CASE 3: For a real space IMAGE, if the size of the IMAGE is smaller than size (cropping + tapter), + % applying the taper will raise an index error. else maxCrop = -1 .* (extendedSize - taperToApply); end error('EMC:taper', ['One axis is too small given the cropping and taper required.\n\nGiven the inputs ', ... - '(IMAGE size: %s and taper size: %s),\nthe maximum cropping for each edges is %s, but got %s.'], ... - mat2str(inSize), mat2str(taperToApply), mat2str(maxCrop), mat2str((LIMITS < 0) .* LIMITS)); + '(IMAGE size: %s and taper size: %s),\nthe maximum cropping for each edges is %s, but got %s.'], ... + mat2str(inSize), mat2str(taperToApply), mat2str(maxCrop), mat2str((LIMITS < 0) .* LIMITS)); end % raiseError diff --git a/masking/EMC_taper.m b/masking/EMC_taper.m index ef01119f..71f8a82e 100644 --- a/masking/EMC_taper.m +++ b/masking/EMC_taper.m @@ -61,8 +61,8 @@ % precision if isfield(OPTION, 'precision') if ~(ischar(OPTION.precision) || isstring(OPTION.precision)) || ... - ~(strcmpi('single', OPTION.precision) || strcmpi('double', OPTION.precision)) - error('EMC:precision', "OPTION.precision should be 'single' or 'double'") + ~(strcmpi('single', OPTION.precision) || strcmpi('double', OPTION.precision)) + error('EMC:precision', "OPTION.precision should be 'single' or 'double'") end else OPTION.precision = 'single'; % default @@ -71,8 +71,8 @@ % method if isfield(OPTION, 'method') if ~(ischar(OPTION.method) || isstring(OPTION.method)) || ... - ~(strcmpi('gpu', OPTION.method) || strcmpi('cpu', OPTION.method)) - error('EMC:method', "OPTION.method should be 'cpu' or 'gpu'") + ~(strcmpi('gpu', OPTION.method) || strcmpi('cpu', OPTION.method)) + error('EMC:method', "OPTION.method should be 'cpu' or 'gpu'") end else OPTION.method = 'cpu'; % default @@ -82,7 +82,7 @@ if isfield(OPTION, 'start') if ~isscalar(OPTION.start) || ~isnumeric(OPTION.start) || isnan(OPTION.start) || isinf(OPTION.start) error('EMC:start', 'OPTION.start should be a float|int, got %s, numel=%d', ... - class(OPTION.start), numel(OPTION.start)) + class(OPTION.start), numel(OPTION.start)) end else OPTION.start = 1; % default @@ -92,7 +92,7 @@ if isfield(OPTION, 'end') if ~isscalar(OPTION.end) || ~isnumeric(OPTION.end) || isnan(OPTION.end) || isinf(OPTION.end) error('EMC:end', 'OPTION.end should be a float|int, got %s, numel=%d', ... - class(OPTION.end), numel(OPTION.end)) + class(OPTION.end), numel(OPTION.end)) end else OPTION.end = 0; % default @@ -108,7 +108,7 @@ end else error('EMC:first', 'OPTION.first should be a (scalar) bool, got %s, numel=%d', ... - class(OPTION.first), numel(OPTION.first)) + class(OPTION.first), numel(OPTION.first)) end else OPTION.first = 0; % default diff --git a/masking/alignmentVol.m b/masking/alignmentVol.m index e76477d2..97be6abf 100644 --- a/masking/alignmentVol.m +++ b/masking/alignmentVol.m @@ -1,14 +1,14 @@ classdef alignmentVol < handle properties (Access = 'private') - data; - input_size; - pad_val; + data; + input_size; + pad_val; end properties (Access = 'public') end - + methods - + function obj = alignmentVol(data, pad_val) pause(3) fprintf('alignmentVol constructor\n') @@ -17,7 +17,7 @@ obj.input_size = size(obj.data); obj.pad_val = pad_val; end - + function [] = isAllocated(obj) if isempty(obj.data) error('data is not allocated') @@ -25,22 +25,22 @@ fprintf('data is allocated\n') end end - + function [] = zero_data(obj) obj.data = obj.data .* 0; end end + + +end +% function [ IMAGE ] = BH_padZerosSimple3d(IMAGE) - end - -% function [ IMAGE ] = BH_padZerosSimple3d(IMAGE) - - %PADLOW, PADTOP, ... - %METHOD, PRECISION, varargin ) +%PADLOW, PADTOP, ... +%METHOD, PRECISION, varargin ) %Pad an image volume with zeros. -% +% % % Input variables: % @@ -55,7 +55,7 @@ % METHOD = case sensitive 'GPU' otherwise cpu % % Output variables:BH_bandpass3d.m -% +% % PADDED_IMG = the padded image. % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -76,7 +76,7 @@ % pause(3) % IMAGE = IMAGE .* 0; -% IMAGE(1:7,1:7,1:7) = 1; +% IMAGE(1:7,1:7,1:7) = 1; % pause(3) % if isnumeric(PADLOW) % padLOW = PADLOW ; @@ -187,7 +187,7 @@ % if (doRand) % PADDED_IMG = randn(padSize,'double').*extrapVal+extrapMean; % else -% PADDED_IMG = zeros(padSize,'double'); +% PADDED_IMG = zeros(padSize,'double'); % end % else % error('PRECISION must be single or double, not %s', PRECISION) @@ -264,7 +264,7 @@ % PADDED_IMG(1:sX1,1:sY1,end-sZ2:end) = IMAGE(1:sX1,1:sY1,end-sZ2:end); % PADDED_IMG(end-sX2:end,1:sY1,end-sZ2:end) = IMAGE(end-sX2:end,1:sY1,end-sZ2:end); % PADDED_IMG(1:sX1,end-sY2:end,end-sZ2:end) = IMAGE(1:sX1,end-sY2:end,end-sZ2:end); -% PADDED_IMG(end-sX2:end,end-sY2:end,end-sZ2:end) = IMAGE(end-sX2:end,end-sY2:end,end-sZ2:end); +% PADDED_IMG(end-sX2:end,end-sY2:end,end-sZ2:end) = IMAGE(end-sX2:end,end-sY2:end,end-sZ2:end); % else % PADDED_IMG(padLOW(1)+1: end - padTOP(1), ... @@ -274,7 +274,7 @@ % end -% clear IMAGE +% clear IMAGE % end of the padZeros3d functions diff --git a/metaData/BH_checkInstall.m b/metaData/BH_checkInstall.m index 618b12a5..fb4bdcfe 100755 --- a/metaData/BH_checkInstall.m +++ b/metaData/BH_checkInstall.m @@ -7,7 +7,7 @@ system(sprintf('%s',runPath)); fOUT = fopen('emClarity_checkInstall.txt','a'); - + [status, returnVal] = system('which chimera'); @@ -38,11 +38,11 @@ fprintf(fOUT,'\n\n##########\ngpu = %d\n##########\n\n',iGPU); g = gpuDevice(iGPU); fprintf(fOUT,['Name:\t%s\nComputeCapability:\t%s\nDriverVersion:\t%f\n',... - 'ToolkitVersion:\t%f\nTotalMemory:\t%e\nAvailableMemory:\t%e\n',... - 'MultiProcessorCount:\t%d\nClockRate:\t%f\nComputeMode:\t%s\n\n'],... - g.Name,g.ComputeCapability,g.DriverVersion,g.ToolkitVersion,... - g.TotalMemory,g.AvailableMemory,g.MultiprocessorCount,... - g.ClockRateKHz,g.ComputeMode); + 'ToolkitVersion:\t%f\nTotalMemory:\t%e\nAvailableMemory:\t%e\n',... + 'MultiProcessorCount:\t%d\nClockRate:\t%f\nComputeMode:\t%s\n\n'],... + g.Name,g.ComputeCapability,g.DriverVersion,g.ToolkitVersion,... + g.TotalMemory,g.AvailableMemory,g.MultiprocessorCount,... + g.ClockRateKHz,g.ComputeMode); end fclose(fOUT); diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index 8bee4d51..8bd349f8 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -14,7 +14,7 @@ if (bh_global_do_profile) profile on; end - + % Initialize the subTomoMeta structure subTomoMeta = struct(); mapBackIter = 0; @@ -36,11 +36,11 @@ angleSgn = 1; conventionMessage = ['\nThere are four ways to use each euler convention :\n ',... - 'zyz fwd, zyz inv, -1*(zyz) fwd, -1*(zyz) inv \n', ... - 'the default is positive, but adding the addidional neg argument will produce the last two\n',... - '\n\n Assuming the zyz,zxz,zyx you selected is correct, one of these should work.\n', ... - 'PLEASE let me know if you have confirmed a convention for a particluar software and ',... - 'how you made extracted the angles and I will add a software specific flag. Thank you!\n']; + 'zyz fwd, zyz inv, -1*(zyz) fwd, -1*(zyz) inv \n', ... + 'the default is positive, but adding the addidional neg argument will produce the last two\n',... + '\n\n Assuming the zyz,zxz,zyx you selected is correct, one of these should work.\n', ... + 'PLEASE let me know if you have confirmed a convention for a particluar software and ',... + 'how you made extracted the angles and I will add a software specific flag. Thank you!\n']; switch convention case 'Protomo' @@ -66,7 +66,7 @@ otherwise if ismember(convention,eulerSet) convention = defMatSet{find(ismember(eulerSet,convention))}; - if (strcmpi(direction,'fwd')) + if (strcmpi(direction,'fwd')) direction = 'forward'; elseif ~(strcmpi(direction,'inv')) error('angular convention direction must be fwd or inv'); @@ -84,7 +84,7 @@ end end - + end end nGPUs = emc.('nGPUs'); @@ -104,7 +104,7 @@ % in the references to keep orientations from diverging. Should be > 2.25 x % your expected resolution. 1.5x to be totally uncorrelated, and another % 1.5x because aligning at ~40A in template matching will (with good data) -% often lead to and FSC of ~ 26A initially. +% often lead to and FSC of ~ 26A initially. % 40 may be too conservative, especially for low defocus tomos try @@ -124,7 +124,7 @@ % be the director convmap, holding convolution maps, model files etc. checkDir = exist('convmap'); -if checkDir ~= 7 +if checkDir ~= 7 error('Did not find directory named ') end @@ -150,18 +150,18 @@ % include the size of the tilt-series which could then be removed from the % TLT geometry. I think it important to not duplicate the information as % this could lead to bugs if one is changed and the other not. The only -% other concern is then linking each tomogram to the parent tilt-series. +% other concern is then linking each tomogram to the parent tilt-series. for iStack = 1:nStacks - - - -sprintf('recon/%s',getCoords(iStack).name) - - -[ recGeom, tiltName, nTomosPossible] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name) ); + + + + sprintf('recon/%s',getCoords(iStack).name) + + + [ recGeom, tiltName, nTomosPossible] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name) ); % Initialize - - + + subTomoMeta.('mapBackGeometry').(tiltName).('coords') = zeros(nTomosPossible,6); if (doImport) iPath = dir(sprintf('convmap/%s_*.csv',tiltName)) @@ -179,9 +179,9 @@ end subTomoMeta.('mapBackGeometry').(sprintf('%s',tiltName)).('nTomos') = nTomos; subTomoMeta.('mapBackGeometry').(sprintf('%s',tiltName)).('tomoCprRePrjSize') = 512; - + for iSt = 1:nTomos - + if (doImport) modName = strsplit(iPath(iSt).name,'.csv'); modName = strsplit(modName{1},'_'); @@ -191,23 +191,23 @@ tomoNumber = EMC_str2double(modName{end-1}); end - + subTomoMeta.('mapBackGeometry').(tiltName).('coords')(tomoNumber,:) = ... - recGeom(tomoNumber,:); + recGeom(tomoNumber,:); subTomoMeta.('mapBackGeometry').('tomoName').(... - sprintf('%s_%d',tiltName,tomoNumber)).('tiltName') = tiltName; + sprintf('%s_%d',tiltName,tomoNumber)).('tiltName') = tiltName; subTomoMeta.('mapBackGeometry').('tomoName').(... - sprintf('%s_%d',tiltName,tomoNumber)).('tomoNumber') = tomoNumber; + sprintf('%s_%d',tiltName,tomoNumber)).('tomoNumber') = tomoNumber; end - - - + + + end for iTomo = 1:nTomogramsTotal iTomo - + sprintf('convmap/%s',getPath(iTomo).name) modName = strsplit(getPath(iTomo).name,'_') if (doImport) @@ -218,67 +218,67 @@ tiltName = strjoin(modName(1:end-2),'_'); tomoNumber = EMC_str2double(modName{end-1}) end - + fileInfo{iTomo,1} = tiltName; fileInfo{iTomo,2} = sprintf('%s_%d',tiltName,tomoNumber); fileInfo{iTomo,3} = sprintf('%s_%d_bin%d',tiltName,tomoNumber,dupSampling); fileInfo{iTomo,4} = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltName,mapBackIter+1); - subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2}) = load(fileInfo{iTomo,4}); - - recCoords = importdata(sprintf('./recon/%s_recon.coords',tiltName)) - recCoords = recCoords.data - - % The reconstruction could be defined based on the aliStacks or the - % fixedStacks. the dimensions - - recGeom = [recCoords(2 + (tomoNumber-1)*6), ... % NX - recCoords(4 + (tomoNumber-1)*6) - recCoords(3 + (tomoNumber-1)*6) + 1, ... % NY - recCoords(5 + (tomoNumber-1)*6), ... % NZ - -1*recCoords(6 + (tomoNumber-1)*6), ... OX (negative shift X in imod reconstruction command); - floor((recCoords(4 + (tomoNumber-1)*6) + recCoords(3 + (tomoNumber-1)*6) - 1)/2 - (subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2})(1,21))/2),... % oY -- need the tilt series size - recCoords(7 + (tomoNumber-1)*6)]; %OZ (negative shift Z in imod reconstruction command -- but rotated during reconstruction so the -1 is implicit); - - subTomoMeta.('reconGeometry').(fileInfo{iTomo,2}) = ... - [recGeom(1:3);recGeom(4:6)]; -% try -% recGeom = load(sprintf('./recon/%s_recon.txt',fileInfo{iTomo,2})); -% subTomoMeta.('reconGeometry').(fileInfo{iTomo,2}) = ... -% [recGeom(1:3);recGeom(4:6)]; -% -% catch -% fileInfo{iTomo,2} -% error('error loading ./recon/%s_recon.txt',fileInfo{iTomo,2}); -% end + subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2}) = load(fileInfo{iTomo,4}); + + recCoords = importdata(sprintf('./recon/%s_recon.coords',tiltName)) + recCoords = recCoords.data + + % The reconstruction could be defined based on the aliStacks or the + % fixedStacks. the dimensions + + recGeom = [recCoords(2 + (tomoNumber-1)*6), ... % NX + recCoords(4 + (tomoNumber-1)*6) - recCoords(3 + (tomoNumber-1)*6) + 1, ... % NY + recCoords(5 + (tomoNumber-1)*6), ... % NZ + -1*recCoords(6 + (tomoNumber-1)*6), ... OX (negative shift X in imod reconstruction command); + floor((recCoords(4 + (tomoNumber-1)*6) + recCoords(3 + (tomoNumber-1)*6) - 1)/2 - (subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2})(1,21))/2),... % oY -- need the tilt series size + recCoords(7 + (tomoNumber-1)*6)]; %OZ (negative shift Z in imod reconstruction command -- but rotated during reconstruction so the -1 is implicit); + + subTomoMeta.('reconGeometry').(fileInfo{iTomo,2}) = ... + [recGeom(1:3);recGeom(4:6)]; + % try + % recGeom = load(sprintf('./recon/%s_recon.txt',fileInfo{iTomo,2})); + % subTomoMeta.('reconGeometry').(fileInfo{iTomo,2}) = ... + % [recGeom(1:3);recGeom(4:6)]; + % + % catch + % fileInfo{iTomo,2} + % error('error loading ./recon/%s_recon.txt',fileInfo{iTomo,2}); + % end + - % Check to make sure no out of bounds conditions were created in X Y rXrY = subTomoMeta.mapBackGeometry.(tiltName).coords; - if rXrY(tomoNumber,2) < -75 - error(['Out of bounds condition for %s yMin at %f,'... - 'please change recon.txt recon.coords'], ... - fileInfo{iTomo,2},rXrY(tomoNumber,2)) - elseif rXrY(tomoNumber,2) < 1 - subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,2) = 1; - end - yMax = subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2})(1,21); - if rXrY(tomoNumber,3) > yMax + 75 - error(['Out of bounds condition for %s yMax at %f (max %d),'... - 'please change recon.txt recon.coords'], ... - fileInfo{iTomo,2},rXrY(tomoNumber,3),yMax) - elseif rXrY(tomoNumber,3) > yMax - subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,3) = yMax; - end - - + if rXrY(tomoNumber,2) < -75 + error(['Out of bounds condition for %s yMin at %f,'... + 'please change recon.txt recon.coords'], ... + fileInfo{iTomo,2},rXrY(tomoNumber,2)) + elseif rXrY(tomoNumber,2) < 1 + subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,2) = 1; + end + yMax = subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2})(1,21); + if rXrY(tomoNumber,3) > yMax + 75 + error(['Out of bounds condition for %s yMax at %f (max %d),'... + 'please change recon.txt recon.coords'], ... + fileInfo{iTomo,2},rXrY(tomoNumber,3),yMax) + elseif rXrY(tomoNumber,3) > yMax + subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,3) = yMax; + end + + subTomoMeta.('ctfGroupSize').(fileInfo{iTomo,2}) = [nCTFgroups,0]; iX = subTomoMeta.('mapBackGeometry').(fileInfo{iTomo,1}).('coords')(tomoNumber,1); subTomoMeta.('ctfGroupSize').(fileInfo{iTomo,2})(2) = floor(iX./... - nCTFgroups); -% % % subTomoMeta.('mapExt').(fileInfo{iTomo,2}) = fileInfo{iTomo,3}; -% % % subTomoMeta.('mapPath').(fileInfo{iTomo,1}) = fileInfo{iTomo,2}; - + nCTFgroups); + % % % subTomoMeta.('mapExt').(fileInfo{iTomo,2}) = fileInfo{iTomo,3}; + % % % subTomoMeta.('mapPath').(fileInfo{iTomo,1}) = fileInfo{iTomo,2}; + end @@ -309,103 +309,103 @@ dupInTheLoop = dupSampling parfor iGPU = 1:nGPUs -% % % for iGPU = 1:nGPUs - + % % % for iGPU = 1:nGPUs + D = gpuDevice(iGPU); tomoResults = struct(); for iTomo = iterList{iGPU} - + if (doImport) mapName = fileInfo{iTomo,2}; else mapName = fileInfo{iTomo,3}; end - - % Load in the template matching geometry for the tomogram, and the model file + + % Load in the template matching geometry for the tomogram, and the model file % which may (or may not) have been edited. tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(fileInfo{iTomo,2}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(fileInfo{iTomo,2}).tiltName; coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); try tmpSearchGeom = importdata(sprintf('convmap/%s.csv',mapName)); - catch + catch error('Did not find the file convmap/%s.csv\n\nMake sure the binning in your parameter file is correct.',mapName) end - - if (doImport) - tmpCSV = tmpSearchGeom; - % TODO replace the 26 with the global for the number of lines FIXME - tmpSearchGeom = zeros([size(tmpCSV,1),26],'single'); - tmpSearchGeom(:,[11:16]) = tmpCSV; - - for iAng = 1:size(tmpSearchGeom,1) + + if (doImport) + tmpCSV = tmpSearchGeom; + % TODO replace the 26 with the global for the number of lines FIXME + tmpSearchGeom = zeros([size(tmpCSV,1),26],'single'); + tmpSearchGeom(:,[11:16]) = tmpCSV; - angleSgn - convention - direction - tmpSearchGeom(iAng,17:25) = reshape( ... - BH_defineMatrix( ... - angleSgn .* tmpSearchGeom(iAng,14:16),... - convention,direction),1,9); - - end + for iAng = 1:size(tmpSearchGeom,1) + + angleSgn + convention + direction + tmpSearchGeom(iAng,17:25) = reshape( ... + BH_defineMatrix( ... + angleSgn .* tmpSearchGeom(iAng,14:16),... + convention,direction),1,9); - % I'm assuming that the proper scaling was done, let the user know - fprintf('\nImporting coordinates, assuming to be scaled properly to match full reconstruction size\n'); - else - % [ ~, binShiftTomo ] = BH_multi_calcBinShift( coords, dupInTheLoop); - - % tmpSearchGeom(:,11:13) = tmpSearchGeom(:,11:13) + repmat(binShiftTomo,size(tmpSearchGeom,1),1); end - - + + % I'm assuming that the proper scaling was done, let the user know + fprintf('\nImporting coordinates, assuming to be scaled properly to match full reconstruction size\n'); + else + % [ ~, binShiftTomo ] = BH_multi_calcBinShift( coords, dupInTheLoop); + + % tmpSearchGeom(:,11:13) = tmpSearchGeom(:,11:13) + repmat(binShiftTomo,size(tmpSearchGeom,1),1); + end + + % Leave in for now, but check with Yunjie to remove for new import % style % New check for all -1 and then convert to Protomo if ( abs(sum(tmpSearchGeom(1,17:25)) + 9) < 1e-3 ) - fprintf('\nconverting euler angles from Protomo trf convention\n'); - for iAng = 1:size(tmpSearchGeom,1) - - tmpSearchGeom(iAng,17:25) = reshape( ... - BH_defineMatrix( ... - -1.* flip(tmpSearchGeom(iAng,14:16)),... - 'Bah','inv'),1,9); - - tmpSearchGeom(iAng,11:13) = tmpSearchGeom(iAng,11:13)+1; - end + fprintf('\nconverting euler angles from Protomo trf convention\n'); + for iAng = 1:size(tmpSearchGeom,1) + + tmpSearchGeom(iAng,17:25) = reshape( ... + BH_defineMatrix( ... + -1.* flip(tmpSearchGeom(iAng,14:16)),... + 'Bah','inv'),1,9); + + tmpSearchGeom(iAng,11:13) = tmpSearchGeom(iAng,11:13)+1; + end end if (convertEulers) - + fprintf('converting euler angles to spider/relion convention\n'); for iAng = 1:size(tmpSearchGeom,1) tmpSearchGeom(iAng,17:25) = reshape( ... - BH_defineMatrix( ... - convertEulers.* ... - tmpSearchGeom(iAng,14:16),... - 'SPIDER',direction),1,9); - + BH_defineMatrix( ... + convertEulers.* ... + tmpSearchGeom(iAng,14:16),... + 'SPIDER',direction),1,9); + tmpSearchGeom(iAng,11:13) = tmpSearchGeom(iAng,11:13)+1; end end - % Note the model coordinates are already scaled down by the sampling rate used + % Note the model coordinates are already scaled down by the sampling rate used % in the template search, which is the same sampling used here for the % comparison. - + % Putting in a try/catch for synthetic data without template matching step try tiltName = sprintf('convmap/%s_convmap.mrc',mapName); iHeader = getHeader(MRCImage(tiltName)); - % first convert the imod model file to a temporary text file + % first convert the imod model file to a temporary text file tmpFile = sprintf('tmp_%d.txt',iGPU); sprintf('model2point convmap/%s.mod %s', mapName, tmpFile) system(sprintf('model2point convmap/%s.mod %s', mapName, tmpFile)) modGeom = load(tmpFile); system(sprintf('rm %s', tmpFile)); - flgLookForPoints = 1; - - % leave IDX in main memory because it is just for reference. + flgLookForPoints = 1; + + % leave IDX in main memory because it is just for reference. sx = floor(iHeader.nX); sy = floor(iHeader.nY); sz = floor(iHeader.nZ); @@ -417,21 +417,21 @@ % Make sure nothing has gone wrong in translating the convmap to the % full size - - -% if (flgLookForPoints) && any(abs([sx,sy,sz].*dupInTheLoop - subTomoMeta.('reconGeometry').(fileInfo{iTomo,2})(1,1:3)) > 2.*dupInTheLoop) -% fprintf('convmap/%s_convmap.mrc\n',mapName); -% error('The binned (bin%d) convmap [%d %d %d] and recon size [%d %d %d] are > %f diff\n',dupInTheLoop,sx,sy,sz,subTomoMeta.('reconGeometry').(fileInfo{iTomo,2})(1,1:3),dupInTheLoop) -% end - - - % Assuming that + + + % if (flgLookForPoints) && any(abs([sx,sy,sz].*dupInTheLoop - subTomoMeta.('reconGeometry').(fileInfo{iTomo,2})(1,1:3)) > 2.*dupInTheLoop) + % fprintf('convmap/%s_convmap.mrc\n',mapName); + % error('The binned (bin%d) convmap [%d %d %d] and recon size [%d %d %d] are > %f diff\n',dupInTheLoop,sx,sy,sz,subTomoMeta.('reconGeometry').(fileInfo{iTomo,2})(1,1:3),dupInTheLoop) + % end + + + % Assuming that if (flgLookForPoints) positionMatrix = zeros(sx, sy, sz, 'single', 'gpuArray'); positionIDX = zeros(sx, sy, sz, 'uint32'); - - + + % Make a volume with ones in the position of the centers of the tomos. for iSubTomo = 1:size(tmpSearchGeom,1) @@ -439,50 +439,50 @@ subTomoOrigin = fix(tmpSearchGeom(iSubTomo,11:13)./dupInTheLoop); if any(subTomoOrigin < 1 + dupRadius) || any([sx,sy,sz] < subTomoOrigin + dupRadius) tmpSearchGeom(iSubTomo,26:26:26*emc.nPeaks) = -9999; - + else - positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = 1; - positionIDX(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... - tmpSearchGeom(iSubTomo, 4); - + positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = 1; + positionIDX(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... + tmpSearchGeom(iSubTomo, 4); + end - + end % loop building position matrix - + for iSubTomo = 1:size(modGeom,1) - + subTomoOrigin = fix(modGeom(iSubTomo,:)); if all(subTomoOrigin > 1) && all([sx,sy,sz] > subTomoOrigin) - + positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... - positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) + 1; + positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) + 1; end % sometimes a point gets moved out of bounds. - end - + end + % Convolve positionmatrix with duplicate mask. Just in case there are % rounding errors at any point use convolution to check a neighborhood of % +/- 1 pixel. - + overlapMatrix = convn(positionMatrix, gpuArray(dupMask), 'same'); - + idxList = positionIDX((overlapMatrix > 1)); size(overlapMatrix) size(positionIDX) - + tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom(ismember(tmpSearchGeom(:,4), idxList),:) sum(idxList(:)) sum(ismember(tmpSearchGeom(:,4), idxList)) else - + tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom; end - + % Fix the retained points end parResults{iGPU} = tomoResults; - - + + end % By my convention, every particle should have a unique id. @@ -494,30 +494,30 @@ tmpGeom = parResults{iGPU}.(mapName); tmpGeom(:,9:26:26*emc.nPeaks) = repmat(ceil(tmpGeom(:,11)./ ... - subTomoMeta.('ctfGroupSize').(mapName)(2)),1,emc.nPeaks); - + subTomoMeta.('ctfGroupSize').(mapName)(2)),1,emc.nPeaks); + % Sort so that CTFs can be left in main mem, and only pulled when needed and only % once per round of alignment. tmpGeom = sortrows(tmpGeom,9); - - + + for iSubTomo = 1:size(tmpGeom,1) tmpGeom(iSubTomo, 4:26:26*emc.nPeaks) = nIDX; - nIDX = nIDX +1; + nIDX = nIDX +1; end %tmpGeom(:,9) = ceil(tmpGeom(:,11)./ ... % subTomoMeta.('ctfGroupSize').(mapName)(2)); - + % Using my template matching, there should never be a tomo so close to % the edge for this to be problem, but when working with coordinates % from relion, I've noticed out of bounds conditions. Check explicitly - % here. + % here. tmpGeom( tmpGeom(:,9)> nCTFgroups, 9 ) = nCTFgroups; - subTomoMeta.('cycle000').('geometry').(mapName) = tmpGeom; + subTomoMeta.('cycle000').('geometry').(mapName) = tmpGeom; end end diff --git a/metaData/BH_geometry_Constraints.m b/metaData/BH_geometry_Constraints.m index 1abf14e4..e4186864 100644 --- a/metaData/BH_geometry_Constraints.m +++ b/metaData/BH_geometry_Constraints.m @@ -37,7 +37,7 @@ function BH_geometry_Constraints(PARAMETER_FILE, nWorkers, distCut, angCut, latt end geom = geom(keepCSV); nModFiles = length(geom); - + angCut = [0,EMC_str2double(angCut)]; distCut = EMC_str2double(distCut)./pixelSize; @@ -59,8 +59,8 @@ function BH_geometry_Constraints(PARAMETER_FILE, nWorkers, distCut, angCut, latt end for iPar = 1:nWorkers -% for iPar = 1:nWorkers - + % for iPar = 1:nWorkers + geom = geomList{iPar}; for i = iterList{iPar} @@ -73,166 +73,166 @@ function BH_geometry_Constraints(PARAMETER_FILE, nWorkers, distCut, angCut, latt reconDims = load(sprintf('recon/%s_recon.txt',reconName)); reconOrigin = reshape(floor(reconDims(1:3)./2) + 1,1,3); end - - + + nVol = size(listIN,1); keepList = zeros(nVol,1); peakList = ones(nVol,1); - + if (flgCentroid) rad_cutoff_sigma = 3; com_iter_max = 35; % First loop over and get rid of any points that are not directed % radially outward. - - % Tomogram may not be centered, either because the estimate in Z is - % off, or the particle is near the edge and the box was shifted. - % First loop over selecting likely candidates, and determine their - % center of mass, then re-loop using this. - COM = [0,0,0]; - nCOM = 0; - % Adjust for outliers to the radius. - mean_radius = 0; - mean_radius_sq = 0; - final_loop = false; - rad_cutoff = 1e6; - orig_reconOrigin = reconOrigin; - for comLoop = 1:com_iter_max - if comLoop > 1 - COM = [COM ./ nCOM] ./ pixelSize; - calc_avg = mean_radius ./ nCOM; - calc_std = sqrt(mean_radius_sq ./ nCOM - (mean_radius ./ nCOM)^2); - rad_cutoff = calc_avg + rad_cutoff_sigma*calc_std; - if norm(COM) < 2 - final_loop = true; - end -% fprintf('Adjusting the origin from recon %f,%f,%f\nTo %f,%f,%f\n',reconOrigin,COM); - reconOrigin = reconOrigin + COM; - COM = [0,0,0]; - nCOM = 0; - mean_radius = 0; - mean_radius_sq = 0; - - + + % Tomogram may not be centered, either because the estimate in Z is + % off, or the particle is near the edge and the box was shifted. + % First loop over selecting likely candidates, and determine their + % center of mass, then re-loop using this. + COM = [0,0,0]; + nCOM = 0; + % Adjust for outliers to the radius. + mean_radius = 0; + mean_radius_sq = 0; + final_loop = false; + rad_cutoff = 1e6; + orig_reconOrigin = reconOrigin; + for comLoop = 1:com_iter_max + if comLoop > 1 + COM = [COM ./ nCOM] ./ pixelSize; + calc_avg = mean_radius ./ nCOM; + calc_std = sqrt(mean_radius_sq ./ nCOM - (mean_radius ./ nCOM)^2); + rad_cutoff = calc_avg + rad_cutoff_sigma*calc_std; + if norm(COM) < 2 + final_loop = true; end - for iPt = 1:nVol - % It might not be the top peak - foundOnePeak = false; - for iPeak = 1:nPeaks - % Vector that points along the Z-axis of the particle - particleAxis = gather(reshape(listIN(iPt,[17:25] + 26*(iPeak-1)),3,3)*[0;0;1]); - % Vector that points from the recon origin to the subtomo origin - particleCoords = gather(pixelSize*(listIN(iPt,[11:13] + 26*(iPeak-1)) - reconOrigin)); - if (norm(particleCoords) > avgRadius(2) || norm(particleCoords) < avgRadius(1) || norm(particleCoords) > rad_cutoff) - break; - end - if acosd(dot(particleCoords./norm(particleCoords), particleAxis)) < 45 - if final_loop - foundOnePeak = true; - peakList(iPt) = iPeak; - keepList(iPt) = iPt; - end - COM = COM + particleCoords; - particleWeight = 1;%listIN(iPt,1 + 26*(iPeak-1)); - nCOM = nCOM + particleWeight; - mean_radius = mean_radius + particleWeight.*norm(particleCoords); - mean_radius_sq = mean_radius_sq + particleWeight.*norm(particleCoords).^2; - break; - end + % fprintf('Adjusting the origin from recon %f,%f,%f\nTo %f,%f,%f\n',reconOrigin,COM); + reconOrigin = reconOrigin + COM; + COM = [0,0,0]; + nCOM = 0; + mean_radius = 0; + mean_radius_sq = 0; + + + end + for iPt = 1:nVol + % It might not be the top peak + foundOnePeak = false; + for iPeak = 1:nPeaks + % Vector that points along the Z-axis of the particle + particleAxis = gather(reshape(listIN(iPt,[17:25] + 26*(iPeak-1)),3,3)*[0;0;1]); + % Vector that points from the recon origin to the subtomo origin + particleCoords = gather(pixelSize*(listIN(iPt,[11:13] + 26*(iPeak-1)) - reconOrigin)); + if (norm(particleCoords) > avgRadius(2) || norm(particleCoords) < avgRadius(1) || norm(particleCoords) > rad_cutoff) + break; end - - if final_loop - if ~(foundOnePeak) - listIN(iPt,26:26:26*nPeaks) = -9999 ; -% keepList(iPt) = iPt; + if acosd(dot(particleCoords./norm(particleCoords), particleAxis)) < 45 + if final_loop + foundOnePeak = true; + peakList(iPt) = iPeak; + keepList(iPt) = iPt; end + COM = COM + particleCoords; + particleWeight = 1;%listIN(iPt,1 + 26*(iPeak-1)); + nCOM = nCOM + particleWeight; + mean_radius = mean_radius + particleWeight.*norm(particleCoords); + mean_radius_sq = mean_radius_sq + particleWeight.*norm(particleCoords).^2; + break; end end + if final_loop -% % Until I can add something linke this to spike_constraint in -% % average just save the offset. Better yet, I'll fit an ellipse -% % or something. -% com_offset = fopen(sprintf('recon/%s_comOffset.txt',reconName),'w'); -% fprintf(com_offset,'%3.3f %3.3f %3.3f',reconOrigin - orig_reconOrigin); -% fclose(com_offset); - break + if ~(foundOnePeak) + listIN(iPt,26:26:26*nPeaks) = -9999 ; + % keepList(iPt) = iPt; + end end - end % loop to deterimne center of mass - + end + if final_loop + % % Until I can add something linke this to spike_constraint in + % % average just save the offset. Better yet, I'll fit an ellipse + % % or something. + % com_offset = fopen(sprintf('recon/%s_comOffset.txt',reconName),'w'); + % fprintf(com_offset,'%3.3f %3.3f %3.3f',reconOrigin - orig_reconOrigin); + % fclose(com_offset); + break + end + end % loop to deterimne center of mass + else - + for iPt = 1:nVol - - if flgCentroid + + if flgCentroid thisPeak = peakList(iPt); else thisPeak = 1; end - + for iPeak = thisPeak if ( listIN(iPt,26 + 26*(iPeak-1)) ~= -9999 ) c1 = sub2ind(size(listIN),[1:size(listIN,1)]',11 + 26.*(peakList-1)); c2 = sub2ind(size(listIN),[1:size(listIN,1)]',12 + 26.*(peakList-1)); c3 = sub2ind(size(listIN),[1:size(listIN,1)]',13 + 26.*(peakList-1)); - + distVect = sqrt( (listIN(c1)-listIN(iPt,11 + 26*(iPeak-1))).^2 + ... - (listIN(c2)-listIN(iPt,12 + 26*(iPeak-1))).^2 + ... - (listIN(c3)-listIN(iPt,13 + 26*(iPeak-1))).^2 ); - + (listIN(c2)-listIN(iPt,12 + 26*(iPeak-1))).^2 + ... + (listIN(c3)-listIN(iPt,13 + 26*(iPeak-1))).^2 ); + closeVect = find(distVect < distCut); if (length(closeVect)>=nNeighbors) - nAngClose = 0; - particleAxis = reshape(listIN(iPt,[17:25] + 26*(iPeak-1)),3,3)*[0;0;1]; - - for iAng = 1:length(closeVect) - iAxis = reshape(listIN(closeVect(iAng),17:25),3,3)*[0;0;1]; - iAngDiff = dot(particleAxis,iAxis); - if (abs(iAngDiff) > 1) - iAngDiff = fix(iAngDiff); - end - if abs(iAngDiff) < abs(angCut(2)) && abs(iAngDiff) > abs(angCut(1)) - nAngClose = nAngClose + 1; - end - end + nAngClose = 0; + particleAxis = reshape(listIN(iPt,[17:25] + 26*(iPeak-1)),3,3)*[0;0;1]; + + for iAng = 1:length(closeVect) + iAxis = reshape(listIN(closeVect(iAng),17:25),3,3)*[0;0;1]; + iAngDiff = dot(particleAxis,iAxis); + if (abs(iAngDiff) > 1) + iAngDiff = fix(iAngDiff); + end + if abs(iAngDiff) < abs(angCut(2)) && abs(iAngDiff) > abs(angCut(1)) + nAngClose = nAngClose + 1; + end + end if nAngClose > nNeighbors-1 keepList(iPt) = iPt; else keepList(iPt) = 0; end end - + end end end - + end - - listIN = listIN(find(keepList),:); - [~,fileName,~] = fileparts(geom{i}); - - if isempty(listIN) - fprintf('NO subtomos retained! for %s\n',fileName); - else - csvOUT = fopen(sprintf('convmap/%s.txt',fileName),'w'); - h = hist(listIN(:,15)) - for iLine = 1:size(listIN,1) - - fprintf(csvOUT,'%4.4f %4.4f %4.4f\n',listIN(iLine,[11:13])./listIN(iLine,2)); - end - fclose(csvOUT); - - system(sprintf('point2model -number 1 -circle 3 -sphere 3 -scat -color 80,191,255 convmap/%s.txt convmap/%s.mod',fileName,fileName)); - nTotal{iPar} = nTotal{iPar} + sum(keepList~=0); + + listIN = listIN(find(keepList),:); + [~,fileName,~] = fileparts(geom{i}); + + if isempty(listIN) + fprintf('NO subtomos retained! for %s\n',fileName); + else + csvOUT = fopen(sprintf('convmap/%s.txt',fileName),'w'); + h = hist(listIN(:,15)) + for iLine = 1:size(listIN,1) + + fprintf(csvOUT,'%4.4f %4.4f %4.4f\n',listIN(iLine,[11:13])./listIN(iLine,2)); end + fclose(csvOUT); + + system(sprintf('point2model -number 1 -circle 3 -sphere 3 -scat -color 80,191,255 convmap/%s.txt convmap/%s.mod',fileName,fileName)); + nTotal{iPar} = nTotal{iPar} + sum(keepList~=0); + end end -% - + % + end all_total = 0; for iWorker = 1:nWorkers all_total = all_total + nTotal{iWorker}; end - fprintf('Total points found %d\n', all_total); - +fprintf('Total points found %d\n', all_total); + end % end of function diff --git a/metaData/BH_mergeClassGeometry.m b/metaData/BH_mergeClassGeometry.m index 8c2c80a1..ac9f15c1 100755 --- a/metaData/BH_mergeClassGeometry.m +++ b/metaData/BH_mergeClassGeometry.m @@ -15,19 +15,19 @@ nTomograms = length(tomoList1); -% Loop through counting particles available. +% Loop through counting particles available. for iTomo = 1:nTomograms % Read in the geometry for each tomogram, update column eight and count the % number included. - positionList1 = geometry1.(tomoList1{iTomo}); - positionList2 = geometry2.(tomoList2{iTomo}); - - positionList1((positionList2(:,7)==2),:) = positionList2((positionList2(:,7)==2),:); - - geometry1.(tomoList1{iTomo}) = positionList1; - - - + positionList1 = geometry1.(tomoList1{iTomo}); + positionList2 = geometry2.(tomoList2{iTomo}); + + positionList1((positionList2(:,7)==2),:) = positionList2((positionList2(:,7)==2),:); + + geometry1.(tomoList1{iTomo}) = positionList1; + + + end GEOM_OUT = geometry1; diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 93b8aaac..83bc0362 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -5,7 +5,7 @@ fileID = fopen(PARAMETER_FILE,'r'); p = textscan(fileID, '%s', 'CommentStyle',{'%'},'Delimiter','\n', ... - 'TreatAsEmpty',{' '}); + 'TreatAsEmpty',{' '}); nParam = 1; p2 = cell(1,1); @@ -21,17 +21,17 @@ emc = struct(); % Check that all paramters are name: value pairs stringValues = {'subTomoMeta'; ... - 'Ali_mType';'Cls_mType';'Cls_mType';'Raw_mType';'Fsc_mType'; ... - 'Pca_distMeasure';'Kms_mType';'flgPrecision';'Tmp_xcfScale';... - 'fastScratchDisk';'Tmp_eraseMaskType';'startingDirection';'Peak_mType';'symmetry'}; + 'Ali_mType';'Cls_mType';'Cls_mType';'Raw_mType';'Fsc_mType'; ... + 'Pca_distMeasure';'Kms_mType';'flgPrecision';'Tmp_xcfScale';... + 'fastScratchDisk';'Tmp_eraseMaskType';'startingDirection';'Peak_mType';'symmetry'}; for i = 1:size(p2,1) pNameVal = strsplit(p2{i,1},'='); if length(pNameVal) == 1 - error('Could not split Name=Value pair for\n\t %s',char(pNameVal)) + error('Could not split Name=Value pair for\n\t %s',char(pNameVal)) elseif length(pNameVal) > 2 error('To many colons in\n\t %s',char(pNameVal)) else - + if any(strcmp(stringValues, pNameVal{1})) emc.(pNameVal{1}) = pNameVal{2}; else @@ -51,7 +51,7 @@ % Used when cutting out subtomos for further processing. Adds extra padding to anticipate shifts etc. % This has not been well tested -% When used in average3d, this value is stored in the subTomoMeta. +% When used in average3d, this value is stored in the subTomoMeta. if ~isfield(emc, 'CUTPADDING') emc.('CUTPADDING') = 20; end diff --git a/metaData/BH_randomSubset.m b/metaData/BH_randomSubset.m index 56c0bd31..8b3f2c88 100755 --- a/metaData/BH_randomSubset.m +++ b/metaData/BH_randomSubset.m @@ -1,7 +1,7 @@ function [ GEOMETRY_UPDATED, nTOTAL, nSUBSET ] = ... - BH_randomSubset( GEOMETRY, PCAorSNR, nPARTICLES, halfSET ) + BH_randomSubset( GEOMETRY, PCAorSNR, nPARTICLES, halfSET ) %Count the particles to be used in PCA, optionally making a random subset. -% +% % % Input variables: % @@ -15,7 +15,7 @@ % Output variables: % % GEOMETRY_UPDATED = updated geometry -% +% % nTOTAL = total number to be used for classification % % nSUBSET = total number to be used for decomposition @@ -49,7 +49,7 @@ % Get the number of tomograms to process. tomoList = fieldnames(GEOMETRY); geometry = GEOMETRY; clear GEOMETRY - catch + catch error('Could not access the fieldnames in the struct geometry.') end else @@ -63,24 +63,24 @@ nTomograms = length(tomoList); nIncluded = 0; includedIDX = []; -% Loop through counting particles available. +% Loop through counting particles available. for iTomo = 1:nTomograms % Read in the geometry for each tomogram, update column eight and count the % number included. - positionList = geometry.(tomoList{iTomo}); - % Count the max number available, use is member for STD alignment - includedPositions = ( positionList(:,26) ~= -9999 & ismember(positionList(:,7),halfSET) ); - nIncluded = nIncluded + sum(includedPositions); - % Save a record of unique particle idx for randomization - includedIDX = [includedIDX; positionList(includedPositions,4)]; - % Update the position lists if all available are to be used. - if ( nPARTICLES == -1 && flgPCA) - positionList(includedPositions, 8) = 1; - positionList(~includedPositions, 8) = 0; - geometry.(tomoList{iTomo}) = positionList; - nSUBSET = nIncluded; - end - + positionList = geometry.(tomoList{iTomo}); + % Count the max number available, use is member for STD alignment + includedPositions = ( positionList(:,26) ~= -9999 & ismember(positionList(:,7),halfSET) ); + nIncluded = nIncluded + sum(includedPositions); + % Save a record of unique particle idx for randomization + includedIDX = [includedIDX; positionList(includedPositions,4)]; + % Update the position lists if all available are to be used. + if ( nPARTICLES == -1 && flgPCA) + positionList(includedPositions, 8) = 1; + positionList(~includedPositions, 8) = 0; + geometry.(tomoList{iTomo}) = positionList; + nSUBSET = nIncluded; + end + end nTOTAL = nIncluded @@ -89,10 +89,10 @@ if ( nIncluded >= nPARTICLES ) fprintf('%d particles are being used out of %d available\n',... - nPARTICLES, nIncluded); + nPARTICLES, nIncluded); else error('%d particles requested, but only %d are available', ... - nPARTICLES,nIncluded) + nPARTICLES,nIncluded) end rng('shuffle'); @@ -101,8 +101,8 @@ if nTOTAL < 1500 error('For now assuming atlease 1500 total particles available.'); end - - + + snrIDX = zeros(numel(includedIDX),2,'uint16'); snrIDX(:,1) = uint16(includedIDX); % 4 replicates at 4 16 36 64 100 144 particles each @@ -124,43 +124,43 @@ includedIDX = includedIDX(remainingSet); end end - - + + for iTomo = 1:nTomograms - + positionList = geometry.(tomoList{iTomo}); iSnrSet = ismember(snrIDX(:,1),positionList(:,4)); iSnrSet = snrIDX(iSnrSet,:); - + for iParticle = 1:size(iSnrSet,1) iParticleIDX = find(positionList(:,4) == iSnrSet(iParticle,1),1,'first'); positionList(iParticleIDX,10) = iSnrSet(iParticle,2); end - + geometry.(tomoList{iTomo}) = positionList; - + end else - + randomSubset = datasample(includedIDX, nPARTICLES, 'Replace', false); nSUBSET = numel(randomSubset); randomSubset = reshape(randomSubset, nSUBSET, 1); - + for iTomo = 1:nTomograms - + positionList = geometry.(tomoList{iTomo}); positionList(:,8) = single(ismember(positionList(:,4), randomSubset)); geometry.(tomoList{iTomo}) = positionList; - + end end end - -GEOMETRY_UPDATED = geometry; + +GEOMETRY_UPDATED = geometry; end % end of pcaRandomize diff --git a/metaData/BH_recordAngularSampling.m b/metaData/BH_recordAngularSampling.m index 65aa4628..bd5bbf8b 100755 --- a/metaData/BH_recordAngularSampling.m +++ b/metaData/BH_recordAngularSampling.m @@ -9,38 +9,38 @@ nANG = 1; for iAngle = 1:size(angleStep,1) - + theta = angleStep(iAngle,1); -% % % thetaInc = angleStep(iAngle,4); + % % % thetaInc = angleStep(iAngle,4); % Calculate the increment in phi so that the azimuthal sampling is % consistent and equal to the out of plane increment. - + phiInc = angleStep(iAngle,3); - - + + % To prevent only searching the same increments each time in a limited % grid search, radomly offset the azimuthal angle by a random number % between 0 and 1/2 the azimuthal increment. - + azimuthalRandomizer = (rand(1)-0.5)*phiInc; - + for iAzimuth = 0:angleStep(iAngle,2) phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); - - for iInPlane = inPlaneSearch - psi = iInPlane; -% % % psiInc = angleStep(iAngle,5); - - try - anglesSampled(nANG,:) = [phi,theta,psi -phi]; - nANG = nANG + 1; - catch + + for iInPlane = inPlaneSearch + psi = iInPlane; + % % % psiInc = angleStep(iAngle,5); + + try + anglesSampled(nANG,:) = [phi,theta,psi -phi]; + nANG = nANG + 1; + catch + end end - end end end -subTomoMeta.(cycleNumber).('anglesSampled') = anglesSampled; - +subTomoMeta.(cycleNumber).('anglesSampled') = anglesSampled; + end diff --git a/metaData/BH_refAlignmentsApply.m b/metaData/BH_refAlignmentsApply.m index acb6b794..4d448359 100755 --- a/metaData/BH_refAlignmentsApply.m +++ b/metaData/BH_refAlignmentsApply.m @@ -1,18 +1,18 @@ function [UPDATED_GEOMETRY] = ... - BH_refAlignmentsApply( INPUT_GEOMETRY,BEST_ANGLES,... - SAMPLING, ... - REF_VECTOR,REF_GROUP,flgGold) + BH_refAlignmentsApply( INPUT_GEOMETRY,BEST_ANGLES,... + SAMPLING, ... + REF_VECTOR,REF_GROUP,flgGold) %Apply class alignments to the full set of subTomograms -% +% % % Input variables % -% ALIGN_GEOM = string with mat file that has the geometry output from +% ALIGN_GEOM = string with mat file that has the geometry output from % alignment. % -% +% % Output variables = none, the clustering geometry updated - eventually this -% +% % % % @@ -45,43 +45,43 @@ for iTomo = 1:nTomograms - positionList = inputGeometry.(tomoList{iTomo}); - newAlignment = alignmentGeometry; - - - includeList = find(ismember(positionList(:,26), refVector) & ... - ismember(positionList(:,7), flgGold)); - removeList = ~ismember(positionList(:,26), refVector) & ... - ismember(positionList(:,7), flgGold); - positionList(removeList,26) = -9999; - + positionList = inputGeometry.(tomoList{iTomo}); + newAlignment = alignmentGeometry; + + + includeList = find(ismember(positionList(:,26), refVector) & ... + ismember(positionList(:,7), flgGold)); + removeList = ~ismember(positionList(:,26), refVector) & ... + ismember(positionList(:,7), flgGold); + positionList(removeList,26) = -9999; + + + for iParticle = includeList' + % assuming all classes are sequential, only discarded between cycles. + class = positionList(iParticle,26); + pIndex = find(newAlignment(:,2) == class); %%% 1-->2 - for iParticle = includeList' - % assuming all classes are sequential, only discarded between cycles. - class = positionList(iParticle,26); - pIndex = find(newAlignment(:,2) == class); %%% 1-->2 - - if ~isempty(pIndex) - - classRot = BH_defineMatrix(newAlignment(pIndex,3:5),'Bah','inv'); - - oldRot = reshape(positionList(iParticle,17:25),3,3); - newRot = gather(reshape( oldRot * classRot, 1,9)); - positionList(iParticle,17:25) = newRot; - - shifts = gather((oldRot*classRot )*(1.*newAlignment(pIndex,8:10).*SAMPLING)')'; -% shifts = gather(positionList(iParticle,14:16) - newAlignment(pIndex,8:10).*SAMPLING); - % replace the classIDX with the groupIDX - positionList(iParticle,11:13) = gather(positionList(iParticle,11:13)) + ... - shifts; - find(refVector == class); - positionList(iParticle,26) = refGroup(find(refVector == class)); - - - - inputGeometry.(tomoList{iTomo}) = positionList; - end + if ~isempty(pIndex) + + classRot = BH_defineMatrix(newAlignment(pIndex,3:5),'Bah','inv'); + + oldRot = reshape(positionList(iParticle,17:25),3,3); + newRot = gather(reshape( oldRot * classRot, 1,9)); + positionList(iParticle,17:25) = newRot; + + shifts = gather((oldRot*classRot )*(1.*newAlignment(pIndex,8:10).*SAMPLING)')'; + % shifts = gather(positionList(iParticle,14:16) - newAlignment(pIndex,8:10).*SAMPLING); + % replace the classIDX with the groupIDX + positionList(iParticle,11:13) = gather(positionList(iParticle,11:13)) + ... + shifts; + find(refVector == class); + positionList(iParticle,26) = refGroup(find(refVector == class)); + + + + inputGeometry.(tomoList{iTomo}) = positionList; end + end end diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index 3a7114c2..f93096d3 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -5,21 +5,21 @@ % % Goals & Limitations: % -% +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % TODO % -% +% %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % radius +/- for duplicate tolerance. Even though you may not expect any % particles w/in a radius equal to that of the particle, this would require % a much more expensive calculation. It is likely that any particle that -% comes within an "event horizon" will end up at a nearly identical origin +% comes within an "event horizon" will end up at a nearly identical origin % volume; a smaller radius is probably okay -CYCLE = EMC_str2double(CYCLE); +CYCLE = EMC_str2double(CYCLE); cycleNumber = sprintf('cycle%0.3u', CYCLE); emc = BH_parseParameterFile(PARAMETER_FILE); @@ -43,10 +43,10 @@ % Backup the current geometry - system(sprintf('cp %s.mat preDupRemoval_%s.mat',emc.('subTomoMeta'),emc.('subTomoMeta'))); - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - geometry = subTomoMeta.(cycleNumber).RawAlign; - masterTM = subTomoMeta; clear subTomoMeta +system(sprintf('cp %s.mat preDupRemoval_%s.mat',emc.('subTomoMeta'),emc.('subTomoMeta'))); +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); +geometry = subTomoMeta.(cycleNumber).RawAlign; +masterTM = subTomoMeta; clear subTomoMeta % Get the number of tomograms to process. @@ -59,7 +59,7 @@ % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); includeList = find(positionList(:,26) ~= -9999); - + nTotal = nTotal + length(includeList); tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; @@ -67,31 +67,31 @@ tomoName = sprintf('%s_%d',tiltName,tomoNumber); recGeom = masterTM.reconGeometry.(tomoName); -% iHeader = getHeader(MRCImage(tomoName)); + % iHeader = getHeader(MRCImage(tomoName)); clear postionMatrix positionIDX % leave IDX in main memory because it is just for reference. -% sx = floor(iHeader.nX ./ dupSampling); -% sy = floor(iHeader.nY ./ dupSampling); -% sz = floor(iHeader.nZ ./ dupSampling); - sx = floor(recGeom(1,1)./dupSampling); - sy = floor(recGeom(1,2)./dupSampling); - sz = floor(recGeom(1,3)./dupSampling); + % sx = floor(iHeader.nX ./ dupSampling); + % sy = floor(iHeader.nY ./ dupSampling); + % sz = floor(iHeader.nZ ./ dupSampling); + sx = floor(recGeom(1,1)./dupSampling); + sy = floor(recGeom(1,2)./dupSampling); + sz = floor(recGeom(1,3)./dupSampling); positionMatrix = zeros([sx,sy,sz],'single','gpuArray'); positionIDX = zeros([sx,sy,sz],'single'); % Make a volume with ones in the position of the centers of the tomos. for iSubTomo = includeList' - + subTomoOrigin = round(positionList(iSubTomo,11:13)./dupSampling); if any(subTomoOrigin < 1 + dupRadius) || any([sx,sy,sz] < subTomoOrigin + dupRadius) positionList(iSubTomo,26) = -9999; else - positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = 1; - positionIDX(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... - positionList(iSubTomo, 4); + positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = 1; + positionIDX(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... + positionList(iSubTomo, 4); end - + end % loop building position matrix % Convolve positionmatrix with duplicate mask. The numbers in the @@ -111,31 +111,31 @@ % from duplicate list by setting value in duplicate list to -1, which % is ignored. - + [i,j,k] = ind2sub([sx,sy,sz], iDup); % Check that the duplicate hasn't already been evaluated try - if positionList((positionList(:,4) == positionIDX(iDup)),3) ~= -9999 - dupWindow = positionIDX( i - dupRadius : i + dupRadius, ... - j - dupRadius : j + dupRadius, ... - k - dupRadius : k + dupRadius ) ; - % From window, select only real particle ids - idxList = dupWindow(dupWindow ~= 0); - % Logical translating particle ids to postions in geometry file - posList = ismember(positionList(:,4), idxList); - % Replace ones in logical with CCC from previous raw Alignment - cccList = max(positionList(:,1:26:26*emc.nPeaks),[],2).*posList; - - [~ , maxCCCcoord] = max(cccList); - % set highest CCC to zero, so the remaining are all inferior - % duplicates, set these to ignore class, and also keep a record at - % column 3 - posList(maxCCCcoord) = 0; - nRemoved = nRemoved + sum(posList); - positionList(posList,26:26:26*emc.nPeaks) = -9999; - positionList(posList,3:26:26*emc.nPeaks) = -9999; - - end + if positionList((positionList(:,4) == positionIDX(iDup)),3) ~= -9999 + dupWindow = positionIDX( i - dupRadius : i + dupRadius, ... + j - dupRadius : j + dupRadius, ... + k - dupRadius : k + dupRadius ) ; + % From window, select only real particle ids + idxList = dupWindow(dupWindow ~= 0); + % Logical translating particle ids to postions in geometry file + posList = ismember(positionList(:,4), idxList); + % Replace ones in logical with CCC from previous raw Alignment + cccList = max(positionList(:,1:26:26*emc.nPeaks),[],2).*posList; + + [~ , maxCCCcoord] = max(cccList); + % set highest CCC to zero, so the remaining are all inferior + % duplicates, set these to ignore class, and also keep a record at + % column 3 + posList(maxCCCcoord) = 0; + nRemoved = nRemoved + sum(posList); + positionList(posList,26:26:26*emc.nPeaks) = -9999; + positionList(posList,3:26:26*emc.nPeaks) = -9999; + + end catch iDup end @@ -143,7 +143,7 @@ end % loop over duplicates - + @@ -164,4 +164,4 @@ save(emc.('subTomoMeta'), 'subTomoMeta'); end - + diff --git a/metaData/BH_returnIncludedTilts.m b/metaData/BH_returnIncludedTilts.m index 0585330b..831b9565 100644 --- a/metaData/BH_returnIncludedTilts.m +++ b/metaData/BH_returnIncludedTilts.m @@ -2,18 +2,18 @@ %UNTITLED Summary of this function goes here % Detailed explanation goes here - STACK_LIST_tmp = fieldnames(mapBackGeom); - STACK_LIST_tmp = STACK_LIST_tmp(~ismember(STACK_LIST_tmp,{'tomoName','viewGroups'})); - tiltList = cell(length(STACK_LIST_tmp),1); - - nTilts = 0; - for iStack = 1:length(STACK_LIST_tmp) - if mapBackGeom.(STACK_LIST_tmp{iStack}).nTomos - nTilts = nTilts + 1; - tiltList{nTilts} = STACK_LIST_tmp{iStack}; - end +STACK_LIST_tmp = fieldnames(mapBackGeom); +STACK_LIST_tmp = STACK_LIST_tmp(~ismember(STACK_LIST_tmp,{'tomoName','viewGroups'})); +tiltList = cell(length(STACK_LIST_tmp),1); + +nTilts = 0; +for iStack = 1:length(STACK_LIST_tmp) + if mapBackGeom.(STACK_LIST_tmp{iStack}).nTomos + nTilts = nTilts + 1; + tiltList{nTilts} = STACK_LIST_tmp{iStack}; end - - tiltList = tiltList(1:nTilts); +end + +tiltList = tiltList(1:nTilts); end diff --git a/metaData/BH_skipClassAlignment.m b/metaData/BH_skipClassAlignment.m index 75c4f20b..00d5f6d6 100755 --- a/metaData/BH_skipClassAlignment.m +++ b/metaData/BH_skipClassAlignment.m @@ -41,31 +41,31 @@ if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - + if (flgMultiRefAlignment && ~flgClassify) subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('Avg_geometry'); - + subTomoMeta.(cycleNumber).('Avg_geometry'); + elseif (flgMultiRefAlignment && flgClassify) - subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('ClusterClsGeom'); + subTomoMeta.(cycleNumber).('RawAlign') = ... + subTomoMeta.(cycleNumber).('ClusterClsGeom'); else - - try - subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('ClusterClsGeom'); + + try + subTomoMeta.(cycleNumber).('RawAlign') = ... + subTomoMeta.(cycleNumber).('ClusterClsGeom'); catch - subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('ClusterRefGeom'); + subTomoMeta.(cycleNumber).('RawAlign') = ... + subTomoMeta.(cycleNumber).('ClusterRefGeom'); end - + end else error(['STAGEofALIGNMENT to skip may be RawAlignment'],... - ['the former requires the latter to exist.\n']); + ['the former requires the latter to exist.\n']); end -save(emc.('subTomoMeta'), 'subTomoMeta'); +save(emc.('subTomoMeta'), 'subTomoMeta'); end diff --git a/metaData/BH_trimImodLocal.m b/metaData/BH_trimImodLocal.m index 4f82c043..366cc565 100644 --- a/metaData/BH_trimImodLocal.m +++ b/metaData/BH_trimImodLocal.m @@ -3,17 +3,17 @@ % Detailed explanation goes here % From the inputs - % Get a filename for the .local .rawtlt - % Get an array for tilts to remove +% Get a filename for the .local .rawtlt +% Get an array for tilts to remove % Read count of starting tilt numbers using len() -% Use text scan to get one array of all alignments inlcuding header - % f = fopen('tilt1.local_patchTracking','r'); - % p = textscan(f,'%f','TreatAsEmpty',{' '}, 'Delimiter','\n') +% Use text scan to get one array of all alignments inlcuding header +% f = fopen('tilt1.local_patchTracking','r'); +% p = textscan(f,'%f','TreatAsEmpty',{' '}, 'Delimiter','\n') % First nine entries are header - %header = p{1}(1:9) - % Total number of blocks is header1 * header2 +%header = p{1}(1:9) +% Total number of blocks is header1 * header2 % 7 and 8 in the header are booleans, if xtilt and zfactor diff --git a/metaData/EMC_parpool.m b/metaData/EMC_parpool.m index 747e38db..3d91d026 100644 --- a/metaData/EMC_parpool.m +++ b/metaData/EMC_parpool.m @@ -13,8 +13,8 @@ error('\The variable EMC_CACHE_ROOT %s\n',EMC_ROOT); end end - - + + [~,emc_rand_name,~] = fileparts( local_cache_root ); @@ -26,7 +26,7 @@ profile_does_not_exist = false; end end - + % if the profile doesn't exist, create it if (profile_does_not_exist) emc_parcluster = parcluster(parallel.defaultClusterProfile); diff --git a/metaData/EMC_str2double.m b/metaData/EMC_str2double.m index f2a69478..97a7d30f 100644 --- a/metaData/EMC_str2double.m +++ b/metaData/EMC_str2double.m @@ -1,13 +1,13 @@ function [ output_double ] = EMC_str2double( input_str ) - %Make a local copy of the default cluster, and modify the job storage - %location - - output_double = str2double(input_str); - if (isnan(output_double)) - output_double = str2num(input_str); - if (isempty(output_double)) - err_msg = sprintf('EMC_str2double: input string is not a number!\nReceived: %s', input_str); - error(err_msg); - end - end +%Make a local copy of the default cluster, and modify the job storage +%location + +output_double = str2double(input_str); +if (isnan(output_double)) + output_double = str2num(input_str); + if (isempty(output_double)) + err_msg = sprintf('EMC_str2double: input string is not a number!\nReceived: %s', input_str); + error(err_msg); + end +end end \ No newline at end of file diff --git a/metaData/SAVE_IMG.m b/metaData/SAVE_IMG.m index 4f9d0e15..1765c652 100644 --- a/metaData/SAVE_IMG.m +++ b/metaData/SAVE_IMG.m @@ -1,5 +1,5 @@ function [ ] = SAVE_IMG( vol, varargin) -%Overloaded MRCImage method SAVE_IMG +%Overloaded MRCImage method SAVE_IMG % This exists so that the call SAVE_IMG(MRCImage( vol)) can be avoided % (using the MRCImage.SAVE_IMG). This also permits passing in a gpuArray % which in turn allows much faster calcs on the stats. @@ -8,37 +8,37 @@ % too many FIDs are left open. This causes system instability. This % function prevents that. - imgMin = gather(min(vol(:))); - imgMax = gather(max(vol(:))); - imgMean = gather(mean(vol(:))); - imgRMS = gather(rms(vol(:))); - - mRCImage = MRCImage(gather(vol)); - - mRCImage.header.minDensity = imgMin; - mRCImage.header.maxDensity = imgMax; - - mRCImage.header.meanDensity = imgMean; - mRCImage.header.densityRMS = imgRMS; -% mRCImage.writeHeader(mRCImage); Changed SAVE_IMG to write out the header no matter what. - - switch length(varargin) - case 0 - SAVE_IMG(mRCImage); - case 1 - SAVE_IMG(mRCImage, varargin{1}); - case 2 - SAVE_IMG(mRCImage, varargin{1},varargin{2}); - case 3 - SAVE_IMG(mRCImage, varargin{1},varargin{2},varargin{3}); - case 4 - SAVE_IMG(mRCImage, varargin{1},varargin{2},varargin{3},varargin{4}); - case 5 - SAVE_IMG(mRCImage, varargin{1},varargin{2},varargin{3},varargin{4},varargin{5}); - otherwise - error('More than 5 additional args in SAVE_IMG'); - end - - +imgMin = gather(min(vol(:))); +imgMax = gather(max(vol(:))); +imgMean = gather(mean(vol(:))); +imgRMS = gather(rms(vol(:))); + +mRCImage = MRCImage(gather(vol)); + +mRCImage.header.minDensity = imgMin; +mRCImage.header.maxDensity = imgMax; + +mRCImage.header.meanDensity = imgMean; +mRCImage.header.densityRMS = imgRMS; +% mRCImage.writeHeader(mRCImage); Changed SAVE_IMG to write out the header no matter what. + +switch length(varargin) + case 0 + SAVE_IMG(mRCImage); + case 1 + SAVE_IMG(mRCImage, varargin{1}); + case 2 + SAVE_IMG(mRCImage, varargin{1},varargin{2}); + case 3 + SAVE_IMG(mRCImage, varargin{1},varargin{2},varargin{3}); + case 4 + SAVE_IMG(mRCImage, varargin{1},varargin{2},varargin{3},varargin{4}); + case 5 + SAVE_IMG(mRCImage, varargin{1},varargin{2},varargin{3},varargin{4},varargin{5}); + otherwise + error('More than 5 additional args in SAVE_IMG'); +end + + end diff --git a/metaData/test_tm.py b/metaData/test_tm.py deleted file mode 100644 index 44f550e6..00000000 --- a/metaData/test_tm.py +++ /dev/null @@ -1,17 +0,0 @@ -import sys -from EMC_parseParameterFile import EMC_parseParameterFile - -if __name__ == '__main__': - # Check that the script was called with the correct number of arguments - if len(sys.argv) != 2: - print(f"Usage: {sys.argv[0]} PARAMETER_FILE") - sys.exit(1) - - # Get the parameter file name from the command-line argument - PARAMETER_FILE = sys.argv[1] - - # Call EMC_parseParameterFile with the parameter file name - pStruct = EMC_parseParameterFile(PARAMETER_FILE) - - # Print the resulting parameter structure - print(pStruct) \ No newline at end of file diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index b38298f6..f1dfa57e 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -1,6 +1,6 @@ function [ ] = BH_clusterPub(PARAMETER_FILE, CYCLE) %Kmeans based classification -% +% % % Input Variables: % @@ -75,7 +75,7 @@ nFeatures = size(featureVector{1,1}); if (nFeatures(1) ~= nRows) - error('There should be a set of indices for each pcaScaleSpace, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') + error('There should be a set of indices for each pcaScaleSpace, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') end clusterVector= emc.('Pca_clusters'); @@ -112,7 +112,7 @@ end -try +try coverSteps = emc.('Pca_som_coverSteps'); catch coverSteps = 100; @@ -140,7 +140,7 @@ geom_name='' if (flgMultiRefAlignment ) - geom_name='ClusterClsGeom'; + geom_name='ClusterClsGeom'; else geom_name='Avg_geometry'; end @@ -161,23 +161,23 @@ else halfSet = 'STD'; randSet = [1,2]; - end + end coeffMatrix = sprintf('%s_%s_%s_pcaFull.mat',cycleNumber,emc.('subTomoMeta'),halfSet); outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); % Get the number of tomograms to process. tomoList = fieldnames(geometry_clean); nTomograms = length(tomoList); - - + + kAlgorithm = 'kMeans'; - % kAlgorithm = 'neuralNetwork' - + % kAlgorithm = 'neuralNetwork' + kDist = sprintf('%s', kDIST) - + switch kDist case 'sqeuclidean' - kDistMeasure = 'sqeuclidean' + kDistMeasure = 'sqeuclidean' case 'cityblock' kDistMeasure = 'cityblock' case 'cosine' @@ -191,17 +191,17 @@ kDistMeasure = 'neural' kAlgorithm = 'neuralNetwork' fprintf('Input params for neural network are %d %d %s\n', ... - coverSteps, initNeighbor, topologyFcn); + coverSteps, initNeighbor, topologyFcn); otherwise kDistMeasure = 'sqeuclidean' fprintf(['\nDefaulting to sqeuclidean b/c %s was not recognized'] ... - , kDist); + , kDist); end - + kReplicates = kREP; - - %kDistMeasure = 'euclidean' - + + %kDistMeasure = 'euclidean' + try oldPca = load(coeffMatrix); coeffsUNTRIMMED = oldPca.coeffs @@ -213,7 +213,7 @@ end clear oldPca; - catch + catch error('trouble loading the previous pcs mat file.') end @@ -224,17 +224,17 @@ pause(3) EMC_parpool(nCores); end - + %%% experimental part of pcaMS nScaleSpace = size(featureVector{iGold},1) featureVector{1} nFeatures = zeros(1,nScaleSpace) -% if length(relativeScale) ~= nScaleSpace -% error('relativeScale has %d elements for %d scaleSpaces', ... -% length(relativeScale), nScaleSpace); -% end -% + % if length(relativeScale) ~= nScaleSpace + % error('relativeScale has %d elements for %d scaleSpaces', ... + % length(relativeScale), nScaleSpace); + % end + % if isa(coeffsUNTRIMMED, 'cell') [nI,nJ] = size(coeffsUNTRIMMED{1}); for iScale = 1:nScaleSpace @@ -245,26 +245,26 @@ for iScale = 1:nScaleSpace fV = featureVector{iGold}(iScale,:); fV = sort(fV(fV~=0)) - + coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) = ... - double(coeffsUNTRIMMED{iScale}(ismember(1:nI,fV),:)); - - % normalizing the variance of the rows gives equal weight to each eigenvector which - % is not reasonable as they are by their nature scaled by the amount of - % variance explained across a given dimension. -% % % coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) = ... -% % % coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) ./ ... -% % % repmat(rms(coeffMat(1+nAdded:nAdded+nFeatures(iScale),:),2),1,nJ).*iScale; - + double(coeffsUNTRIMMED{iScale}(ismember(1:nI,fV),:)); + + % normalizing the variance of the rows gives equal weight to each eigenvector which + % is not reasonable as they are by their nature scaled by the amount of + % variance explained across a given dimension. + % % % coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) = ... + % % % coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) ./ ... + % % % repmat(rms(coeffMat(1+nAdded:nAdded+nFeatures(iScale),:),2),1,nJ).*iScale; + if (flgFlattenEigs) coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) = ... coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) ./ ... repmat(rms(coeffMat(1+nAdded:nAdded+nFeatures(iScale),:),2),1,nJ); end - % Instead, maintain option to weight the features from different scale - % spaces relative to each other. -% coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) = ... -% coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) .* relativeScale(iScale); + % Instead, maintain option to weight the features from different scale + % spaces relative to each other. + % coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) = ... + % coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) .* relativeScale(iScale); nAdded = nAdded + nFeatures(iScale) end @@ -275,169 +275,169 @@ for iCluster = 1:length(clusterVector) nClusters = clusterVector(iCluster); - + + if strcmpi(kAlgorithm, 'kMeans') + [class, classCenters, sumd, D] = kmeans(coeffMat', nClusters, ... + 'replicates', kReplicates, ... + 'Distance', kDistMeasure, ... + 'MaxIter', 50000, ... % Default was 100 + 'Options', statset('UseParallel', 1) ); + + elseif strcmpi(kAlgorithm, 'kMedoids') + [class, classCenters, sumd, D] = kmedoids(coeffMat', nClusters, ... + 'replicates', kReplicates, ... + 'Distance', kDistMeasure, ... + 'Options', statset('UseParallel', 1, ... + 'MaxIter', 50000) ); + + elseif strcmpi(kAlgorithm, 'HAC') + [class] = clusterdata(coeffMat', ... + 'maxclust', nClusters, ... + 'linkage', 'ward', ... + 'distance', 'euclidean', ... + 'savememory', 'off'); + sumd = 0; + + elseif strcmpi(kAlgorithm, 'neuralNetwork') + + net = selforgmap([1 nClusters], coverSteps, initNeighbor, topologyFcn); + [net, tr] = train(net, coeffMat); + y = net(coeffMat) + class = vec2ind(y) + + else + error('kAlgorithm must be kMeans, or kMedoids, not %s', kAlgorithm); + end + + + fprintf('\n\nSum of dist to all centroids for each replicate.\n\n') + fprintf('%g\n',sumd) + totSum1 = sum(sumd); + totStd1 = std(sumd); + fprintf('Total kmeans dist = %g\n', totSum1) + fprintf('Total kmeans std = %g\n', totStd1) + + if (flgRefineKmeans) + % Using the postions found, refine the original estimates + + kMin = min(classCenters,[],1); + kMax = max(classCenters,[],1); + kRange = kMax - kMin; + + % Maximum percentages of the range to search around + + k01 = 0.0001 .* kRange; + k05 = 0.001 .* kRange; + k10 = 0.01 .* kRange; + k25 = 0.1 .* kRange; + + seedMatrix = rand([size(classCenters),512],'single'); + + seedMatrix(:,:,1:128) = repmat(k01,size(classCenters,1),1,128) .* ... + seedMatrix(:,:,1:128); + + seedMatrix(:,:,129:256) = repmat(k05,size(classCenters,1),1,128) .* ... + seedMatrix(:,:,129:256); + + seedMatrix(:,:,257:384) = repmat(k10,size(classCenters,1),1,128) .* ... + seedMatrix(:,:,257:384); + + seedMatrix(:,:,385:512) = repmat(k25,size(classCenters,1),1,128) .* ... + seedMatrix(:,:,385:512); + + + % [class, classCenters, sumd] = kmeans(coeffs(features, :)', nClusters, ... + % 'Start', seedMatrix, ... + % 'Distance', kDistMeasure, ... + % 'MaxIter', 20000, ... % Default was 100 + % 'Options', statset('UseParallel', 1) ); if strcmpi(kAlgorithm, 'kMeans') - [class, classCenters, sumd, D] = kmeans(coeffMat', nClusters, ... - 'replicates', kReplicates, ... - 'Distance', kDistMeasure, ... - 'MaxIter', 50000, ... % Default was 100 - 'Options', statset('UseParallel', 1) ); - + [class, classCenters, sumd,D] = kmeans(coeffMat', nClusters, ... + 'replicates', kReplicates, ... + 'Distance', kDistMeasure, ... + 'MaxIter', 50000, ... % Default was 100 + 'Options', statset('UseParallel', 1) ); + elseif strcmpi(kAlgorithm, 'kMedoids') - [class, classCenters, sumd, D] = kmedoids(coeffMat', nClusters, ... - 'replicates', kReplicates, ... - 'Distance', kDistMeasure, ... - 'Options', statset('UseParallel', 1, ... - 'MaxIter', 50000) ); - - elseif strcmpi(kAlgorithm, 'HAC') - [class] = clusterdata(coeffMat', ... - 'maxclust', nClusters, ... - 'linkage', 'ward', ... - 'distance', 'euclidean', ... - 'savememory', 'off'); - sumd = 0; - - elseif strcmpi(kAlgorithm, 'neuralNetwork') - - net = selforgmap([1 nClusters], coverSteps, initNeighbor, topologyFcn); - [net, tr] = train(net, coeffMat); - y = net(coeffMat) - class = vec2ind(y) - + [class, classCenters, sumd,D] = kmedoids(coeffMat', nClusters, ... + 'replicates', kReplicates, ... + 'Distance', kDistMeasure, ... + 'Options', statset('UseParallel', 1, ... + 'MaxIter', 50000) ); else error('kAlgorithm must be kMeans, or kMedoids, not %s', kAlgorithm); end - fprintf('\n\nSum of dist to all centroids for each replicate.\n\n') - fprintf('%g\n',sumd) - totSum1 = sum(sumd); - totStd1 = std(sumd); - fprintf('Total kmeans dist = %g\n', totSum1) - fprintf('Total kmeans std = %g\n', totStd1) - - if (flgRefineKmeans) - % Using the postions found, refine the original estimates - - kMin = min(classCenters,[],1); - kMax = max(classCenters,[],1); - kRange = kMax - kMin; - - % Maximum percentages of the range to search around - - k01 = 0.0001 .* kRange; - k05 = 0.001 .* kRange; - k10 = 0.01 .* kRange; - k25 = 0.1 .* kRange; - - seedMatrix = rand([size(classCenters),512],'single'); - - seedMatrix(:,:,1:128) = repmat(k01,size(classCenters,1),1,128) .* ... - seedMatrix(:,:,1:128); - - seedMatrix(:,:,129:256) = repmat(k05,size(classCenters,1),1,128) .* ... - seedMatrix(:,:,129:256); - - seedMatrix(:,:,257:384) = repmat(k10,size(classCenters,1),1,128) .* ... - seedMatrix(:,:,257:384); - - seedMatrix(:,:,385:512) = repmat(k25,size(classCenters,1),1,128) .* ... - seedMatrix(:,:,385:512); - - -% [class, classCenters, sumd] = kmeans(coeffs(features, :)', nClusters, ... -% 'Start', seedMatrix, ... -% 'Distance', kDistMeasure, ... -% 'MaxIter', 20000, ... % Default was 100 -% 'Options', statset('UseParallel', 1) ); - if strcmpi(kAlgorithm, 'kMeans') - [class, classCenters, sumd,D] = kmeans(coeffMat', nClusters, ... - 'replicates', kReplicates, ... - 'Distance', kDistMeasure, ... - 'MaxIter', 50000, ... % Default was 100 - 'Options', statset('UseParallel', 1) ); - - elseif strcmpi(kAlgorithm, 'kMedoids') - [class, classCenters, sumd,D] = kmedoids(coeffMat', nClusters, ... - 'replicates', kReplicates, ... - 'Distance', kDistMeasure, ... - 'Options', statset('UseParallel', 1, ... - 'MaxIter', 50000) ); - else - error('kAlgorithm must be kMeans, or kMedoids, not %s', kAlgorithm); - end - - fprintf('\n\nSum of dist to all centroids for each replicate.\n\n') - fprintf('%g\n',sumd); - totSum2 = sum(sumd); - totStd2 = std(sumd); - fprintf('Total kmeans dist = %g\n', totSum2); - fprintf('Total kmeans std = %g\n', totStd2); - fprintf('percent change in mean %g\n', ... - (totSum2 - totSum1)./max(totSum1,totSum2) .* 100); - fprintf('percent change in std %g\n', ... - (totStd2 - totStd1)./max(totStd1,totStd2) .* 100); - - - end - - % This leaves each cluster untouched, but changes the cluster label - % such that the cluster lablelled 1 is also the most populated - % cluster. - classCount = zeros(nClusters,1); - for i = 1:nClusters - % counts of class numbers - classCount(i) = sum(class(:) == i); - end - % list of classIDX with highest count first - [~, ndx] = sort(classCount, 'descend'); - newClass = class; - for i = 1:nClusters - newClass(class == ndx(i)) = i; - end - - fileOUT = fopen(sprintf('%s_%s_ClassIDX.txt',emc.('subTomoMeta'),cycleNumber), 'a'); - fprintf(fileOUT, '\n\n%s, %s, %s\n','position','idx','count'); - for iClass = 1:nClusters - fprintf(fileOUT, '%d, %d\n',iClass,sum(newClass == iClass)); - end - fclose(fileOUT); - - class = newClass; - clear newClass ndx; - - - if length(idxList) ~= length(class) - error('idxList ~= class') - end - - % This isn't great, and maybe my brain is just tired. - - if ( strcmpi(kAlgorithm, 'kMedoids') || strcmpi(kAlgorithm, 'kMeans') ) - save(sprintf('clusterTrouble_%d.mat',iCluster), 'idxList', 'class','classCenters','D'); - else - save(sprintf('clusterTrouble_%d.mat',iCluster), 'idxList', 'class'); - end - - for iTomo = 1:nTomograms - positionList = geometry_clean.(tomoList{iTomo}); - includedClass = ( positionList(:,26) ~= -9999 & ismember(positionList(:,7),randSet)); - positionList(:,8) = includedClass; - particleIDX = positionList( includedClass, 4); - - % Returns the lowest index where this is true - [~, lIndClass] = ismember(particleIDX, idxList); - [~, lIndPart] = ismember(particleIDX, positionList(:,4)); - - % for trouble shooting - try + fprintf('%g\n',sumd); + totSum2 = sum(sumd); + totStd2 = std(sumd); + fprintf('Total kmeans dist = %g\n', totSum2); + fprintf('Total kmeans std = %g\n', totStd2); + fprintf('percent change in mean %g\n', ... + (totSum2 - totSum1)./max(totSum1,totSum2) .* 100); + fprintf('percent change in std %g\n', ... + (totStd2 - totStd1)./max(totStd1,totStd2) .* 100); + + + end + + % This leaves each cluster untouched, but changes the cluster label + % such that the cluster lablelled 1 is also the most populated + % cluster. + classCount = zeros(nClusters,1); + for i = 1:nClusters + % counts of class numbers + classCount(i) = sum(class(:) == i); + end + % list of classIDX with highest count first + [~, ndx] = sort(classCount, 'descend'); + newClass = class; + for i = 1:nClusters + newClass(class == ndx(i)) = i; + end + + fileOUT = fopen(sprintf('%s_%s_ClassIDX.txt',emc.('subTomoMeta'),cycleNumber), 'a'); + fprintf(fileOUT, '\n\n%s, %s, %s\n','position','idx','count'); + for iClass = 1:nClusters + fprintf(fileOUT, '%d, %d\n',iClass,sum(newClass == iClass)); + end + fclose(fileOUT); + + class = newClass; + clear newClass ndx; + + + if length(idxList) ~= length(class) + error('idxList ~= class') + end + + % This isn't great, and maybe my brain is just tired. + + if ( strcmpi(kAlgorithm, 'kMedoids') || strcmpi(kAlgorithm, 'kMeans') ) + save(sprintf('clusterTrouble_%d.mat',iCluster), 'idxList', 'class','classCenters','D'); + else + save(sprintf('clusterTrouble_%d.mat',iCluster), 'idxList', 'class'); + end + + for iTomo = 1:nTomograms + positionList = geometry_clean.(tomoList{iTomo}); + includedClass = ( positionList(:,26) ~= -9999 & ismember(positionList(:,7),randSet)); + positionList(:,8) = includedClass; + particleIDX = positionList( includedClass, 4); + + % Returns the lowest index where this is true + [~, lIndClass] = ismember(particleIDX, idxList); + [~, lIndPart] = ismember(particleIDX, positionList(:,4)); - if (emc.nPeaks > 1) + % for trouble shooting + try + + if (emc.nPeaks > 1) for thisIDX = 1:length(lIndClass) for iPeak = 0:emc.nPeaks-1 positionList(lIndPart(thisIDX), 26 + 26*iPeak) = class(lIndClass(thisIDX)+iPeak); -% fprintf('iTomo %d iSubtomo %d iPeak %d Class %d\n',iTomo,lIndPart(thisIDX),iPeak+1,class(lIndClass(thisIDX)+iPeak)); + % fprintf('iTomo %d iSubtomo %d iPeak %d Class %d\n',iTomo,lIndPart(thisIDX),iPeak+1,class(lIndClass(thisIDX)+iPeak)); end end geometry.(tomoList{iTomo}) = positionList; @@ -447,27 +447,27 @@ geometry.(tomoList{iTomo}) = positionList; end - catch - save('ClusterLine391Err.mat') - error('Caught error, saving workspace for evaluation.\n') - end - + catch + save('ClusterLine391Err.mat') + error('Caught error, saving workspace for evaluation.\n') end - fout = sprintf('%s_%d_%d_nClass_%d_%s', outputPrefix, featureVector{iGold}(1,1), ... - featureVector{iGold}(1,end), nClusters, halfSet); - - % Save a copy of the geometry in the subTomoMeta and also save the name for - % easy reference in a text file. - masterTM.(cycleNumber).('ClusterResults').(fout) = geometry; - - - + + end + fout = sprintf('%s_%d_%d_nClass_%d_%s', outputPrefix, featureVector{iGold}(1,1), ... + featureVector{iGold}(1,end), nClusters, halfSet); + + % Save a copy of the geometry in the subTomoMeta and also save the name for + % easy reference in a text file. + masterTM.(cycleNumber).('ClusterResults').(fout) = geometry; + + + end % loop over cluster size - + subTomoMeta = masterTM; - + save(emc.('subTomoMeta'), 'subTomoMeta'); - + %save(sprintf('%s_pca.mat',OUTPUT_PREFIX), 'nTOTAL','U', 'S', 'V', 'coeffs') fprintf('Total execution time on set %s: %f seconds\n', halfSet,etime(clock, startTime)); delete(gcp('nocreate')); diff --git a/statistics/BH_diffMap.m b/statistics/BH_diffMap.m index 021047e0..d5fdba28 100755 --- a/statistics/BH_diffMap.m +++ b/statistics/BH_diffMap.m @@ -1,7 +1,7 @@ function [ diffMap, normMap ] = BH_diffMap( refMap, ... - particle, particleCTF, ... - flgNorm, pixelSize,... - radialGrid, padVal ) + particle, particleCTF, ... + flgNorm, pixelSize,... + radialGrid, padVal ) %Scale a higher SNR map down do approximate the power of a noisy map % Maps are assumed to be the same size and also be masked in real space. If % the particle ctf is just 1, and flgNorm than this should behave like Niko's @@ -21,7 +21,7 @@ end elseif (flgNorm && ~iscell(radialGrid)) [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(size(refMap),'Cartesian',... - 'GPU',{'none'},1,0,1); + 'GPU',{'none'},1,0,1); radialGrid = radialGrid ./ pixelSize; end @@ -34,7 +34,7 @@ if isreal(refMap(1)) refMap = fftn(BH_padZeros3d(refMap,padVal(1,:),padVal(2,:),'GPU','single')).*(particleCTF); else - refMap = refMap.*(particleCTF); + refMap = refMap.*(particleCTF); end if isreal(particle) @@ -58,35 +58,35 @@ if (flgNorm) % Normalize over predefined radial bins - scalar_vals = zeros(length(radialGrid),1); - for iBin = 1:length(radialGrid) - particle_sum = sum(abs(particle(radialGrid{iBin})).^2,'all'); - if ( particle_sum == 0 ) - scalar_vals(iBin) = 0; + scalar_vals = zeros(length(radialGrid),1); + for iBin = 1:length(radialGrid) + particle_sum = sum(abs(particle(radialGrid{iBin})).^2,'all'); + if ( particle_sum == 0 ) + scalar_vals(iBin) = 0; + else + ref_sum = sum(abs(refMap(radialGrid{iBin})).^2,'all'); + if ( ref_sum == 0 ) + scalar_vals(iBin) = 0; + else + scalar = sqrt(particle_sum ./ ref_sum); + if (isfinite(scalar)) + scalar_vals(iBin) = scalar; else - ref_sum = sum(abs(refMap(radialGrid{iBin})).^2,'all'); - if ( ref_sum == 0 ) - scalar_vals(iBin) = 0; - else - scalar = sqrt(particle_sum ./ ref_sum); - if (isfinite(scalar)) - scalar_vals(iBin) = scalar; - else - scalar_vals(iBin) = 0; - end - end - end - end - - for iBin = 1:length(radialGrid) - refMap(radialGrid{iBin}) = refMap(radialGrid{iBin}) .* scalar_vals(iBin); + scalar_vals(iBin) = 0; + end + end end - + end + + for iBin = 1:length(radialGrid) + refMap(radialGrid{iBin}) = refMap(radialGrid{iBin}) .* scalar_vals(iBin); + end + diffMap = real(ifftn(refMap - particle)); else -% refMap = refMap ./ sum(abs(refMap(:)).^2); % FIXME -% particle = particle ./ sum(abs(particle(:)).^2); - + % refMap = refMap ./ sum(abs(refMap(:)).^2); % FIXME + % particle = particle ./ sum(abs(particle(:)).^2); + normMap = ''; diffMap = real(ifftn(refMap - particle)); end @@ -96,8 +96,8 @@ % % figure, imshow3D(gather(real(ifftn(refMap)))); % % if isnumeric(normMap); figure, imshow3D(gather(real(ifftn(normMap)))); end % % figure, imshow3D(gather(real((diffMap)))); -% % -% % +% % +% % % % error('sdf') % [ PEAK_COORD ] = BH_multi_xcf_Translational( particle, conj(refMap), ... diff --git a/statistics/BH_exposureFilter.m b/statistics/BH_exposureFilter.m index 36cf445c..55013dfe 100755 --- a/statistics/BH_exposureFilter.m +++ b/statistics/BH_exposureFilter.m @@ -1,6 +1,6 @@ function [ exposureFilter ] = BH_exposureFilter( SIZE, TILT_GEOMETRY, METHOD,... - SAMPLING, SHIFT_ORIGIN, ... - varargin) + SAMPLING, SHIFT_ORIGIN, ... + varargin) %UNTITLED2 Summary of this function goes here % Detailed explanation goes here @@ -16,17 +16,17 @@ end doFullGrid = true; - % The A-D are the defaults as published in the optimal exposure paper - % optW controls the exponential fall-off when the current exposure - % exceeds the optimal exposure. In unblur, this is "inf" as everything - % beyond this point is set to zero. A small number still gives a steep - % fall off, but with some taper. - expA = 0.24499; - expB =-1.66490; - expC = 2.81410; - optD = 2.51284; - optW = 0.0; - +% The A-D are the defaults as published in the optimal exposure paper +% optW controls the exponential fall-off when the current exposure +% exceeds the optimal exposure. In unblur, this is "inf" as everything +% beyond this point is set to zero. A small number still gives a steep +% fall off, but with some taper. +expA = 0.24499; +expB =-1.66490; +expC = 2.81410; +optD = 2.51284; +optW = 0.0; + if nargin > 5 if(length(varargin{1}) > 1) expA = varargin{1}(1); @@ -65,7 +65,7 @@ d3 = nPrjs; % if numel(SIZE) == 2 % d2 = SIZE(2); -% else +% else % d2 = d1; % end @@ -89,18 +89,18 @@ elseif strcmp(METHOD, 'cpu') exposureFilter = zeros([h1,d2,d3], 'single'); else - error('METHOD must be GPU or %s\n','cpu'); + error('METHOD must be GPU or %s\n','cpu'); end if (doFullGrid) [criticalDose,~,~,~,~,~] = BH_multi_gridCoordinates( ... - [d1,d2],'Cartesian',... - METHOD,{'none'},1,SHIFT_ORIGIN,1); + [d1,d2],'Cartesian',... + METHOD,{'none'},1,SHIFT_ORIGIN,1); else - [criticalDose,~,~,~,~,~] = BH_multi_gridCoordinates( ... - [d1,d2],'Cartesian',... - METHOD,{'none'},1,SHIFT_ORIGIN,1,{'halfGrid'}); + [criticalDose,~,~,~,~,~] = BH_multi_gridCoordinates( ... + [d1,d2],'Cartesian',... + METHOD,{'none'},1,SHIFT_ORIGIN,1,{'halfGrid'}); end % Assuming the pixelSize is constant across projections, the only change is % the cummulative dose, so precompute everything @@ -111,46 +111,46 @@ % Precompute some values % criticalDose = exp(-0.5.*criticalDose.^-1); - - - -for iPrj = 1:nPrjs + + +for iPrj = 1:nPrjs + CUMeDOSE = TILT_GEOMETRY(iPrj,11); - + % Frequency where the dose exceeds the optimal dose (2.51*critical) are % set to zero in the unblur code .. use a gaussian falloff. -%%%%% This creates a more agressive dose filter perpendicular to the tilt axis. -%%%%% With these parameters I saw no change, better or worse which is a little -%%%%% odd. Maybe check it out in the future. -% % % sX = SIZE(1); -% % % sY = floor(SIZE(2).*(2-cosd(TILT_GEOMETRY(iPrj,4)))); -% % % [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates( ... -% % % [sX,sY],'Cartesian',... -% % % METHOD,{'none'},1,1,1); -% % % oX = ceil((sX+1)./2); -% % % oY = ceil((sY+1)./2); -% % % radialGrid = radialGrid ./ (radialGrid(oX,oY+oX-2).*2); -% % % trimVal = BH_multi_padVal([sX,sY],SIZE(1:2)); -% % % radialGrid = BH_padZeros3d(radialGrid,trimVal(1,:),trimVal(2,:),METHOD,'single'); -% % % if ~(SHIFT_ORIGIN) -% % % radialGrid = ifftshift(radialGrid); -% % % end -% exp((-0.5*CUMeDOSE)./criticalDose).* ... - -% -% optimalMask = ( (optimalDose>=CUMeDOSE) + exp(-optW*(CUMeDOSE-optimalDose)).*(optimalDoseCUMeDOSE); - + %%%%% This creates a more agressive dose filter perpendicular to the tilt axis. + %%%%% With these parameters I saw no change, better or worse which is a little + %%%%% odd. Maybe check it out in the future. + % % % sX = SIZE(1); + % % % sY = floor(SIZE(2).*(2-cosd(TILT_GEOMETRY(iPrj,4)))); + % % % [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates( ... + % % % [sX,sY],'Cartesian',... + % % % METHOD,{'none'},1,1,1); + % % % oX = ceil((sX+1)./2); + % % % oY = ceil((sY+1)./2); + % % % radialGrid = radialGrid ./ (radialGrid(oX,oY+oX-2).*2); + % % % trimVal = BH_multi_padVal([sX,sY],SIZE(1:2)); + % % % radialGrid = BH_padZeros3d(radialGrid,trimVal(1,:),trimVal(2,:),METHOD,'single'); + % % % if ~(SHIFT_ORIGIN) + % % % radialGrid = ifftshift(radialGrid); + % % % end + % exp((-0.5*CUMeDOSE)./criticalDose).* ... + + % + % optimalMask = ( (optimalDose>=CUMeDOSE) + exp(-optW*(CUMeDOSE-optimalDose)).*(optimalDoseCUMeDOSE); + end diff --git a/statistics/BH_fit_ellipsoidal_prior.m b/statistics/BH_fit_ellipsoidal_prior.m index 6268f635..75fed29d 100644 --- a/statistics/BH_fit_ellipsoidal_prior.m +++ b/statistics/BH_fit_ellipsoidal_prior.m @@ -1,9 +1,9 @@ function [ normal_vect, chi2] = BH_fit_ellipsoidal_prior(particle_coords, ... - particle_radius_Z, ... - radial_shrink_factor, ... - display_fit) + particle_radius_Z, ... + radial_shrink_factor, ... + display_fit) % particle_coords = load('gag.txt').*10.*1.33; % input as Angstrom -% +% % particle_radius_Z = 50; % Angstrom box_size = 1.*256.*[1,1,1]; @@ -20,37 +20,37 @@ try gFit = fitgmdist(ad,2,'RegularizationValue',regularizer,'Start','plus',... - 'Replicates',20,'Options', statset('UseParallel', true, ... - 'MaxIter', 1000)); - save('g.mat','gFit','ad'); + 'Replicates',20,'Options', statset('UseParallel', true, ... + 'MaxIter', 1000)); + save('g.mat','gFit','ad'); catch fprintf('Failed to fit, increasing regularizer\n'); gFit = fitgmdist(ad,2,'RegularizationValue',regularizer*10); end - gPosterior = posterior(gFit,ad); +gPosterior = posterior(gFit,ad); - if (display_fit) - gFit.ComponentProportion - save('gmFit.mat','gFit'); - figure, histogram(ad) - hold on - histogram(ad(gPosterior(:,1) > 1- gFit.ComponentProportion(1))) - histogram(ad(gPosterior(:,2) > 1- gFit.ComponentProportion(2))) - end +if (display_fit) + gFit.ComponentProportion + save('gmFit.mat','gFit'); + figure, histogram(ad) + hold on + histogram(ad(gPosterior(:,1) > 1- gFit.ComponentProportion(1))) + histogram(ad(gPosterior(:,2) > 1- gFit.ComponentProportion(2))) +end + +if (abs(diff(gFit.mu)) < 2*particle_radius_Z) + fprintf('The best mixture model shows a difference (%3.3f) which is less than particle diameter (%3.3f), assuming no outliers\n',abs(diff(gFit.mu)) , 2*particle_radius_Z); + ignore_outliers = false(size(particle_coords,1),1); +else - if (abs(diff(gFit.mu)) < 2*particle_radius_Z) - fprintf('The best mixture model shows a difference (%3.3f) which is less than particle diameter (%3.3f), assuming no outliers\n',abs(diff(gFit.mu)) , 2*particle_radius_Z); - ignore_outliers = false(size(particle_coords,1),1); - else - - [~,minClass] = min(sum(gPosterior)); - gFit.ComponentProportion(minClass) - ignore_outliers = gPosterior(:, minClass) > 1- gFit.ComponentProportion(minClass); - fprintf('ignoring %d outliers\n',sum(ignore_outliers)); - end - + [~,minClass] = min(sum(gPosterior)); + gFit.ComponentProportion(minClass) + ignore_outliers = gPosterior(:, minClass) > 1- gFit.ComponentProportion(minClass); + fprintf('ignoring %d outliers\n',sum(ignore_outliers)); +end + % Get "robust" fit ignoring outliers [ center, ~, ~, v, chi2] = ellipsoid_fit(particle_coords(~ignore_outliers,:),''); @@ -71,9 +71,9 @@ % ellipse as a binary mask around -v(10) Ellipsoid = v(1) *gX.*gX + v(2) * gY.*gY + v(3) * gZ.*gZ + ... - 2*v(4) *gX.*gY + 2*v(5)*gX.*gZ + 2*v(6) * gY.*gZ + ... - 2*v(7) *gX + 2*v(8)*gY + 2*v(9) * gZ; - + 2*v(4) *gX.*gY + 2*v(5)*gX.*gZ + 2*v(6) * gY.*gZ + ... + 2*v(7) *gX + 2*v(8)*gY + 2*v(9) * gZ; + Ellipsoid_mask = abs(Ellipsoid +v(10)) < 1e-3; Ellipsoid = Ellipsoid .* Ellipsoid_mask; @@ -111,37 +111,37 @@ ellipsoid_copy = Ellipsoid_mask .* real(bhF.invFFT(bhF.fwdFFT(ellipsoid_copy) .* searchKernel)); - + [~, mc] = min(ellipsoid_copy(:)); [i_min,j_min,k_min] = ind2sub(box_size,mc); - + % Calculate the gradient at the closest point on the ellipse % I've ommitted a factor of 2 b/c it is set to unit length anyway grad = [ v(1).*vX(i_min) + v(4).*vY(j_min) + v(5).*vZ(k_min) + v(7) , ... - v(2).*vY(j_min) + v(4).*vX(i_min) + v(6).*vZ(k_min) + v(8) , ... - v(3).*vZ(k_min) + v(5).*vX(i_min) + v(6).*vY(j_min) + v(9) ]; + v(2).*vY(j_min) + v(4).*vX(i_min) + v(6).*vZ(k_min) + v(8) , ... + v(3).*vZ(k_min) + v(5).*vX(i_min) + v(6).*vY(j_min) + v(9) ]; normal_vect(iPt,:) = -gather(grad ./ norm(grad)); xyz_vect(iPt,:) = gather([vX(i_min),vY(j_min),vZ(k_min)]); if (display_fit) scatter3(j,i,k,'rx'); scatter3(j_min,i_min,k_min,'ro'); - - end - + + end + end if (display_fit) iPt = 1; figure, quiver3(xyz_vect(iPt,2),xyz_vect(iPt,1),xyz_vect(iPt,3),... - normal_vect(iPt,2),normal_vect(iPt,1),normal_vect(iPt,3),30,'MaxHeadSize',6,'Marker','o','MarkerSize',2); + normal_vect(iPt,2),normal_vect(iPt,1),normal_vect(iPt,3),30,'MaxHeadSize',6,'Marker','o','MarkerSize',2); hold on for iPt = 2:size(particle_coords,1) quiver3(xyz_vect(iPt,2),xyz_vect(iPt,1),xyz_vect(iPt,3), ... - normal_vect(iPt,2),normal_vect(iPt,1),normal_vect(iPt,3),30,'MaxHeadSize',6,'Marker','o','MarkerSize',2); + normal_vect(iPt,2),normal_vect(iPt,1),normal_vect(iPt,3),30,'MaxHeadSize',6,'Marker','o','MarkerSize',2); end fprintf('Chi2 is %3.3e\n',chi2); hold off @@ -158,5 +158,5 @@ % mask with ellipse -% get coords +% get coords %$[i,j,k] = ind2sub(sigZe(e),cv) diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 3c126397..d4d57779 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -15,7 +15,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% if (nargin < 3 && nargin > 4) - error('args = PARAMETER_FILE, CYCLE, STAGEofALIGNMENT') + error('args = PARAMETER_FILE, CYCLE, STAGEofALIGNMENT') end @@ -62,7 +62,7 @@ try shape_mask_lowpass = emc.('shape_mask_lowpass'); catch - shape_mask_lowpass = 14; + shape_mask_lowpass = 14; end try @@ -80,7 +80,7 @@ % Estimating the particle volume still occasionaly goes awry. Place a cap and return a cautionary message. -try +try minimumParticleVolume = emc.('minimumparticleVolume'); catch minimumParticleVolume = 0.1; @@ -90,7 +90,7 @@ catch fscWithChimera = 0; end -outputPrefix = sprintf('./FSC/%s_%s', cycleNumber, emc.('subTomoMeta')); +outputPrefix = sprintf('./FSC/%s_%s', cycleNumber, emc.('subTomoMeta')); samplingRate = emc.('Ali_samplingRate'); pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; @@ -108,22 +108,22 @@ coneList = 0; nCones = 0; halfAngle = 0; - calcCones = 0; + calcCones = 0; end if ( calcCones ) n=2; coneList = cell(nCones,1); halfAngle= cell(nCones,1); - halfAngle{1} = 1.2*coneInc; % coneInc should be afactor of 90 and 360 so - % this should return an integer + halfAngle{1} = 1.2*coneInc; % coneInc should be afactor of 90 and 360 so + % this should return an integer coneList{1} = [0,0,0]; for j = coneInc:coneInc:90; for i = 0:coneInc:360-coneInc coneList{n} = [i,j,0]; halfAngle{n}= halfAngle{1} ; n=n+1; - end + end end end @@ -146,7 +146,7 @@ % The default is fsc-Gold Standard so the two images should need some degree of -% alignment prior to calculating the fsc. +% alignment prior to calculating the fsc. flgAlignImages = 1; flgJustFSC=0; % check to see if images supplied, or to be read in. @@ -160,32 +160,32 @@ error('Size of img1 and img2 are inconsistent.') else IMG1 = STAGEofALIGNMENT{1}; - IMG2 = STAGEofALIGNMENT{2}; + IMG2 = STAGEofALIGNMENT{2}; flgJustFSC=1; nReferences=2; - refVector{1} =1; - refVector{2}= 1; - STAGEofALIGNMENT = 'RawAlignment'; - fieldPrefix = 'REF' + refVector{1} =1; + refVector{2}= 1; + STAGEofALIGNMENT = 'RawAlignment'; + fieldPrefix = 'REF' end else switch STAGEofALIGNMENT case 'RawAlignment' savePrefix = 'Raw'; - if (flgClassify) - fieldPrefix = 'Raw'; - - className = 0; - classVector = [0;1]; - else + if (flgClassify) + fieldPrefix = 'Raw'; + + className = 0; + classVector = [0;1]; + else className = emc.(sprintf('Raw_className')); classVector = emc.(sprintf('Raw_classes_odd')); - fieldPrefix = 'REF'; + fieldPrefix = 'REF'; end - - + + nReferences = length(classVector(1,:)) imageName{1} = sprintf('class_%d_Locations_%s_ODD_NoWgt', className,fieldPrefix); @@ -193,18 +193,18 @@ weightName{1} = sprintf('class_%d_Locations_%s_ODD_Wgt', className,fieldPrefix); weightName{2} = sprintf('class_%d_Locations_%s_EVE_Wgt', className,fieldPrefix); imageName{1} - + refVector{1} =1; refVector{2}= 1; outputPrefix = sprintf('%s_Raw', outputPrefix); - + case 'NoAlignment' savePrefix = 'Raw'; if (flgClassify) fieldPrefix = 'NoA'; else - fieldPrefix = 'REF'; + fieldPrefix = 'REF'; end imageName{1} = sprintf('class_0_Locations_%s_ODD_NoWgt', fieldPrefix); imageName{2} = sprintf('class_0_Locations_%s_EVE_NoWgt', fieldPrefix); @@ -214,7 +214,7 @@ nReferences = 1; refVector{1} =1; refVector{2}= 1; - + outputPrefix = sprintf('%s_NoA', outputPrefix); case 'Cluster' error('Fsc calculation for cluster results is not implemented.') @@ -227,7 +227,7 @@ fieldPrefix = 'NoA' end imageName{1} = sprintf('class_%d_Locations_%s_ODD_NoWgt', 25,fieldPrefix); - imageName{2} = sprintf('class_%d_Locations_%s_EVE_NoWgt', 25,fieldPrefix); + imageName{2} = sprintf('class_%d_Locations_%s_EVE_NoWgt', 25,fieldPrefix); outputPrefix = sprintf('%s_Snr', outputPrefix); refVector{1} = [1:25]; refVector{2} = [1:25]; @@ -244,13 +244,13 @@ % [ maskType, maskSize, maskRadius, maskCenter ] = ... % BH_multi_maskCheck(emc, 'Ali', pixelSize,'FSC') - + [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', pixelSize) - + BH_multi_maskCheck(emc, 'Ali', pixelSize) + [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) - + BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) + padDIM = max(max(sizeWindow),384); padREF = [0,0,0;0,0,0]; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -259,38 +259,38 @@ refIMG = cell(2,1); refWGT = cell(2,1); for iGold = 1:2 - + if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - + [ refIMG{iGold} ] = BH_unStackMontage4d(1:nReferences, ... - masterTM.(cycleNumber).(imageName{iGold}){1},... - masterTM.(cycleNumber).(imageName{iGold}){2},... - sizeWindow); - + masterTM.(cycleNumber).(imageName{iGold}){1},... + masterTM.(cycleNumber).(imageName{iGold}){2},... + sizeWindow); + [ refWGT{iGold} ] = BH_unStackMontage4d(1:nReferences, ... - masterTM.(cycleNumber).(weightName{iGold}){1},... - masterTM.(cycleNumber).(weightName{iGold}){2},... - sizeCalc); - + masterTM.(cycleNumber).(weightName{iGold}){1},... + masterTM.(cycleNumber).(weightName{iGold}){2},... + sizeCalc); + padLSQ = BH_multi_padVal(sizeWindow,sizeCalc); trimLSQ = BH_multi_padVal(sizeCalc,sizeWindow); for iLSQ = 1:nReferences padIMG = fftn(BH_padZeros3d(refIMG{iGold}{iLSQ},padLSQ(1,:),padLSQ(2,:),'GPU','single')); padWGT = ifftshift(gpuArray(refWGT{iGold}{iLSQ})); - + [padWGT, wienerThreshold] = BH_multi_cRef_wgtCritical(padWGT); - -% padIMG = real(ifftn(padIMG./(padWGT+wienerThreshold))); + + % padIMG = real(ifftn(padIMG./(padWGT+wienerThreshold))); padIMG = real(ifftn(padIMG./(padWGT+100))); - + clear padWGT refIMG{iGold}{iLSQ} = BH_padZeros3d(gather(padIMG),trimLSQ(1,:),trimLSQ(2,:),'cpu','single'); - clear padIMG - end + clear padIMG + end end clear refWGT else @@ -302,7 +302,7 @@ % Make a mask, and apply to the average motif && save a masked, binned copy of -% the average for inspection. +% the average for inspection. % % % % % % % [ tmpMask ] = BH_mask3d(maskType, sizeMask, maskRadius-7, maskCenter); @@ -311,21 +311,21 @@ volMask{1} = gather(BH_multi_randomizeTaper(tmpMask)); volMask{2} = gather(BH_multi_randomizeTaper(tmpMask)); clear tmpMask -% % % % % % % [ tmpMask] = BH_mask3d(maskType, sizeMask, peakSearch, maskCenter); +% % % % % % % [ tmpMask] = BH_mask3d(maskType, sizeMask, peakSearch, maskCenter); [ tmpMask ] = EMC_maskShape(maskType, sizeMask, peakSearch, 'gpu', {'shift', maskCenter}); peakMask{1} = gather(BH_multi_randomizeTaper(tmpMask)); peakMask{2} = gather(BH_multi_randomizeTaper(tmpMask)); clear tmpMask bandpassFilt = cell(2,1); - bandpassFilt{1} = BH_bandpass3d(sizeCalc,0,0,0,'cpu','nyquist'); - bandpassFilt{2} = BH_bandpass3d(sizeCalc,0,0,0,'cpu','nyquist'); +bandpassFilt{1} = BH_bandpass3d(sizeCalc,0,0,0,'cpu','nyquist'); +bandpassFilt{2} = BH_bandpass3d(sizeCalc,0,0,0,'cpu','nyquist'); clear radialGrid - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % Default on. +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Default on. % if ( flgEstSolvent ) - particleVolume = zeros(nReferences,1); +particleVolume = zeros(nReferences,1); % else % particleVolume = ones(nReferences,1); % end @@ -341,86 +341,86 @@ if (flgAlignImages) && ~(flgJustFSC) && ~(flgEstSNR) refRotAvg = cell(nReferences,1); for iGold = 1:2 - + for iRef = 1:nReferences - % iHalf = refVector{iGold}(iRef); + % iHalf = refVector{iGold}(iRef); if iGold == 1 refRotAvg{iRef} = refIMG{iGold}{iRef};%BH_axialSymmetry(refIMG{iGold}{iRef}, 120,... - %0, 'GPU',[0,0,0]); + %0, 'GPU',[0,0,0]); end - - + + end end - - - bestAnglesTotal = zeros(nReferences,12); - + + + bestAnglesTotal = zeros(nReferences,12); + nCount = 1; for iRef = 1:nReferences - fprintf('working on %d/ %d references FscGold\n', iRef, nReferences); - - - [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(refIMG{1}{iRef}), pixelSize, {'fsc', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); - [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(refIMG{2}{iRef}), pixelSize, {'fsc', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); - - if (shape_mask_test) - fprintf('\nSaving your masks and exiting!\n'); - SAVE_IMG(shapeMask_1,sprintf('%s-shape_mask_%2.2f_lowpass_%2.2f_threshold.mrc', ... - outputPrefix, shape_mask_lowpass,shape_mask_threshold),pixelSize); - return; - end - - if (flgFscShapeMask) - shapeMask_1 = gather((shapeMask_1.*volMask{1}).^flgFscShapeMask); - shapeMask_2 = gather((shapeMask_2.*volMask{2}).^flgFscShapeMask); - else - shapeMask_1 = 1; - shapeMask_2 = 1; - end - - particleVolume(iRef) = gather(mean([particleFraction1,particleFraction2])); - pV1 = gather(pV1.*volMask{1}); - pV2 = gather(pV2.*volMask{2}); - - % Save a copy with headers set (headers only important for chimera) - % TODO: check to see if these are used anywhere else, otherwise, they can probably be - % written only in the if fscWithChimera block. - eveName = sprintf('%s-eveAli.mrc', outputPrefix); - oddName = sprintf('%s-oddAli.mrc', outputPrefix); - SAVE_IMG(MRCImage(gather(refIMG{2}{iRef}.*pV2)), ... - eveName, 1.0, 1); - SAVE_IMG(MRCImage(gather(refIMG{1}{iRef}.*pV1)), ... - oddName, 1.0,1); - - if (fscWithChimera) - [whereIsChimera, ~] = system('which chimera'); - if (whereIsChimera) - error('fscWithChimera is called, but "chimera" not in system PATH.') - end - writeOutPyAli() - system(sprintf('chimera --nogui --script "FSC/fitInMap.py %s %s %s-fitInMap.txt" ',... - oddName,eveName,outputPrefix)); - % Read in the results - % 1:9 rotation matrix 10:12 = dXYZ - bestAnglesTotal(nCount,:) = load(sprintf('%s-fitInMap.txt',outputPrefix)); - % The results from fit in map are the transpose of Bah, forward rotmat - bestAnglesTotal(nCount,1:9) = bestAnglesTotal(nCount,[1,4,7,2,5,8,3,6,9]); - % The results are in Angstrom - bestAnglesTotal(nCount,(10:12)) = bestAnglesTotal(nCount,(10:12)); - else - % If we did not align the two halfsets together, we just need to return dummy values - % for an identity matrix and zero translation. - bestAnglesTotal(nCount,:) = [1,0,0,0,1,0,0,0,1,0,0,0]; + fprintf('working on %d/ %d references FscGold\n', iRef, nReferences); + + + [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(refIMG{1}{iRef}), pixelSize, {'fsc', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); + [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(refIMG{2}{iRef}), pixelSize, {'fsc', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); + + if (shape_mask_test) + fprintf('\nSaving your masks and exiting!\n'); + SAVE_IMG(shapeMask_1,sprintf('%s-shape_mask_%2.2f_lowpass_%2.2f_threshold.mrc', ... + outputPrefix, shape_mask_lowpass,shape_mask_threshold),pixelSize); + return; + end + + if (flgFscShapeMask) + shapeMask_1 = gather((shapeMask_1.*volMask{1}).^flgFscShapeMask); + shapeMask_2 = gather((shapeMask_2.*volMask{2}).^flgFscShapeMask); + else + shapeMask_1 = 1; + shapeMask_2 = 1; + end + + particleVolume(iRef) = gather(mean([particleFraction1,particleFraction2])); + pV1 = gather(pV1.*volMask{1}); + pV2 = gather(pV2.*volMask{2}); + + % Save a copy with headers set (headers only important for chimera) + % TODO: check to see if these are used anywhere else, otherwise, they can probably be + % written only in the if fscWithChimera block. + eveName = sprintf('%s-eveAli.mrc', outputPrefix); + oddName = sprintf('%s-oddAli.mrc', outputPrefix); + SAVE_IMG(MRCImage(gather(refIMG{2}{iRef}.*pV2)), ... + eveName, 1.0, 1); + SAVE_IMG(MRCImage(gather(refIMG{1}{iRef}.*pV1)), ... + oddName, 1.0,1); + + if (fscWithChimera) + [whereIsChimera, ~] = system('which chimera'); + if (whereIsChimera) + error('fscWithChimera is called, but "chimera" not in system PATH.') end - - nCount = nCount+1; + writeOutPyAli() + system(sprintf('chimera --nogui --script "FSC/fitInMap.py %s %s %s-fitInMap.txt" ',... + oddName,eveName,outputPrefix)); + % Read in the results + % 1:9 rotation matrix 10:12 = dXYZ + bestAnglesTotal(nCount,:) = load(sprintf('%s-fitInMap.txt',outputPrefix)); + % The results from fit in map are the transpose of Bah, forward rotmat + bestAnglesTotal(nCount,1:9) = bestAnglesTotal(nCount,[1,4,7,2,5,8,3,6,9]); + % The results are in Angstrom + bestAnglesTotal(nCount,(10:12)) = bestAnglesTotal(nCount,(10:12)); + else + % If we did not align the two halfsets together, we just need to return dummy values + % for an identity matrix and zero translation. + bestAnglesTotal(nCount,:) = [1,0,0,0,1,0,0,0,1,0,0,0]; + end + + nCount = nCount+1; end elseif (flgAlignImages) && ~(flgJustFSC) && (flgEstSNR) - error('this block in FSC alignment is slated for removal'); + error('this block in FSC alignment is slated for removal'); end @@ -444,7 +444,7 @@ nRep = 1; for iRef = 1:5 - for iRep = 1:4 + for iRep = 1:4 for iPerm = iRep+1:5 tmpRef{1}{nRep} = refIMG{1}{iRep + 5*(iRef-1)}; tmpRef{2}{nRep} = refIMG{2}{iPerm + 5*(iRef-1)}; @@ -458,47 +458,47 @@ end end - end -refSymmetry = [1:100;ones(1,100)]; -refIMG = tmpRef; clear tmpRef -nReferences = 100; + end + refSymmetry = [1:100;ones(1,100)]; + refIMG = tmpRef; clear tmpRef + nReferences = 100; end for iRef = 1:nReferences if (flgAlignImages) && ~(flgJustFSC) - - + + img2 = refIMG{2}{iRef}; - + if (flgEstSNR) - img1 = BH_resample3d(gather(refIMG{1}{iRef}), ... - bestAnglesTotal(1,3:5), ... - bestAnglesTotal(1,8:10), ... - {'Bah',1,'spline'}, 'cpu', 'forward'); + img1 = BH_resample3d(gather(refIMG{1}{iRef}), ... + bestAnglesTotal(1,3:5), ... + bestAnglesTotal(1,8:10), ... + {'Bah',1,'spline'}, 'cpu', 'forward'); else - - rotMat = bestAnglesTotal(iRef,1:9); - dXYZ = bestAnglesTotal(iRef,10:12); - - if abs(3-sum(rotMat([1,5,9]))) < .005 - % Should I pad this? The shifts are small enough it prob is okay. - fprintf('\nRotation xform is very small, just applying phase shifts\n') - [ dU, dV, dZ ] = BH_multi_gridCoordinates(size(refIMG{1}{iRef}),'Cartesian','cpu', ... - {'none'},1,0,0); - + + rotMat = bestAnglesTotal(iRef,1:9); + dXYZ = bestAnglesTotal(iRef,10:12); + + if abs(3-sum(rotMat([1,5,9]))) < .005 + % Should I pad this? The shifts are small enough it prob is okay. + fprintf('\nRotation xform is very small, just applying phase shifts\n') + [ dU, dV, dZ ] = BH_multi_gridCoordinates(size(refIMG{1}{iRef}),'Cartesian','cpu', ... + {'none'},1,0,0); + shiftVect = exp((-2i*pi).*(dU.*dXYZ(1) + dV.*dXYZ(2)+ dZ.*dXYZ(3))); clear dU dV dZ img1 = real(ifftn(fftn(gather(refIMG{1}{iRef})).*shiftVect)); - else - img1 = BH_resample3d(gather(refIMG{1}{iRef}), ... - rotMat, ... - dXYZ, ... - {'Bah',1,'spline'}, 'cpu', 'forward'); - end + else + img1 = BH_resample3d(gather(refIMG{1}{iRef}), ... + rotMat, ... + dXYZ, ... + {'Bah',1,'spline'}, 'cpu', 'forward'); + end end - - + + halfSet = 'GLD'; fprintf('resampling ref %d.\n', iRef); img1 = BH_padZeros3d(img1,[0,0,0],[0,0,0],'cpu','singleTaper'); @@ -507,71 +507,71 @@ img1=IMG1; img2=IMG2; - - [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(img1), pixelSize, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); - [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(img2), pixelSize, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); - - if (shape_mask_test) - fprintf('\nSaving your masks and exiting!\n'); - SAVE_IMG(shapeMask_1,sprintf('%s-shape_mask_%2.2f_lowpass_%2.2f_threshold.mrc', ... - outputPrefix, shape_mask_lowpass,shape_mask_threshold),pixelSize); - return; - end - - - if (flgFscShapeMask) - shapeMask_1 = gather((shapeMask_1.*volMask{1}).^flgFscShapeMask); - shapeMask_2 = gather((shapeMask_2.*volMask{2}).^flgFscShapeMask); - else - shapeMask_1 = 1; - shapeMask_2 = 1; - end - - - particleVolume(iRef) = gather(mean([particleFraction1,particleFraction2])); - pV1 = gather(pV1.*volMask{1}); - pV2 = gather(pV2.*volMask{2}); - - - + + [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(img1), pixelSize, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); + [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(img2), pixelSize, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); + + if (shape_mask_test) + fprintf('\nSaving your masks and exiting!\n'); + SAVE_IMG(shapeMask_1,sprintf('%s-shape_mask_%2.2f_lowpass_%2.2f_threshold.mrc', ... + outputPrefix, shape_mask_lowpass,shape_mask_threshold),pixelSize); + return; + end + + + if (flgFscShapeMask) + shapeMask_1 = gather((shapeMask_1.*volMask{1}).^flgFscShapeMask); + shapeMask_2 = gather((shapeMask_2.*volMask{2}).^flgFscShapeMask); + else + shapeMask_1 = 1; + shapeMask_2 = 1; + end + + + particleVolume(iRef) = gather(mean([particleFraction1,particleFraction2])); + pV1 = gather(pV1.*volMask{1}); + pV2 = gather(pV2.*volMask{2}); + + + halfSet = 'OUT'; else halfSet = 'STD'; end - - - imgFilt1 = BH_bandLimitCenterNormalize(img1.*volMask{1}.*shapeMask_1,bandpassFilt{iGold}, ... - (volMask{1} > 0.01),padCalc, 'single'); - imgFilt1 = real(ifftn(imgFilt1)); - imgFilt1 = single(gather(imgFilt1(padCalc(1,1)+1 : end - padCalc(2,1), ... - padCalc(1,2)+1 : end - padCalc(2,2), ... - padCalc(1,3)+1 : end - padCalc(2,3) ))); - imgFilt2 = BH_bandLimitCenterNormalize(img2.*volMask{2}.*shapeMask_2,bandpassFilt{iGold}, ... - (volMask{2} > 0.01),padCalc, 'single'); - imgFilt2 = real(ifftn(imgFilt2)); - imgFilt2 = single(gather(imgFilt2(padCalc(1,1)+1 : end - padCalc(2,1), ... - padCalc(1,2)+1 : end - padCalc(2,2), ... - padCalc(1,3)+1 : end - padCalc(2,3) ))); - % Should replace these with a call to padZeros3d - img1 = img1(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3) ); - - img2 = img2(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3) ); - + + + imgFilt1 = BH_bandLimitCenterNormalize(img1.*volMask{1}.*shapeMask_1,bandpassFilt{iGold}, ... + (volMask{1} > 0.01),padCalc, 'single'); + imgFilt1 = real(ifftn(imgFilt1)); + imgFilt1 = single(gather(imgFilt1(padCalc(1,1)+1 : end - padCalc(2,1), ... + padCalc(1,2)+1 : end - padCalc(2,2), ... + padCalc(1,3)+1 : end - padCalc(2,3) ))); + imgFilt2 = BH_bandLimitCenterNormalize(img2.*volMask{2}.*shapeMask_2,bandpassFilt{iGold}, ... + (volMask{2} > 0.01),padCalc, 'single'); + imgFilt2 = real(ifftn(imgFilt2)); + imgFilt2 = single(gather(imgFilt2(padCalc(1,1)+1 : end - padCalc(2,1), ... + padCalc(1,2)+1 : end - padCalc(2,2), ... + padCalc(1,3)+1 : end - padCalc(2,3) ))); + % Should replace these with a call to padZeros3d + img1 = img1(padWindow(1,1)+1 : end - padWindow(2,1), ... + padWindow(1,2)+1 : end - padWindow(2,2), ... + padWindow(1,3)+1 : end - padWindow(2,3) ); + + img2 = img2(padWindow(1,1)+1 : end - padWindow(2,1), ... + padWindow(1,2)+1 : end - padWindow(2,2), ... + padWindow(1,3)+1 : end - padWindow(2,3) ); + % Save a temp copy of the aligned references to visualy inspect the results SAVE_IMG(MRCImage(imgFilt1), sprintf('./FSC/fscTmp_%d_ODD.mrc',iRef)); - SAVE_IMG(MRCImage(imgFilt2), sprintf('./FSC/fscTmp_%d_EVE.mrc',iRef)); + SAVE_IMG(MRCImage(imgFilt2), sprintf('./FSC/fscTmp_%d_EVE.mrc',iRef)); clear imgFilt1 imgFilt2 [ fscPAD ] = BH_multi_padVal(size(img1), padDIM(1)); [ rad,~,~,~,~,~ ] = BH_multi_gridCoordinates(padDIM.*[1,1,1], 'Cartesian', 'GPU', ... - {'none'}, 1, 0, 1 ); + {'none'}, 1, 0, 1 ); rad = single(rad)./pixelSize; if (flgFscShapeMask) @@ -580,9 +580,9 @@ % TODO add a flag since the phase randomized is not used in practice %!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - % Only need to calculate phase randomized masks if the fscShapeMask is - % applied during the FSC calculation. If instead it is used to estimate - % the particle volume (flgEstSolvent) then no mask is directly applied. + % Only need to calculate phase randomized masks if the fscShapeMask is + % applied during the FSC calculation. If instead it is used to estimate + % the particle volume (flgEstSolvent) then no mask is directly applied. fscRandCutoffRes = 3*masterTM.currentResForDefocusError(1); lowResShift = pixelSize*2 - 10; if lowResShift <= 0 @@ -598,42 +598,42 @@ fscTcutoff = (floor((1/(fscRandCutoffRes*.95)-lowResShift)/ shellInc)) * shellInc; %Calculate the fsc on phase randomized masked volumes. [randGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(padDIM.*[1,1,1], 'Cartesian', ... - 'cpu', {'none'}, 1, 0, 1 ); + 'cpu', {'none'}, 1, 0, 1 ); randGrid = single(randGrid./pixelSize); randLowRES = (randGrid < randCutoff); randHighRES = (randGrid >= randCutoff); - - + + clear randGrid [ fou1 ] = fftn(BH_padZeros3d(img1, fscPAD(1,:), fscPAD(2,:), 'cpu', 'singleTaper')); - - for iShuffle = 1 + + for iShuffle = 1 rng('shuffle') fou1 = single(real(ifftn(abs(fou1) .* exp(1i .* (... - randLowRES .* angle(fou1) + ... - randHighRES .* pi.*(rand(size(fou1),'single'))))))); + randLowRES .* angle(fou1) + ... + randHighRES .* pi.*(rand(size(fou1),'single'))))))); end - - + + [ fou2 ] = fftn(BH_padZeros3d(img2 , fscPAD(1,:), fscPAD(2,:), 'cpu', 'singleTaper')); for iShuffle = 1 rng('shuffle') - fou2 = single(real(ifftn(abs(fou2) .* exp(1i .* (... - randLowRES .* angle(fou2) + ... - randHighRES .* pi.*(rand(size(fou2),'single'))))))); - + fou2 = single(real(ifftn(abs(fou2) .* exp(1i .* (... + randLowRES .* angle(fou2) + ... + randHighRES .* pi.*(rand(size(fou2),'single'))))))); + end - - + + clear randLowRES randHighRES fou1 = fftn(fou1.*BH_padZeros3d(pV1, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); fou2 = fftn(fou2.*BH_padZeros3d(pV2 , fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); - + [shellsRandFreq, shellsRandFSC, ~,~] = ... - calc_shells(fou1, fou2, rad, pixelSize, coneList,'rand'); + calc_shells(fou1, fou2, rad, pixelSize, coneList,'rand'); clear fou1 fou2 else @@ -641,7 +641,7 @@ shapeMask_1 = 1.*volMask{1}; shapeMask_2 = 1.*volMask{2}; end - + img1 = img1 - mean(img1(shapeMask_1>0.01)); img1 = img1 ./ rms(img1(shapeMask_1>0.01)); img1 = img1 .* shapeMask_1; @@ -652,25 +652,25 @@ SAVE_IMG(MRCImage(single(gather(img1))), sprintf('./FSC/fscTmp_%d_noFilt_ODD.mrc',iRef)); SAVE_IMG(MRCImage(single(gather(img2))), sprintf('./FSC/fscTmp_%d_noFilt_EVE.mrc',iRef)); - + [ img1 ] = BH_padZeros3d(img1 , fscPAD(1,:), fscPAD(2,:), 'GPU', 'singleTaper'); fou1 = fftn(img1); %clear img1 [ img2 ] = BH_padZeros3d(img2 , fscPAD(1,:), fscPAD(2,:), 'GPU', 'singleTaper'); fou2 = fftn(img2); %clear img2 [shellsFreq, shellsFSC, shellsNUM,shellsPOWER] = ... - calc_shells(fou1, fou2, rad, pixelSize,coneList, halfAngle); + calc_shells(fou1, fou2, rad, pixelSize,coneList, halfAngle); clear fou1 fou2 if (flgFscShapeMask) - fou1 = fftn(img1.*BH_padZeros3d(pV1, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); - clear pv1 - fou2 = fftn(img2.*BH_padZeros3d(pV2, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); + fou1 = fftn(img1.*BH_padZeros3d(pV1, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); + clear pv1 + fou2 = fftn(img2.*BH_padZeros3d(pV2, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); [tightFreq, tightFSC,~,~] = ... - calc_shells(fou1, fou2, rad, pixelSize,coneList, halfAngle); - fitTightFSC = csape(tightFreq(:,1),tightFSC(:,1),'variational'); - clear pv2 + calc_shells(fou1, fou2, rad, pixelSize,coneList, halfAngle); + fitTightFSC = csape(tightFreq(:,1),tightFSC(:,1),'variational'); + clear pv2 end clear img1 img2 - + for iFSC = 1:size(shellsFSC,2) f = 2.* particleVolume(iRef); @@ -680,60 +680,60 @@ fscParticle = f.*fscUnMasked ./ (1 + (f-1).*abs(fscUnMasked)); shellsFSC(1:firstZero,iFSC) = fscParticle(1:firstZero); end - - % Oversampled curve - osX = [0:0.001:0.5]'./pixelSize; - - + % Oversampled curve + osX = [0:0.001:0.5]'./pixelSize; + + + forceMask = cell(nCones+1,1); forceMaskAlign = cell(nCones+1,1); fitFSC = cell(nCones+1,1); fitNUM = cell(nCones+1,1); oneBitCut = zeros(nCones+1,1); halfBitCut = zeros(nCones+1,1); - - - - for iCone = 1:nCones+1 - fitFSC{iCone} = csape(shellsFreq(:,iCone),shellsFSC(:,iCone),'variational'); - fitNUM{iCone} = csape(shellsFreq(:,iCone),shellsNUM(:,iCone)); - end - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% test save masking curve + + + + for iCone = 1:nCones+1 + fitFSC{iCone} = csape(shellsFreq(:,iCone),shellsFSC(:,iCone),'variational'); + fitNUM{iCone} = csape(shellsFreq(:,iCone),shellsNUM(:,iCone)); + end + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% test save masking curve if (flgFscShapeMask) - - + + transitionFreq = find(osX > fscTcutoff,1,'first'); fscRand = csape( shellsRandFreq(:,1), shellsRandFSC(:,1), 'variational'); - + fscDiff = csape( osX, (fnval(fitTightFSC,osX)-fnval(fscRand,osX))./(1-fnval(fscRand,osX)), 'variational'); fscTrue = csape( osX, [fnval(fitTightFSC,osX(1:transitionFreq));fnval(fscDiff,osX(transitionFreq+1:end))],'variational'); - - SAVE_IMG(MRCImage(single(gather(shapeMask_1))), ... - sprintf('%s-%d-shapeMask_%d.mrc', outputPrefix, iRef, 1)); - - SAVE_IMG(MRCImage(single(gather(shapeMask_2))), ... - sprintf('%s-%d-shapeMask_%d.mrc', outputPrefix, iRef, 2)); - if (minimumParticleVolume < 1) - % Only save if used. - SAVE_IMG(MRCImage(single(gather(pV1))), ... - sprintf('%s-%d-particleVolEst_%d.mrc', outputPrefix, iRef, 1)); - - SAVE_IMG(MRCImage(single(gather(pV2))), ... - sprintf('%s-%d-particleVolEst_%d.mrc', outputPrefix, iRef, 2)); - end - - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%% - - + + SAVE_IMG(MRCImage(single(gather(shapeMask_1))), ... + sprintf('%s-%d-shapeMask_%d.mrc', outputPrefix, iRef, 1)); + + SAVE_IMG(MRCImage(single(gather(shapeMask_2))), ... + sprintf('%s-%d-shapeMask_%d.mrc', outputPrefix, iRef, 2)); + if (minimumParticleVolume < 1) + % Only save if used. + SAVE_IMG(MRCImage(single(gather(pV1))), ... + sprintf('%s-%d-particleVolEst_%d.mrc', outputPrefix, iRef, 1)); + + SAVE_IMG(MRCImage(single(gather(pV2))), ... + sprintf('%s-%d-particleVolEst_%d.mrc', outputPrefix, iRef, 2)); + end + + end + %%%%%%%%%%%%%%%%%%%%%%%%%%%% + + whiteningFilter = 1; % Initial pass at the 1bit and 1/2bit curves, for now set the D/L parameter ast % 2/3 as planned for first cycle. Later use (volumeEst)^1/3 as saved from % cRef_Vnorm. - + DbyL = 2/3; % This should be the same for both half sets if ~(flgEstSNR) @@ -751,19 +751,19 @@ end nCones nEffective = fnval(fitNUM{1},osX).*(3/2.*DbyL).^2 ./ (2.*gridSearch.number_of_asymmetric_units); - + oneBIT = ( 0.5+2.4142./sqrt(nEffective) ) ./ ... - ( 1.5 + 1.4142./sqrt(nEffective) ); - - + ( 1.5 + 1.4142./sqrt(nEffective) ); + + halfBIT= ( 0.207+1.9102./sqrt(nEffective) ) ./ ... - ( 1.2071 + 0.9102./sqrt(nEffective) ); - - aliBIT = mean([oneBIT,halfBIT]); + ( 1.2071 + 0.9102./sqrt(nEffective) ); + + aliBIT = mean([oneBIT,halfBIT]); % Find the two common cutoff values -- need a better way to determine the second % value that handles non-monotonic curves and is still smooth/gentle without % falling off too slowly. - + lowCut1 = find(fnval(fitFSC{1},osX) <= 0.143 & osX > 1/100, 1, 'first'); try @@ -776,145 +776,145 @@ catch halfBitCut(1) = find(osX .* pixelSize > 0.425, 1, 'first'); end - -% Particularly for working at higher binning, this allows using the full -% frequency range, which is the most information/calc. -if isempty(lowCut1) - lowCut1 = find(osX .* pixelSize > 0.425, 1, 'first'); -end - - - -% The value 0.005 looked steep but not too steep when I did a first test. This -% should be chosen with some more consideration. -forceMask{1} = exp(-0.005.^-2 .* (osX-osX(halfBitCut(1))).^2); -forceMask{1}(1:halfBitCut(1)) = 1; - -% Make a more conservative taper for alignment, forcing the cRef curve to zero -% starting from ssnr 2 -lowCutAlign = find(fnval(fitFSC{1},osX) <= 1/2 & osX > 1/100, 1, 'first'); -% Particularly for working at higher binning, this allows using the full -% frequency range, which is the most information/calc. -if isempty(lowCutAlign) - lowCutAlign = find(osX .* pixelSize > 0.425, 1, 'first'); -end - -forceMaskAlign{1} = exp(-0.005.^-2 .* (osX-osX(oneBitCut(1))).^2); -forceMaskAlign{1}(1:oneBitCut(1)) = 1; - -lowestRes = 0; -highestRes = 0; - -if (flgCones) - lowestRes = 1./osX(lowCut1); - highestRes = 1./osX(lowCut1); - for iCone = 1:nCones - - forceMask{iCone+1} = ones(size(osX)); - lowCut1 = find(fnval(fitFSC{iCone+1},osX) <= 0.143 & osX > 1/100, 1, 'first'); - try - oneBitCut(iCone+1) = find(fnval(fitFSC{iCone+1},osX)- aliBIT < 0 & osX > 1/100, 1, 'first'); - catch - oneBitCut(iCone+1) = find(osX .* pixelSize > 0.425, 1, 'first'); + + % Particularly for working at higher binning, this allows using the full + % frequency range, which is the most information/calc. + if isempty(lowCut1) + lowCut1 = find(osX .* pixelSize > 0.425, 1, 'first'); + end + + + + % The value 0.005 looked steep but not too steep when I did a first test. This + % should be chosen with some more consideration. + forceMask{1} = exp(-0.005.^-2 .* (osX-osX(halfBitCut(1))).^2); + forceMask{1}(1:halfBitCut(1)) = 1; + + % Make a more conservative taper for alignment, forcing the cRef curve to zero + % starting from ssnr 2 + lowCutAlign = find(fnval(fitFSC{1},osX) <= 1/2 & osX > 1/100, 1, 'first'); + % Particularly for working at higher binning, this allows using the full + % frequency range, which is the most information/calc. + if isempty(lowCutAlign) + lowCutAlign = find(osX .* pixelSize > 0.425, 1, 'first'); + end + + forceMaskAlign{1} = exp(-0.005.^-2 .* (osX-osX(oneBitCut(1))).^2); + forceMaskAlign{1}(1:oneBitCut(1)) = 1; + + lowestRes = 0; + highestRes = 0; + + if (flgCones) + lowestRes = 1./osX(lowCut1); + highestRes = 1./osX(lowCut1); + for iCone = 1:nCones + + forceMask{iCone+1} = ones(size(osX)); + lowCut1 = find(fnval(fitFSC{iCone+1},osX) <= 0.143 & osX > 1/100, 1, 'first'); + try + oneBitCut(iCone+1) = find(fnval(fitFSC{iCone+1},osX)- aliBIT < 0 & osX > 1/100, 1, 'first'); + catch + oneBitCut(iCone+1) = find(osX .* pixelSize > 0.425, 1, 'first'); + end + try + halfBitCut(iCone+1)= find(fnval(fitFSC{iCone+1},osX)-halfBIT < 0 & osX > 1/100, 1, 'first'); + catch + halfBitCut(iCone+1) = find(osX .* pixelSize > 0.425, 1, 'first'); + end + + % first try to use 0.5, if not default to 0.425 cyc/pix + if isempty(lowCut1) + lowCut1 = find(fnval(fitFSC{iCone+1},osX) <= 0.5 & osX > 1/100, 1, 'first'); + end + if isempty(lowCut1) + lowCut1 = find(osX .* pixelSize > 0.425, 1, 'first'); + end + + if lowestRes < 1./osX(lowCut1) + lowestRes = 1./osX(lowCut1); + elseif highestRes > 1./ osX(lowCut1) + highestRes = 1./osX(lowCut1); + end + + forceMask{iCone+1} = exp(-0.005.^-2 .* (osX-osX(lowCut1)).^2); + forceMask{iCone+1}(1:lowCut1) = 1; + + forceMaskAlign{iCone+1} = exp(-0.005.^-2 .* (osX-osX(oneBitCut(iCone+1))).^2); + forceMaskAlign{iCone+1}(1:oneBitCut(iCone+1)) = 1; end - try - halfBitCut(iCone+1)= find(fnval(fitFSC{iCone+1},osX)-halfBIT < 0 & osX > 1/100, 1, 'first'); - catch - halfBitCut(iCone+1) = find(osX .* pixelSize > 0.425, 1, 'first'); + end + + % Calculate cRef as in Rosenthal & Henderson 2003 + + cRef = cell(nCones+1,1); + cRefAli= cell(nCones+1,1); + + cRefCurve = sqrt( abs(2.*fnval(fitFSC{1},osX)./(1+fnval(fitFSC{1},osX))) ) ... + .* forceMask{1} .* whiteningFilter; + + cRef{1} = fit(osX, cRefCurve./max(cRefCurve(:)),'cubicSpline'); + + cRefCurve = sqrt( abs(2.*fnval(fitFSC{1},osX)./(1+fnval(fitFSC{1},osX))) ) ... + .* forceMaskAlign{1} .* whiteningFilter; + cRefAli{1} = fit(osX,cRefCurve./max(cRefCurve(:)),'cubicSpline'); + if (flgCones) + for iCone = 1:nCones + + cRefCurve = sqrt( abs(2.*fnval(fitFSC{iCone+1},osX) ./ ... + (1+fnval(fitFSC{iCone+1},osX))) ) .* ... + forceMask{iCone+1} .* whiteningFilter; + + cRef{iCone+1} = fit(osX,cRefCurve./max(cRefCurve(:)),'cubicSpline'); + + cRefCurve = sqrt( abs(2.*fnval(fitFSC{iCone+1},osX) ./ ... + (1+fnval(fitFSC{iCone+1},osX))) ) .* ... + forceMaskAlign{iCone+1} .* whiteningFilter; + + cRefAli{iCone+1} = fit(osX,cRefCurve./max(cRefCurve(:)),'cubicSpline'); + end + end + + + % Save the fsc info, for calculating cRef. For some reason, when compiled, fit + % function can run, but storing a cFit object in a struct or cell doesn't seem + % to work? + if ~(flgJustFSC) + masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRef)) = ... + {shellsFreq,shellsFSC,{cRef,cRefAli,bh_global_MTF},osX,forceMaskAlign,forceMask,nCones,coneList,halfAngle,samplingRate}; - % first try to use 0.5, if not default to 0.425 cyc/pix - if isempty(lowCut1) - lowCut1 = find(fnval(fitFSC{iCone+1},osX) <= 0.5 & osX > 1/100, 1, 'first'); - end - if isempty(lowCut1) - lowCut1 = find(osX .* pixelSize > 0.425, 1, 'first'); - end - - if lowestRes < 1./osX(lowCut1) - lowestRes = 1./osX(lowCut1); - elseif highestRes > 1./ osX(lowCut1) - highestRes = 1./osX(lowCut1); + sprintf('Resample_%s%d',savePrefix,iRef) + + if (flgEstSNR) + masterTM.(cycleNumber).('fitFSC').(sprintf('fit%s%d',savePrefix,1))=... + [bestAnglesTotal(1,1:9); ... + bestAnglesTotal(1,10:12),0,0,0,0,0,0]; + else + masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRef))=... + [bestAnglesTotal(iRef,1:9); ... + bestAnglesTotal(iRef,10:12),0,0,0,0,0,0]; end - - forceMask{iCone+1} = exp(-0.005.^-2 .* (osX-osX(lowCut1)).^2); - forceMask{iCone+1}(1:lowCut1) = 1; - - forceMaskAlign{iCone+1} = exp(-0.005.^-2 .* (osX-osX(oneBitCut(iCone+1))).^2); - forceMaskAlign{iCone+1}(1:oneBitCut(iCone+1)) = 1; - end -end - -% Calculate cRef as in Rosenthal & Henderson 2003 - -cRef = cell(nCones+1,1); -cRefAli= cell(nCones+1,1); - -cRefCurve = sqrt( abs(2.*fnval(fitFSC{1},osX)./(1+fnval(fitFSC{1},osX))) ) ... - .* forceMask{1} .* whiteningFilter; - -cRef{1} = fit(osX, cRefCurve./max(cRefCurve(:)),'cubicSpline'); - -cRefCurve = sqrt( abs(2.*fnval(fitFSC{1},osX)./(1+fnval(fitFSC{1},osX))) ) ... - .* forceMaskAlign{1} .* whiteningFilter; -cRefAli{1} = fit(osX,cRefCurve./max(cRefCurve(:)),'cubicSpline'); -if (flgCones) - for iCone = 1:nCones - - cRefCurve = sqrt( abs(2.*fnval(fitFSC{iCone+1},osX) ./ ... - (1+fnval(fitFSC{iCone+1},osX))) ) .* ... - forceMask{iCone+1} .* whiteningFilter; - - cRef{iCone+1} = fit(osX,cRefCurve./max(cRefCurve(:)),'cubicSpline'); - - cRefCurve = sqrt( abs(2.*fnval(fitFSC{iCone+1},osX) ./ ... - (1+fnval(fitFSC{iCone+1},osX))) ) .* ... - forceMaskAlign{iCone+1} .* whiteningFilter; - - cRefAli{iCone+1} = fit(osX,cRefCurve./max(cRefCurve(:)),'cubicSpline'); - - end -end - - -% Save the fsc info, for calculating cRef. For some reason, when compiled, fit -% function can run, but storing a cFit object in a struct or cell doesn't seem -% to work? -if ~(flgJustFSC) - masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRef)) = ... - {shellsFreq,shellsFSC,{cRef,cRefAli,bh_global_MTF},osX,forceMaskAlign,forceMask,nCones,coneList,halfAngle,samplingRate}; - - sprintf('Resample_%s%d',savePrefix,iRef) - - if (flgEstSNR) - masterTM.(cycleNumber).('fitFSC').(sprintf('fit%s%d',savePrefix,1))=... - [bestAnglesTotal(1,1:9); ... - bestAnglesTotal(1,10:12),0,0,0,0,0,0]; - else - masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRef))=... - [bestAnglesTotal(iRef,1:9); ... - bestAnglesTotal(iRef,10:12),0,0,0,0,0,0]; - end - - masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRef)) = ... - {maskType, sizeMask, ... - maskRadius, maskCenter, ... - flgFscShapeMask, shape_mask_lowpass, shape_mask_threshold}; - - masterTM.('currentResForDefocusError') = osX(oneBitCut).^-1; -end - - -imgOUT = STAGEofALIGNMENT; - -clear fout famp1 famp2 fphase1 fphase2 - - - fmid = osX(find(fnval(fitFSC{1},osX) < 0.5 & osX > 1/100, 1, 'first')) - fgold = osX(find(fnval(fitFSC{1},osX) < 0.143 & osX > 1/100, 1, 'first')) - fprintf('\n0.5 = 1/%f\n0.143 = 1/%f\n', fmid, fgold) - + masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRef)) = ... + {maskType, sizeMask, ... + maskRadius, maskCenter, ... + flgFscShapeMask, shape_mask_lowpass, shape_mask_threshold}; + + masterTM.('currentResForDefocusError') = osX(oneBitCut).^-1; + end + + + imgOUT = STAGEofALIGNMENT; + + clear fout famp1 famp2 fphase1 fphase2 + + + fmid = osX(find(fnval(fitFSC{1},osX) < 0.5 & osX > 1/100, 1, 'first')) + fgold = osX(find(fnval(fitFSC{1},osX) < 0.143 & osX > 1/100, 1, 'first')) + + fprintf('\n0.5 = 1/%f\n0.143 = 1/%f\n', fmid, fgold) + fscOUT = fopen(sprintf('%s-%d-fsc_%s.txt', outputPrefix, iRef, halfSet),'w'); % This seems wildly unecessary, but adding the new fscFull print section @@ -926,7 +926,7 @@ fscMat(:,iCol+1) = fnval(fitFSC{iCol},osX); end fscMat = fscMat'; - nRow = 1; + nRow = 1; nTot = 1; while nTot < numel(fscMat) while nRow <= nCol + 1 @@ -938,40 +938,40 @@ fprintf(fscOUT,'\n'); end fclose(fscOUT); - - + + if (flgCones) - - figure('Visible','off'), plot(osX,fnval(fitFSC{1},osX),'kd','MarkerSize',2.5); hold on; - plot(osX, oneBIT,'c'); - plot(osX, halfBIT,'b'); - plot(osX, 0.*osX,'k'); - plot(osX,fnval(fitFSC{2},osX),'k--'); - for iCone = 3:length(fitFSC) - plot(osX,fnval(fitFSC{iCone},osX),'k--'); - end - -% title({'FSC',sprintf('0.5 %3.2f\n0.143 %3.2f',1./fmid,1./fgold)}); - title({'FSC',sprintf('0.5 - %3.2f\n0.143 - %3.2f (%3.2f-%3.2f)\noneBit %3.2f\n halfBit %3.2f\n',1./fmid,1./fgold,lowestRes,highestRes,osX(oneBitCut(1)).^-1,osX(halfBitCut(1)).^-1)}); - + + figure('Visible','off'), plot(osX,fnval(fitFSC{1},osX),'kd','MarkerSize',2.5); hold on; + plot(osX, oneBIT,'c'); + plot(osX, halfBIT,'b'); + plot(osX, 0.*osX,'k'); + plot(osX,fnval(fitFSC{2},osX),'k--'); + for iCone = 3:length(fitFSC) + plot(osX,fnval(fitFSC{iCone},osX),'k--'); + end + + % title({'FSC',sprintf('0.5 %3.2f\n0.143 %3.2f',1./fmid,1./fgold)}); + title({'FSC',sprintf('0.5 - %3.2f\n0.143 - %3.2f (%3.2f-%3.2f)\noneBit %3.2f\n halfBit %3.2f\n',1./fmid,1./fgold,lowestRes,highestRes,osX(oneBitCut(1)).^-1,osX(halfBitCut(1)).^-1)}); + xlabel('Spatial Freq'); ylabel('fsc'); legend('FSC','oneBit','halfBit','Location', ... - 'northeast','Orientation','vertical'); - ylim([-.05 1.025]) + 'northeast','Orientation','vertical'); + ylim([-.05 1.025]) else figure('Visible','off'), plot(osX,fnval(fitFSC{1},osX),'k',... - osX, oneBIT, 'c', ... - osX, halfBIT, 'b', ... - osX, 0.*osX,'k'); - title({'FSC',sprintf('0.5 %3.2f\n0.143 %3.2f\noneBit %3.2f\n halfBit %3.2f\n',1./fmid,1./fgold,osX(oneBitCut(1)).^-1,osX(halfBitCut(1)).^-1)}); + osX, oneBIT, 'c', ... + osX, halfBIT, 'b', ... + osX, 0.*osX,'k'); + title({'FSC',sprintf('0.5 %3.2f\n0.143 %3.2f\noneBit %3.2f\n halfBit %3.2f\n',1./fmid,1./fgold,osX(oneBitCut(1)).^-1,osX(halfBitCut(1)).^-1)}); xlabel('Spatial Freq'); ylabel('fsc'); legend('FSC','oneBit','halfBit','Location', ... - 'northeast','Orientation','vertical'); - ylim([-.05 1.025]) - + 'northeast','Orientation','vertical'); + ylim([-.05 1.025]) + end - + file_out = sprintf('%s-%d-fsc_%s', outputPrefix, iRef, halfSet); saveas(gcf, file_out,'pdf') @@ -979,56 +979,56 @@ % For comparing results from a tight mask and solvent normalized fsc % The tight mask calculation is not so reliable (which is by it isn't used in the first place. % This is used for the figure S3 in the Nature Methods paper, but not in regular us. - figure('Visible','off'), plot(osX,fnval(fitTightFSC,osX),'k-.',... - osX,fnval(fscTrue,osX),'k',... - osX,fnval(fitFSC{1},osX),'b',... - osX, 0.*osX+0.143, 'k--',... - osX,zeros(length(osX)),'k'); - - fTightGold = osX(find(fnval(fscTrue,osX) < 0.143 & osX > 1/100, 1, 'first')); - - title({'FSC',sprintf('0.143 - %3.2f, %3.2f\n(paritcleVolume,tightMask)',1./fgold,1./fTightGold)}); - xlabel('Spatial Freq'); ylabel('fsc'); - ylim([-.05 1.025]) - file_out = sprintf('%s-%d-fscFull_%s', outputPrefix, iRef, halfSet); - % saveas(gcf, file_out,'pdf') - - % fscRandOUT = fopen(sprintf('%s-%d-fscFull_%s.txt', outputPrefix, iRef, halfSet),'w'); - % fprintf(fscRandOUT,'%4.4f\t%4.4f\t%4.4f\t%4.4f\n',[osX,fnval(fitTightFSC,osX),fnval(fscTrue,osX),fnval(fitFSC{1},osX)]'); - % fclose(fscRandOUT); - figure('Visible','off'), plot(osX,cRef{1}(osX),'kd','MarkerSize',3); hold on; - if (flgCones) - - plot(osX,cRef{2}(osX),'k--'); - for iCone = 3:length(cRef) - plot(osX,cRef{iCone}(osX),'k--'); - end + figure('Visible','off'), plot(osX,fnval(fitTightFSC,osX),'k-.',... + osX,fnval(fscTrue,osX),'k',... + osX,fnval(fitFSC{1},osX),'b',... + osX, 0.*osX+0.143, 'k--',... + osX,zeros(length(osX)),'k'); + + fTightGold = osX(find(fnval(fscTrue,osX) < 0.143 & osX > 1/100, 1, 'first')); + + title({'FSC',sprintf('0.143 - %3.2f, %3.2f\n(paritcleVolume,tightMask)',1./fgold,1./fTightGold)}); + xlabel('Spatial Freq'); ylabel('fsc'); + ylim([-.05 1.025]) + file_out = sprintf('%s-%d-fscFull_%s', outputPrefix, iRef, halfSet); + % saveas(gcf, file_out,'pdf') + + % fscRandOUT = fopen(sprintf('%s-%d-fscFull_%s.txt', outputPrefix, iRef, halfSet),'w'); + % fprintf(fscRandOUT,'%4.4f\t%4.4f\t%4.4f\t%4.4f\n',[osX,fnval(fitTightFSC,osX),fnval(fscTrue,osX),fnval(fitFSC{1},osX)]'); + % fclose(fscRandOUT); + figure('Visible','off'), plot(osX,cRef{1}(osX),'kd','MarkerSize',3); hold on; + if (flgCones) + + plot(osX,cRef{2}(osX),'k--'); + for iCone = 3:length(cRef) + plot(osX,cRef{iCone}(osX),'k--'); end - - title({'cRef',sprintf('0.5 %3.2f\n0.143 %3.2f',1./fmid,1./fgold)}); - xlabel('Spatial Freq'); ylabel('fsc'); - legend('cRef','Location', ... - 'northeastoutside','Orientation','vertical'); - ylim([-.05 1.025]) - - file_out = sprintf('%s-%d-cRef_%s', outputPrefix, iRef, halfSet); - saveas(gcf, file_out,'pdf') - figure('Visible','off'), plot(osX,cRefAli{1}(osX),'kd','MarkerSize',3); hold on; - if (flgCones) - - plot(osX,cRefAli{2}(osX),'k--'); - for iCone = 3:length(cRefAli) - plot(osX,cRefAli{iCone}(osX),'k--'); - end - end - title({'cRefAli',sprintf('0.5 %3.2f\n0.143 %3.2f',1./fmid,1./fgold)}); - xlabel('Spatial Freq'); ylabel('fsc'); - legend('cRefAli','Location', ... - 'northeastoutside','Orientation','vertical'); - ylim([-.05 1.025]) - - file_out = sprintf('%s-%d-cRefAli_%s', outputPrefix, iRef, halfSet); - saveas(gcf, file_out,'pdf') + end + + title({'cRef',sprintf('0.5 %3.2f\n0.143 %3.2f',1./fmid,1./fgold)}); + xlabel('Spatial Freq'); ylabel('fsc'); + legend('cRef','Location', ... + 'northeastoutside','Orientation','vertical'); + ylim([-.05 1.025]) + + file_out = sprintf('%s-%d-cRef_%s', outputPrefix, iRef, halfSet); + saveas(gcf, file_out,'pdf') + figure('Visible','off'), plot(osX,cRefAli{1}(osX),'kd','MarkerSize',3); hold on; + if (flgCones) + + plot(osX,cRefAli{2}(osX),'k--'); + for iCone = 3:length(cRefAli) + plot(osX,cRefAli{iCone}(osX),'k--'); + end + end + title({'cRefAli',sprintf('0.5 %3.2f\n0.143 %3.2f',1./fmid,1./fgold)}); + xlabel('Spatial Freq'); ylabel('fsc'); + legend('cRefAli','Location', ... + 'northeastoutside','Orientation','vertical'); + ylim([-.05 1.025]) + + file_out = sprintf('%s-%d-cRefAli_%s', outputPrefix, iRef, halfSet); + saveas(gcf, file_out,'pdf') % % try % fitPower = fit(shellsFreq(:,1).^2,log(shellsPOWER(:,1)),'cubicSpline'); % LR = 10; @@ -1047,12 +1047,12 @@ % log(shellsPOWER(lowRes:midRes,1)),'poly1'); % plot1= true; % end - % if (endRes-midRes > 2) + % if (endRes-midRes > 2) % bFactorFIT2 = fit(shellsFreq(midRes:endRes,1).^2 , ... - % log(shellsPOWER(midRes:endRes,1)),'poly1'); + % log(shellsPOWER(midRes:endRes,1)),'poly1'); % plot2 = true; - % end - + % end + % figure('Visible','off'), plot(osX.^2,fitPower(osX.^2),'k'); hold on; % if (plot1) % plot(osX.^2,bFactorFIT1(osX.^2),'b--'); @@ -1061,7 +1061,7 @@ % bFactorFIT1.('p1') = 0; % end % if (plot2) - % plot(osX.^2,bFactorFIT2(osX.^2),'b--'); + % plot(osX.^2,bFactorFIT2(osX.^2),'b--'); % else % bFactorFIT2 = struct() % bFactorFIT2.('p1') = 0; @@ -1069,31 +1069,31 @@ % line([(1/LR)^2,(1/LR)^2], ... % [min(log(shellsPOWER(:,1))), ... % max(log(shellsPOWER(:,1)))], ... - % 'Color','k','LineStyle','--'); + % 'Color','k','LineStyle','--'); % line([(1/MR)^2,(1/MR)^2], ... % [min(log(shellsPOWER(:,1))), ... % max(log(shellsPOWER(:,1)))], ... - % 'Color','k','LineStyle','--'); + % 'Color','k','LineStyle','--'); % line([HR,HR], ... % [min(log(shellsPOWER(:,1))), ... % max(log(shellsPOWER(:,1)))], ... % 'Color','k','LineStyle','--'); % % plot(osX.^2,bFactorFIT2(osX.^2),'b--'); % % outCurve(:,1).^2,outCurve(:,8),'g'); - + % title({'Guinier Plot',sprintf('\nbFactor(%2.1f-%2.1f-%2.1f)\n %d,%d', ... % LR,MR,sqrt(1./HR),round(bFactorFIT1.p1*-4),... % round(bFactorFIT2.p1*-4))}); ... - % xlabel('1/Ang^2'),... + % xlabel('1/Ang^2'),... % ylabel('log(F)'); % ylim([0.95*min(log(shellsPOWER(:,1))),... % 1.05*max(log(shellsPOWER(:,1)))]) - + % legend('uncorrected','corrected','Location','northeast',... % 'Orientation', 'vertical'); % file_out = sprintf('%s-%d-guinier_%s', outputPrefix, iRef, halfSet); % savefig(gcf,file_out); - % saveas(gcf, file_out,'pdf') + % saveas(gcf, file_out,'pdf') % catch % fprintf('\nRan into some error in the guinier analysis.\n'); % fprintf('\nSince this is not critical, skipping and continue.\n'); @@ -1107,7 +1107,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% Make cones - +%%% Make cones - function [ coneMask ] = make_cones( coneAngles, padDIM, halfAngle) @@ -1120,32 +1120,32 @@ coneOrientation = BH_defineMatrix(coneAngles,'Bah','invVector'); [ radius,~,height,~,~,~ ] = ... - BH_multi_gridCoordinates( padDIM.*[1,1,1], ... - 'Cylindrical', ... - 'GPU', ... - {'single',... - coneOrientation,... - coneShift, ... - 'invVector',... - 1,1},... - 0, 0, 0 ); + BH_multi_gridCoordinates( padDIM.*[1,1,1], ... + 'Cylindrical', ... + 'GPU', ... + {'single',... + coneOrientation,... + coneShift, ... + 'invVector',... + 1,1},... + 0, 0, 0 ); coneMask = (rad2deg(atan2(radius,abs(height))) < halfAngle); - - -clear radius height + + +clear radius height end % end of the make cones function %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% Calc FSC +%%% Calc FSC function [shellsFreq, shellsFSC, shellsNsamples, shellsPOWER] = ... - calc_shells(fou1, fou2, rad, pixelSize, coneList, halfAngle) - + calc_shells(fou1, fou2, rad, pixelSize, coneList, halfAngle) + + - padDIM = size(fou1); if isempty(fou2) @@ -1175,7 +1175,7 @@ % Keep same binDiv but override to calc only the spherical FSC for the % randomized set. -if ~isa(halfAngle,'cell') +if ~isa(halfAngle,'cell') if strcmpi(halfAngle, 'rand') nIters = 1; end @@ -1196,37 +1196,37 @@ for iCalc = 1:nIters - if iCalc ==1 + if iCalc ==1 iConeMask = 1; else -% iConeMask = principleAxes{iCalc -1}; + % iConeMask = principleAxes{iCalc -1}; iConeMask = make_cones( coneList{iCalc-1}, padDIM, halfAngle{iCalc-1}); end - - for q = 1:bin - qdep=0; + for q = 1:bin + + qdep=0; iMask = gpuArray((q-1)*inc <= rad & rad < (q+qdep)*inc & iConeMask); shellsFreq(q, iCalc) = inc.*(q-1/2); - + a = real(sum((cross(iMask)))); - + if (flgPowerSpectrum) shellsFSC(q, iCalc) = (a ./ sum(iMask(:))); else b = sum((auto1(iMask).^2)); c = sum((auto2(iMask).^2)); shellsFSC(q, iCalc) = a ./ sqrt(b * c); - shellsNsamples(q, iCalc) = sum(iMask(:)); + shellsNsamples(q, iCalc) = sum(iMask(:)); % We want the Average Fourier Amplitudes for Guinier analysis, not the average power shellsPOWER(q,iCalc) = sum(auto1(iMask)+auto2(iMask))./(2*shellsNsamples(q,iCalc)); end - + end clear iConeMask -end +end shellsFreq = gather(shellsFreq); shellsFSC = gather(shellsFSC); shellsNsamples = gather(shellsNsamples); @@ -1240,41 +1240,41 @@ function writeOutPyAli() fOUT = fopen(sprintf('FSC/fitInMap.py'),'w'); fprintf(fOUT,['\n'... -'from sys import argv\n\n',... -'def fit_map_in_map(map1_path, map2_path, xformName,\n',... -' initial_map1_transform = None,\n',... -' map1_threshold = 3.0,\n',... -' ijk_step_size_min = 0.01,\n',... -' ijk_step_size_max = 1.5,\n',... -' max_steps = 5000,\n',... -' optimize_translation = True,\n',... -' optimize_rotation = True):\n',... -' from VolumeViewer import open_volume_file\n',... -' map1 = open_volume_file(map1_path)[0]\n',... -' map2 = open_volume_file(map2_path)[0]\n\n',... -' if initial_map1_transform:\n',... -' from Matrix import chimera_xform\n',... -' xf = chimera_xform(initial_map1_transform)\n',... -' map1.surface_model().openState.globalXform(xf)\n\n',... -' use_threshold = (map1_threshold != None)\n\n',... -' from FitMap.fitmap import map_points_and_weights, motion_to_maximum\n',... -' points, point_weights = map_points_and_weights(map1, use_threshold)\n\n',... -' move_tf, stats = motion_to_maximum(points, point_weights, map2, max_steps,\n',... -' ijk_step_size_min, ijk_step_size_max,\n',... -' optimize_translation, optimize_rotation)\n\n',... -' import Matrix\n',... -' if initial_map1_transform:\n',... -' move_tf = Matrix.multiply_matrices(move_tf, initial_map1_transform)\n\n',... -' f = open(xformName,''w'')\n',... -' for i in range(4):\n',... -' for j in range(3):\n',... -' f.write(''{:f}''.format(move_tf[j][i]))\n',... -' f.write("\\n")\n\n',... -' f.close()\n',... -' print move_tf\n',... -' tfs = Matrix.transformation_description(move_tf)\n',... -' print tfs\n\n',... -'t = fit_map_in_map(argv[1],argv[2],argv[3])\n']); + 'from sys import argv\n\n',... + 'def fit_map_in_map(map1_path, map2_path, xformName,\n',... + ' initial_map1_transform = None,\n',... + ' map1_threshold = 3.0,\n',... + ' ijk_step_size_min = 0.01,\n',... + ' ijk_step_size_max = 1.5,\n',... + ' max_steps = 5000,\n',... + ' optimize_translation = True,\n',... + ' optimize_rotation = True):\n',... + ' from VolumeViewer import open_volume_file\n',... + ' map1 = open_volume_file(map1_path)[0]\n',... + ' map2 = open_volume_file(map2_path)[0]\n\n',... + ' if initial_map1_transform:\n',... + ' from Matrix import chimera_xform\n',... + ' xf = chimera_xform(initial_map1_transform)\n',... + ' map1.surface_model().openState.globalXform(xf)\n\n',... + ' use_threshold = (map1_threshold != None)\n\n',... + ' from FitMap.fitmap import map_points_and_weights, motion_to_maximum\n',... + ' points, point_weights = map_points_and_weights(map1, use_threshold)\n\n',... + ' move_tf, stats = motion_to_maximum(points, point_weights, map2, max_steps,\n',... + ' ijk_step_size_min, ijk_step_size_max,\n',... + ' optimize_translation, optimize_rotation)\n\n',... + ' import Matrix\n',... + ' if initial_map1_transform:\n',... + ' move_tf = Matrix.multiply_matrices(move_tf, initial_map1_transform)\n\n',... + ' f = open(xformName,''w'')\n',... + ' for i in range(4):\n',... + ' for j in range(3):\n',... + ' f.write(''{:f}''.format(move_tf[j][i]))\n',... + ' f.write("\\n")\n\n',... + ' f.close()\n',... + ' print move_tf\n',... + ' tfs = Matrix.transformation_description(move_tf)\n',... + ' print tfs\n\n',... + 't = fit_map_in_map(argv[1],argv[2],argv[3])\n']); fclose(fOUT); diff --git a/statistics/BH_fscSplit.m b/statistics/BH_fscSplit.m index e186ea7b..3c6905c1 100755 --- a/statistics/BH_fscSplit.m +++ b/statistics/BH_fscSplit.m @@ -1,6 +1,6 @@ function [ GEOMETRY_SPLIT ] = BH_fscSplit( GEOMETRY, splitOnTomos, nPeaks ) %Randomly divide a data set into two halves for fsc-gold analysis. -% +% % % Input variables: % @@ -57,8 +57,8 @@ % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); - - + + nSubTomos = size(positionList,1); % Generate random half sets for fsc if (floorCeil) @@ -70,18 +70,18 @@ end -% % % % Add randomized positions for ML intialization -% % % if (nPeaks > 1) -% % % positionList = repmat(positionList,1,nPeaks); -% % % for iPeak = 1:nPeaks-1 -% % % -% % % for iRand = 1:size(positionList,1) -% % % positionList(iRand,[17:25] + 26*iPeak) = ... -% % % reshape(BH_defineMatrix('rand','Bah','inv')*reshape(positionList(iRand,[17:25]),3,3),1,9); -% % % end -% % % -% % % end -% % % end + % % % % Add randomized positions for ML intialization + % % % if (nPeaks > 1) + % % % positionList = repmat(positionList,1,nPeaks); + % % % for iPeak = 1:nPeaks-1 + % % % + % % % for iRand = 1:size(positionList,1) + % % % positionList(iRand,[17:25] + 26*iPeak) = ... + % % % reshape(BH_defineMatrix('rand','Bah','inv')*reshape(positionList(iRand,[17:25]),3,3),1,9); + % % % end + % % % + % % % end + % % % end if (splitOnTomos) positionList(:,7:26:26*nPeaks) = 1 + floorCeil; @@ -95,7 +95,7 @@ % Update geometry end - + geometry.(tomoList{iTomo}) = positionList; diff --git a/statistics/BH_localWiener2d.m b/statistics/BH_localWiener2d.m index 47377927..5998b196 100644 --- a/statistics/BH_localWiener2d.m +++ b/statistics/BH_localWiener2d.m @@ -1,8 +1,8 @@ function [filtIMG,lVar,lMean,particleMask] = BH_localWiener2d( IMAGE, bandPass,... - flgZeroMean,... - flgParticle,varargin) + flgZeroMean,... + flgParticle,varargin) %Calculate a locally adaptive wiener filter and smooth image -% +% % Use a gaussian disc with 2*stdDev = radius of area for calculating stats. % Estimate local noise variance as the average of all local variances @@ -47,10 +47,10 @@ if ( useGPU ) bandPassFilt = BH_bandpass3d([d1,d2,1],10^-6,bandPass(1),bandPass(2), ... - 'GPU',pixelSize); + 'GPU',pixelSize); else bandPassFilt = BH_bandpass3d([d1,d2,1],10^-6,bandPass(1),bandPass(2), ... - 'cpu',pixelSize); + 'cpu',pixelSize); end if ( flgMovie ) @@ -69,17 +69,17 @@ K = kurtosis(imgSUM(:)); if abs(K - 3) > 0.75 - - + + medKernel = min(9,max(3,ceil(sqrt(abs(K-3))))); % medfilt wants an odd kernel size medKernel = (medKernel + (1-mod(medKernel,2))); - + % For some stupid effing reason, medfilt2 returns an error if I pass the % medKernel directly, but will take a number of the same class -% fprintf(['Found strong outliers, kurtosis = %3.3f,',... -% 'running median filter size %d\n'],K,medKernel(1)); - + % fprintf(['Found strong outliers, kurtosis = %3.3f,',... + % 'running median filter size %d\n'],K,medKernel(1)); + switch medKernel case 3 imgSUM = medfilt2(imgSUM,[3,3]); @@ -90,8 +90,8 @@ case 9 imgSUM = medfilt2(imgSUM,[9,9]); end - - + + end @@ -154,7 +154,7 @@ if ( flgParticle || flgIterate ) - [ particleMask ] = calc_particleMask(wienerFilt.*imgSUM, useGPU,totalDose); + [ particleMask ] = calc_particleMask(wienerFilt.*imgSUM, useGPU,totalDose); else particleMask = ''; end @@ -164,33 +164,33 @@ lastRatio = lNoiseVar; iIter = 1; while lastRatio > 1.001 && iIter <= maxIter - if iIter == 1 + if iIter == 1 oldBestEst = lNoiseVar; - else + else oldBestEst = newBestEst; end newBestEst = mean(lVar(particleMask<0.05)); fprintf('Current noise variance estimates are %3.3e and now %3.3e\n',... - lNoiseVar, newBestEst); - + lNoiseVar, newBestEst); + if ( flgZeroMean ) wienerFilt = (max(lVar-newBestEst,0)./(lVar)).^2; else wienerFilt = lMean + (max(lVar-newBestEst,0))./lVar; end - - [ particleMask ] = calc_particleMask(wienerFilt.*imgSUM, useGPU,totalDose); - iIter = iIter + 1; - lastRatio = newBestEst/oldBestEst; + + [ particleMask ] = calc_particleMask(wienerFilt.*imgSUM, useGPU,totalDose); + iIter = iIter + 1; + lastRatio = newBestEst/oldBestEst; end - - + + end if ( flgMovie ) - + filtIMG = zeros([d1,d2,d3],'single'); - + for iPrj = 1:d3 if iPrj == 1 iIMG = real(ifftn(fftn(sum(IMAGE(:,:,2:end),3)).*bandPassFilt)); @@ -199,7 +199,7 @@ else iIMG = real(ifftn(fftn(sum(IMAGE(:,:,1:end-1),3)).*bandPassFilt)); end - % Scale the mean + % Scale the mean filtIMG(:,:,iPrj) = gather(wienerFilt.*(iIMG-(lMean.*((d3-1)/d3)))); filtIMG(:,:,iPrj) = filtIMG(:,:,iPrj) ./ rms(filtIMG(:,:,iPrj)); end @@ -208,57 +208,57 @@ filtIMG = filtIMG - mean(filtIMG(:)); filtIMG = filtIMG ./ rms(filtIMG(:)); end - -clear IMAGE imgSUM wienerFilt iIMG + +clear IMAGE imgSUM wienerFilt iIMG end function [ particleMask ] = calc_particleMask(filtIMG, useGPU,totalDose) - % Use the filtered image to make a simplified mask - %%% ADD adaptive cutoff based on number of initial pixels wanted (1% for - %%% example) - filtIMG = filtIMG ./ rms(filtIMG(:)); +% Use the filtered image to make a simplified mask +%%% ADD adaptive cutoff based on number of initial pixels wanted (1% for +%%% example) +filtIMG = filtIMG ./ rms(filtIMG(:)); % intCutoff = 3.0; - intCutoff = 0.5*sqrt(totalDose); - [gx] = BH_multi_gaussian2d(2.*[5,5],2.0,0); +intCutoff = 0.5*sqrt(totalDose); +[gx] = BH_multi_gaussian2d(2.*[5,5],2.0,0); - - if ( useGPU ) - gx = gpuArray(gx); - end - % avoid disconnected densitys by setting high initial cutoff - dilatedIMG = convn(filtIMG,gx,'same') > 0.01; %real(ifftn(fftn(filtIMG).*gx)); - binaryCut = (abs(filtIMG) .* dilatedIMG) > intCutoff; - sum(binaryCut(:))./numel(binaryCut); - - - % relax the cutoff and iteratively dilate the region - intCutoff = intCutoff./3 ; - growthFactor = 1.2; - while growthFactor > 1.05 - nStart = sum(binaryCut(:)); +if ( useGPU ) + gx = gpuArray(gx); +end - dilatedIMG = convn(binaryCut,gx,'same') > 0.01; %real(ifftn(fftn(binaryCut).*gx)); - binaryCut = ( abs(filtIMG) .* dilatedIMG ) > intCutoff; - growthFactor = sum(binaryCut(:))/nStart; - end - +% avoid disconnected densitys by setting high initial cutoff +dilatedIMG = convn(filtIMG,gx,'same') > 0.01; %real(ifftn(fftn(filtIMG).*gx)); +binaryCut = (abs(filtIMG) .* dilatedIMG) > intCutoff; +sum(binaryCut(:))./numel(binaryCut); + + +% relax the cutoff and iteratively dilate the region +intCutoff = intCutoff./3 ; +growthFactor = 1.2; +while growthFactor > 1.05 + nStart = sum(binaryCut(:)); + + dilatedIMG = convn(binaryCut,gx,'same') > 0.01; %real(ifftn(fftn(binaryCut).*gx)); + binaryCut = ( abs(filtIMG) .* dilatedIMG ) > intCutoff; + growthFactor = sum(binaryCut(:))/nStart; +end - % relax the cutoff and iteratively dilate the region - intCutoff = intCutoff./5; - growthFactor = 1.2; - while growthFactor > 1.05 - nStart = sum(binaryCut(:)); +% relax the cutoff and iteratively dilate the region +intCutoff = intCutoff./5; - dilatedIMG = convn(binaryCut,gx,'same') > 0.01; %real(ifftn(fftn(binaryCut).*gx)); - binaryCut = ( abs(filtIMG) .* dilatedIMG) > intCutoff; - growthFactor = sum(binaryCut(:))/nStart; - end +growthFactor = 1.2; +while growthFactor > 1.05 + nStart = sum(binaryCut(:)); + + dilatedIMG = convn(binaryCut,gx,'same') > 0.01; %real(ifftn(fftn(binaryCut).*gx)); + binaryCut = ( abs(filtIMG) .* dilatedIMG) > intCutoff; + growthFactor = sum(binaryCut(:))/nStart; +end % particleMask = real(ifftn(fftn(binaryCut).*gx.^2)); - [gx] = BH_multi_gaussian2d(7.*[1,1],3.0,0); - particleMask = convn(convn(binaryCut,gx,'same'),gx,'same'); - particleMask = particleMask ./ max(particleMask(:)); +[gx] = BH_multi_gaussian2d(7.*[1,1],3.0,0); +particleMask = convn(convn(binaryCut,gx,'same'),gx,'same'); +particleMask = particleMask ./ max(particleMask(:)); end diff --git a/statistics/BH_movingAverage.m b/statistics/BH_movingAverage.m index f0147b7c..5a1f22f0 100755 --- a/statistics/BH_movingAverage.m +++ b/statistics/BH_movingAverage.m @@ -41,7 +41,7 @@ img = BH_padZeros3d(fftshift(real(ifftn(fftn(meanFilter).*fftn(img)))),... - trimVal(1,:),trimVal(2,:),'GPU','single'); + trimVal(1,:),trimVal(2,:),'GPU','single'); clearvars -except img flgOOM end diff --git a/statistics/BH_movingRMS.m b/statistics/BH_movingRMS.m index 175f8a1a..b09abad8 100755 --- a/statistics/BH_movingRMS.m +++ b/statistics/BH_movingRMS.m @@ -41,9 +41,9 @@ img = BH_padZeros3d(fftshift(real(ifftn(fftn(rmsFilter).*fftn(img.^2)))),... - trimVal(1,:),trimVal(2,:),'GPU','single'); + trimVal(1,:),trimVal(2,:),'GPU','single'); -clearvars -except img +clearvars -except img diff --git a/statistics/BH_multi_cRef.m b/statistics/BH_multi_cRef.m index 0391444d..2f85439d 100755 --- a/statistics/BH_multi_cRef.m +++ b/statistics/BH_multi_cRef.m @@ -1,7 +1,7 @@ function [ cRefFilter, cRefBfactorFilter, targetFuncShells ] = ... - BH_multi_cRef( fscINFO, radialGrid, ... - bFactor, flgAlignRef, ... - varargin) + BH_multi_cRef( fscINFO, radialGrid, ... + bFactor, flgAlignRef, ... + varargin) %Calculate cRef bandpass filters. % Detailed explanation goes here @@ -34,15 +34,15 @@ cRefFilter = zeros(size(radialGrid),'single'); %cRefBfactorFilter = zeros(size(radialGrid),'single'); cRefBfactorFilter = ''; - % oversampled frequency axis - osX = fscINFO{4}; - +% oversampled frequency axis +osX = fscINFO{4}; + % Include an MTF correction for falcon II, based on Henderson's 2014 findings, % move this to a param file option including other detectors mtfX = 0:0.5/(length(osX)-1):0.5; flgPrintUsage =1; for iFilter = 1+coneOffset:1+nCones - iFilter + iFilter % Set any negative FSC values to zero prior to fitting to prevent knots in the % cRef curve. nonNegFSC = fscINFO{2}(:,iFilter); @@ -53,16 +53,16 @@ coneOrientation = BH_defineMatrix(coneList{iFilter-1},'Bah','invVector'); [ radius,~,height,~,~,~ ] = ... - BH_multi_gridCoordinates( size(radialGrid), ... - 'Cylindrical', ... - 'GPU', ... - {'single',... - coneOrientation,... - [0,0,0]', ... - 'invVector',... - 1,1},... - 0, 0, 0 ); - + BH_multi_gridCoordinates( size(radialGrid), ... + 'Cylindrical', ... + 'GPU', ... + {'single',... + coneOrientation,... + [0,0,0]', ... + 'invVector',... + 1,1},... + 0, 0, 0 ); + iConeMask = (rad2deg(atan2(radius,abs(height))) < halfAngle{iFilter-1}); clear radius height coneOverlap = coneOverlap + uint8(iConeMask); @@ -74,7 +74,7 @@ % This should be true everywhere except alignemnt for FSC calculation. if (flgAlignRef) % gaussian fall off to make sure cRef goes to zero if the curve gets wonky. - % force form fsc = 0.5 + % force form fsc = 0.5 forceMask = fscINFO{5}{iFilter}; else % force from fsc = 0.143 @@ -84,91 +84,91 @@ if (calcTargetFunction && iFilter == 1) [targetFuncShells] = calc_shells(radialGrid, forceMask, osX) end - switch fscINFO{3}{3} - case 0 - adHocMTF = 1; - if flgPrintUsage - fprintf('\n\nUsing MTF 0\n\n'); - flgPrintUsage = 0; - end - case 1 - detector = -100; - capVal = .05; - adHocMTF =((exp(detector.*osX.^1.25)+capVal)./capVal).^-1; - - if flgPrintUsage - fprintf('\n\nUsing MTF new\n\n'); - flgPrintUsage = 0; - end - case 2 - detector = -25; - capVal = .06; - adHocMTF = ((exp(detector.*osX.^1.25)+capVal)./capVal).^-1; - - if flgPrintUsage - fprintf('\n\nUsing MTF orig\n\n'); - flgPrintUsage = 0; - end - otherwise - detector = -1.*round(fscINFO{3}{3}); - capVal = fscINFO{3}{3}-round(fscINFO{3}{3}); - adHocMTF = ((exp(detector.*osX'.^1.25)+capVal)./capVal).^-1; - - if flgPrintUsage - fprintf('\n\nUsing MTF exp %d %3.3f\n\n',detector,capVal); - flgPrintUsage = 0; - end - - end - - + switch fscINFO{3}{3} + case 0 + adHocMTF = 1; + if flgPrintUsage + fprintf('\n\nUsing MTF 0\n\n'); + flgPrintUsage = 0; + end + case 1 + detector = -100; + capVal = .05; + adHocMTF =((exp(detector.*osX.^1.25)+capVal)./capVal).^-1; + + if flgPrintUsage + fprintf('\n\nUsing MTF new\n\n'); + flgPrintUsage = 0; + end + case 2 + detector = -25; + capVal = .06; + adHocMTF = ((exp(detector.*osX.^1.25)+capVal)./capVal).^-1; + + if flgPrintUsage + fprintf('\n\nUsing MTF orig\n\n'); + flgPrintUsage = 0; + end + otherwise + detector = -1.*round(fscINFO{3}{3}); + capVal = fscINFO{3}{3}-round(fscINFO{3}{3}); + adHocMTF = ((exp(detector.*osX'.^1.25)+capVal)./capVal).^-1; + + if flgPrintUsage + fprintf('\n\nUsing MTF exp %d %3.3f\n\n',detector,capVal); + flgPrintUsage = 0; + end + + end + + cRef = fit(osX, sqrt(abs( 2.*fitFSC(osX)./(1+fitFSC(osX)))) .* ... - adHocMTF .* forceMask, 'cubicSpline'); - - -% % % cRef= fit(osX, ((exp(-25.*mtfX'.^1.25)+0.06)./1.06).^-1 .* ... -% % % sqrt(abs( 2.*fitFSC(osX)./(1+fitFSC(osX)))) .* ... -% % % forceMask, 'cubicSpline'); + adHocMTF .* forceMask, 'cubicSpline'); + + + % % % cRef= fit(osX, ((exp(-25.*mtfX'.^1.25)+0.06)./1.06).^-1 .* ... + % % % sqrt(abs( 2.*fitFSC(osX)./(1+fitFSC(osX)))) .* ... + % % % forceMask, 'cubicSpline'); cRefFilter = cRefFilter + ... - iConeMask .* reshape(cRef(radialGrid),size(radialGrid)); -% figure, plot(osX,cRef(osX)); title(sprintf('fit %d',iFilter)); - -% % % if (bFactor) -% % % -% % % cRef= fit(osX, exp(bFactor.*osX.^2) .* ... -% % % ((exp(-25.*mtfX'.^1.25)+0.06)./1.06).^-1 .* ... -% % % sqrt(abs( 2.*fitFSC(osX)./(1+fitFSC(osX)))) .* ... -% % % forceMask, 'cubicSpline'); -% % % cRefBfactorFilter = cRefBfactorFilter+ ... -% % % iConeMask .* reshape(cRef(radialGrid),size(radialGrid)); -% % % end + iConeMask .* reshape(cRef(radialGrid),size(radialGrid)); + % figure, plot(osX,cRef(osX)); title(sprintf('fit %d',iFilter)); + + % % % if (bFactor) + % % % + % % % cRef= fit(osX, exp(bFactor.*osX.^2) .* ... + % % % ((exp(-25.*mtfX'.^1.25)+0.06)./1.06).^-1 .* ... + % % % sqrt(abs( 2.*fitFSC(osX)./(1+fitFSC(osX)))) .* ... + % % % forceMask, 'cubicSpline'); + % % % cRefBfactorFilter = cRefBfactorFilter+ ... + % % % iConeMask .* reshape(cRef(radialGrid),size(radialGrid)); + % % % end clear iConeMask end if (nCones) coneOverlap = single(coneOverlap); -% figure, imshow3D(coneOverlap); + % figure, imshow3D(coneOverlap); divZeroMask = (coneOverlap~=0); cRefFilter(divZeroMask) = cRefFilter(divZeroMask) ./coneOverlap(divZeroMask); -% % % cRefBfactorFilter(divZeroMask) = cRefBfactorFilter(divZeroMask) ./coneOverlap(divZeroMask); + % % % cRefBfactorFilter(divZeroMask) = cRefBfactorFilter(divZeroMask) ./coneOverlap(divZeroMask); [ gaussKernel ] = gpuArray(BH_multi_gaussian3d(7, 1.5 )); - + cRefFilter = ifftshift(gather(convn(fftshift(cRefFilter), gaussKernel, 'same'))); -% % % cRefBfactorFilter = ifftshift(gather(convn(fftshift(cRefBfactorFilter), gaussKernel, 'same'))); + % % % cRefBfactorFilter = ifftshift(gather(convn(fftshift(cRefBfactorFilter), gaussKernel, 'same'))); end cRefFilter = gather(cRefFilter); % % % cRefBfactorFilter = gather(cRefBfactorFilter); - -clear fitFSC osX forceMask cRef radialGrid fscINFO + +clear fitFSC osX forceMask cRef radialGrid fscINFO end function [targetFuncShells] = calc_shells(radialGrid, forceMask, osX) - - + + padDIM = size(radialGrid) rad = radialGrid(1,1:ceil(padDIM(2)/2),1); % Lump all lower resolution info into one shell and don't go out further than @@ -184,9 +184,9 @@ % check a few bin sizes if (highIDX - lowIDX)/5 < 30 binInc = 5; -else +else binInc = floor((highIDX-lowIDX)/30); -end +end % The first two positions are a "header" for 0/1 to take abs value, and the % "wiener filter constant" = 1/(N included)^3/2 targetFuncShells = cell(binInc+2,1); @@ -202,14 +202,14 @@ targetFuncShells{3} = gather(idxVector(iMask(:))); while nIDX < highIDX - - - iMask = ( rad(nIDX) <= radialGrid & radialGrid < rad(nIDX+binInc)); - - targetFuncShells{nShell} = gather(idxVector(iMask(:))); - targetFuncShells{2} = targetFuncShells{2} + length(targetFuncShells{nShell}); - nShell = nShell + 1; - nIDX = nIDX + binInc; + + + iMask = ( rad(nIDX) <= radialGrid & radialGrid < rad(nIDX+binInc)); + + targetFuncShells{nShell} = gather(idxVector(iMask(:))); + targetFuncShells{2} = targetFuncShells{2} + length(targetFuncShells{nShell}); + nShell = nShell + 1; + nIDX = nIDX + binInc; end diff --git a/statistics/BH_multi_cRef_Vnorm.m b/statistics/BH_multi_cRef_Vnorm.m index 896b186f..235ca3c2 100755 --- a/statistics/BH_multi_cRef_Vnorm.m +++ b/statistics/BH_multi_cRef_Vnorm.m @@ -1,8 +1,8 @@ -function [weightedImgs] = BH_multi_cRef_Vnorm( ... - fscParams, aliParams, mskParams,... - imgs, weights, ... - flgCombine, flgReference, ... - pixelSize, bFactor,varargin) +function [weightedImgs] = BH_multi_cRef_Vnorm( ... + fscParams, aliParams, mskParams,... + imgs, weights, ... + flgCombine, flgReference, ... + pixelSize, bFactor,varargin) % Don't force to zero - particularly for comparison with ground truth, should @@ -11,7 +11,7 @@ if any(bFactor < 0) bFactor = abs(bFactor) noForceMask = 1 -else +else noForceMask = 0 end @@ -23,10 +23,10 @@ highPassFilter = varargin{2}; else highPassFilter = [0,0]; -end +end + +[ padVal ] = BH_multi_padVal(size(imgs{1}), size(weights{1})); -[ padVal ] = BH_multi_padVal(size(imgs{1}), size(weights{1})); - % Only minor differences in half sets are expected, so make a mask from combined @@ -42,22 +42,22 @@ img1_orig = img1_orig - mean(img1_orig(:)); img1_orig = img1_orig ./ rms(img1_orig(:)); imgs{1} = BH_resample3d(gather(img1_orig), ... - aliParams(1,:), ... - aliParams(2,1:3), ... - {'Bah',1,'spline'}, 'cpu', 'forward'); + aliParams(1,:), ... + aliParams(2,1:3), ... + {'Bah',1,'spline'}, 'cpu', 'forward'); end - -% Could replace the spline with Fourier resampling. + +% Could replace the spline with Fourier resampling. for iWgt = 1:2 -% + % imgs{iWgt} = imgs{iWgt} - mean(imgs{iWgt}(:)); imgs{iWgt} = gpuArray(imgs{iWgt} ./rms(imgs{iWgt}(:))); - + [weights{iWgt}, ~] = BH_multi_cRef_wgtCritical(gpuArray(weights{iWgt})); - + end @@ -76,17 +76,17 @@ if (mskParams{5}) % Soft shape mask used for FSC calculation -% [ mShape2 ]= BH_mask3d(imgs{1} + imgs{2}, 1.*pixelSize,'',''); -% These are inhereted from the param file during the call to fscGold_class -shape_mask_lowpass = mskParams{6}; -shape_mask_threshold = mskParams{7}; - - -% padIMG = real(ifftn(padIMG./(padWGT+wienerThreshold))); - mShape2 = BH_padZeros3d((imgs{1} + imgs{2}),'fwd',padVal,'GPU','single'); - mShape2 = real(ifftn(fftn(mShape2)./(ifftshift(weights{1}+weights{2})+100))); - [ mShape2 ] = EMC_maskReference(gpuArray(mShape2), pixelSize, {'fsc', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); - mShape2 = BH_padZeros3d(mShape2,'inv',padVal,'GPU','single'); + % [ mShape2 ]= BH_mask3d(imgs{1} + imgs{2}, 1.*pixelSize,'',''); + % These are inhereted from the param file during the call to fscGold_class + shape_mask_lowpass = mskParams{6}; + shape_mask_threshold = mskParams{7}; + + + % padIMG = real(ifftn(padIMG./(padWGT+wienerThreshold))); + mShape2 = BH_padZeros3d((imgs{1} + imgs{2}),'fwd',padVal,'GPU','single'); + mShape2 = real(ifftn(fftn(mShape2)./(ifftshift(weights{1}+weights{2})+100))); + [ mShape2 ] = EMC_maskReference(gpuArray(mShape2), pixelSize, {'fsc', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); + mShape2 = BH_padZeros3d(mShape2,'inv',padVal,'GPU','single'); particleMask = mShape2 .* particleMask; end @@ -97,14 +97,14 @@ radialGrid = BH_multi_gridCoordinates(size(weights{1}),'Cartesian','GPU', ... - {'none'},1,0,1); -radialGrid = radialGrid ./ pixelSize; + {'none'},1,0,1); +radialGrid = radialGrid ./ pixelSize; [ anisoFSC, avgCTF ] = calc_anisoFSC(fscParams, radialGrid,weights, bFactor, pixelSize); -if any(bFactor) +if any(bFactor) [ bFactor, bandFilter ] = calc_bfact(fscParams, radialGrid,flgReference, bFactor, noForceMask,highPassFilter,pixelSize); else bFactor = {1}; @@ -125,24 +125,24 @@ for iBfact = 1:length(bFactor) if (flgCombine) snrWeight = 1; - + [ weightedImgs{iBfact} ] = gather(apply_weights((anisoFSC), ... - (avgCTF{1}+avgCTF{2})./2, ... - (radialGrid < 0.5./pixelSize),... - (imgs{1}+imgs{2}), ... - ifftshift(weights{1}+weights{2}),... - particleMask ,... - padVal, fPfM, bFactor{iBfact}, snrWeight)); + (avgCTF{1}+avgCTF{2})./2, ... + (radialGrid < 0.5./pixelSize),... + (imgs{1}+imgs{2}), ... + ifftshift(weights{1}+weights{2}),... + particleMask ,... + padVal, fPfM, bFactor{iBfact}, snrWeight)); else snrWeight = 0.5; for iGold = 1:2 - [ weightedImgs{iGold} ] = gather(apply_weights((anisoFSC), ... - (avgCTF{iGold}), ... - (radialGrid < 0.5./pixelSize),... - (imgs{iGold}), ... - ifftshift(weights{iGold}),... - particleMask ,... - padVal, fPfM, bFactor{iBfact}, snrWeight)); + [ weightedImgs{iGold} ] = gather(apply_weights((anisoFSC), ... + (avgCTF{iGold}), ... + (radialGrid < 0.5./pixelSize),... + (imgs{iGold}), ... + ifftshift(weights{iGold}),... + particleMask ,... + padVal, fPfM, bFactor{iBfact}, snrWeight)); end end @@ -155,140 +155,140 @@ function [ fsc3D, weights ] = calc_anisoFSC(fscParams, radialGrid, weights, bFactor, pixelSize) - nCones = fscParams{7}; - if (nCones) - firstCone = 1; - else - firstCone = 0; - end +nCones = fscParams{7}; +if (nCones) + firstCone = 1; +else + firstCone = 0; +end - coneList = fscParams{8}; - halfAngle = fscParams{9}; - samplingRate = fscParams{10}; +coneList = fscParams{8}; +halfAngle = fscParams{9}; +samplingRate = fscParams{10}; - osX = fscParams{4}; +osX = fscParams{4}; - radialBinary = (radialGrid < 0.5./pixelSize); - - mtfX = 0:0.5/(length(osX)-1):0.5; - - - coneMask = zeros(size(radialBinary) ,'single','gpuArray'); - fsc3D = zeros(size(radialBinary), 'single','gpuArray'); - - bin = floor(size(weights{1},1)/1); - inc = 0.5 / (bin*pixelSize); +radialBinary = (radialGrid < 0.5./pixelSize); - for iCone = firstCone:nCones +mtfX = 0:0.5/(length(osX)-1):0.5; - iFSCfit = csape(fscParams{1}(:,1),fscParams{2}(:,iCone+1),'variational'); - - % Don't lowpass in-case of FSC calculation - if any(bFactor) - iForceMask = fscParams{6}{iCone+1}; - else - iForceMask = 1 - end - - iFSCclean = csape(osX,fnval(iFSCfit,osX).*iForceMask,'variational'); - - - - if ( nCones ) - coneOrientation = BH_defineMatrix(coneList{iCone},'Bah','invVector'); - [ radius,~,height,~,~,~ ] = ... - BH_multi_gridCoordinates( size(radialBinary), ... - 'Cylindrical', ... - 'GPU', ... - {'single',... - coneOrientation,... - [0,0,0]', ... - 'invVector',... - 1,1},... - 0, 0, 0 ); - iConeMask = (rad2deg(atan2(radius,abs(height))) < halfAngle{iCone}); - - - - coneMask = coneMask + iConeMask; - else - iConeMask = radialBinary; - coneMask = ones(size(radialBinary), 'single', 'gpuArray'); - end - - +coneMask = zeros(size(radialBinary) ,'single','gpuArray'); +fsc3D = zeros(size(radialBinary), 'single','gpuArray'); +bin = floor(size(weights{1},1)/1); +inc = 0.5 / (bin*pixelSize); - - iConeMask = iConeMask & radialBinary; - iFSC = iConeMask; - iFSC(iConeMask) = fnval(iFSCclean,radialGrid(iConeMask)); - fsc3D = fsc3D + iFSC; +for iCone = firstCone:nCones + + iFSCfit = csape(fscParams{1}(:,1),fscParams{2}(:,iCone+1),'variational'); + + + % Don't lowpass in-case of FSC calculation + if any(bFactor) + iForceMask = fscParams{6}{iCone+1}; + else + iForceMask = 1 + end + + iFSCclean = csape(osX,fnval(iFSCfit,osX).*iForceMask,'variational'); + + + + if ( nCones ) + coneOrientation = BH_defineMatrix(coneList{iCone},'Bah','invVector'); + [ radius,~,height,~,~,~ ] = ... + BH_multi_gridCoordinates( size(radialBinary), ... + 'Cylindrical', ... + 'GPU', ... + {'single',... + coneOrientation,... + [0,0,0]', ... + 'invVector',... + 1,1},... + 0, 0, 0 ); + iConeMask = (rad2deg(atan2(radius,abs(height))) < halfAngle{iCone}); - - - clear iFSC iConeMask radius height - + coneMask = coneMask + iConeMask; + else + iConeMask = radialBinary; + coneMask = ones(size(radialBinary), 'single', 'gpuArray'); + end + + + + + + iConeMask = iConeMask & radialBinary; + iFSC = iConeMask; + iFSC(iConeMask) = fnval(iFSCclean,radialGrid(iConeMask)); + fsc3D = fsc3D + iFSC; + + + + + + clear iFSC iConeMask radius height + + + +end - end - +fsc3D = fftshift(fsc3D./coneMask); +fsc3D(fsc3D < 10^-10) = 10^-10; - fsc3D = fftshift(fsc3D./coneMask); - fsc3D(fsc3D < 10^-10) = 10^-10; - - for i = 1.5:-0.5:0.5 - g = gpuArray(BH_multi_gaussian3d(5.*[1,1,1],i)); - fsc3D = convn(fsc3D,g,'same'); - for iGold = 1:2 - weights{iGold} = convn(weights{iGold},g,'same'); - end - end - +for i = 1.5:-0.5:0.5 + g = gpuArray(BH_multi_gaussian3d(5.*[1,1,1],i)); + fsc3D = convn(fsc3D,g,'same'); for iGold = 1:2 - weights{iGold} = ifftshift(weights{iGold}); + weights{iGold} = convn(weights{iGold},g,'same'); end - fsc3D = ifftshift(fsc3D); +end - clear radialBinary coneMask +for iGold = 1:2 + weights{iGold} = ifftshift(weights{iGold}); +end +fsc3D = ifftshift(fsc3D); + +clear radialBinary coneMask end % Should be adapted to work with CONES function [ avgCTF ] = calc_avgCTF( radialGrid, weights,pixelSize) + +radialGrid = fftshift(radialGrid); +g = gpuArray(BH_multi_gaussian3d(5.*[1,1,1],.75)); +avgCTF = cell(2,1); +for iGold = 1:2 + bin = floor(size(weights{iGold},1)/4); + inc = 0.5 / (bin*pixelSize); - radialGrid = fftshift(radialGrid); - g = gpuArray(BH_multi_gaussian3d(5.*[1,1,1],.75)); - avgCTF = cell(2,1); - for iGold = 1:2 - bin = floor(size(weights{iGold},1)/4); - inc = 0.5 / (bin*pixelSize); - - avgWgt = zeros(size(weights{iGold}),'single','gpuArray'); + avgWgt = zeros(size(weights{iGold}),'single','gpuArray'); - for q = 1:bin - iMask = gpuArray((q-1)*inc <= radialGrid & radialGrid < (q)*inc); - avgWgt(iMask) = sum(weights{iGold}(iMask))./(sum(iMask(:))); - end - - avgCTF{iGold} = ifftshift(convn(avgWgt, g, 'same')); - - - clear tmpIMG + for q = 1:bin + iMask = gpuArray((q-1)*inc <= radialGrid & radialGrid < (q)*inc); + avgWgt(iMask) = sum(weights{iGold}(iMask))./(sum(iMask(:))); end - clear avgWgt iMask radialGrid + + avgCTF{iGold} = ifftshift(convn(avgWgt, g, 'same')); + + + clear tmpIMG +end +clear avgWgt iMask radialGrid end function [ weightedImg ] = apply_weights(anisoFSC, avgCTF, nyquistLimit, img, ... - wgt, mShape2, ... - padVal, fPfM, bFactor, snrWeight) - + wgt, mShape2, ... + padVal, fPfM, bFactor, snrWeight) + % bFactor contains any sharpening, and/or forced cutoffs @@ -309,20 +309,20 @@ weightedImg = BH_bandLimitCenterNormalize(img-mean(img(:)),weight, ... - '',padVal,'doubleTaper'); - -clear img + '',padVal,'doubleTaper'); + +clear img weightedImg = real(ifftn(weightedImg)); size(weightedImg) padVal size(mShape2) weightedImg = weightedImg(padVal(1,1)+1:end-padVal(2,1),... - padVal(1,2)+1:end-padVal(2,2),... - padVal(1,3)+1:end-padVal(2,3)) .* mShape2; - + padVal(1,2)+1:end-padVal(2,2),... + padVal(1,3)+1:end-padVal(2,3)) .* mShape2; + clear mShape2 - + end function [ bFactorFilter, bandFilter ] = calc_bfact(fscParams, radialGrid, flgReference, bFactor,noForceMask,highPassFilter,pixelSize) @@ -349,76 +349,76 @@ for iBfact = 1:nBfactors bFactorFilter{iBfact} = zeros(size(radialGrid),'single'); end - % Should just add pixel Size to fitFSC.Raw1 - - osX = fscParams{4}; +% Should just add pixel Size to fitFSC.Raw1 + +osX = fscParams{4}; - mtfX = 0:0.5/(length(osX)-1):0.5; +mtfX = 0:0.5/(length(osX)-1):0.5; flgPrintUsage = 1; for iFilter = 1+coneOffset:1+nCones - iFilter; - + iFilter; + if (flgReference) forceMask = fscParams{5}{iFilter}; else forceMask = fscParams{6}{iFilter}; end if iFilter > 1 - + coneOrientation = BH_defineMatrix(coneList{iFilter-1},'Bah','invVector'); [ radius,~,height,~,~,~ ] = ... - BH_multi_gridCoordinates( size(radialGrid), ... - 'Cylindrical', ... - 'GPU', ... - {'single', ... - coneOrientation,... - [0,0,0]', ... - 'invVector',... - 1,1},... - 0, 0, 0 ); - + BH_multi_gridCoordinates( size(radialGrid), ... + 'Cylindrical', ... + 'GPU', ... + {'single', ... + coneOrientation,... + [0,0,0]', ... + 'invVector',... + 1,1},... + 0, 0, 0 ); + iConeMask = (rad2deg(atan2(radius,abs(height))) < halfAngle{iFilter-1}); clear radius height coneOverlap = coneOverlap + uint8(iConeMask); else iConeMask = 1; - + end -if any(highPassFilter) - bandPassFilter = (BH_bandpass3d(size(radialGrid),highPassFilter(1),highPassFilter(2),pixelSize,'GPU',pixelSize)); -end - - for iBfact = 1:nBfactors -% bFit = fit(osX, exp(bFactor(iBfact).*osX.^2) .* ... -% ((exp(-10.*mtfX'.^1.25)+0.06)./1.06).^-1 .* ... -% forceMask, 'cubicSpline'); - + if any(highPassFilter) + bandPassFilter = (BH_bandpass3d(size(radialGrid),highPassFilter(1),highPassFilter(2),pixelSize,'GPU',pixelSize)); + end + + for iBfact = 1:nBfactors + % bFit = fit(osX, exp(bFactor(iBfact).*osX.^2) .* ... + % ((exp(-10.*mtfX'.^1.25)+0.06)./1.06).^-1 .* ... + % forceMask, 'cubicSpline'); + % detector = -10 is very close to the mtf for a falcon DDD switch fscParams{3}{3} case 0 adHocMTF = 1; - if flgPrintUsage + if flgPrintUsage fprintf('\n\nUsing MTF 0\n\n'); flgPrintUsage = 0; end case 1 detector = -100; capVal = .05; - adHocMTF =((exp(detector.*osX.^1.25)+capVal)./capVal).^-1; - - if flgPrintUsage + adHocMTF =((exp(detector.*osX.^1.25)+capVal)./capVal).^-1; + + if flgPrintUsage fprintf('\n\nUsing MTF new\n\n'); flgPrintUsage = 0; - end + end case 2 detector = -20; capVal = .13; adHocMTF = ((exp(detector.*osX.^1.25)+capVal)./capVal).^-1; - if flgPrintUsage + if flgPrintUsage fprintf('\n\nUsing MTF orig\n\n'); flgPrintUsage = 0; end @@ -427,37 +427,37 @@ capVal = fscParams{3}{3}-round(fscParams{3}{3}); adHocMTF = ((exp(detector.*osX.^1.25)+capVal)./capVal).^-1; - if flgPrintUsage + if flgPrintUsage fprintf('\n\nUsing MTF exp %d %3.3f\n\n',detector,capVal); flgPrintUsage = 0; - end + end end - + if (noForceMask) bFit = fit(osX, exp(bFactor(iBfact)./4.*osX.^2) .* ... - adHocMTF,'cubicSpline'); + adHocMTF,'cubicSpline'); else bFit = fit(osX, exp(bFactor(iBfact)./4.*osX.^2) .* ... - adHocMTF .* forceMask, 'cubicSpline'); + adHocMTF .* forceMask, 'cubicSpline'); end - + bFactorFilter{iBfact} = bFactorFilter{iBfact} + ... - iConeMask .* reshape(bFit(radialGrid),size(radialGrid)); - + iConeMask .* reshape(bFit(radialGrid),size(radialGrid)); + if any(highPassFilter) bFactorFilter{iBfact} = bFactorFilter{iBfact} .* bandPassFilter; end - - % if iBfact ==1 - % bLimit = fit(osX, forceMask, 'cubicSpline'); - % bandFilter = bandFilter + iConeMask .* reshape(bLimit(radialGrid),size(radialGrid)); - % end - - end + + % if iBfact ==1 + % bLimit = fit(osX, forceMask, 'cubicSpline'); + % bandFilter = bandFilter + iConeMask .* reshape(bLimit(radialGrid),size(radialGrid)); + % end + + end end %if any(highPassFilt) @@ -466,18 +466,18 @@ if (nCones) coneOverlap = single(coneOverlap); -% figure, imshow3D(coneOverlap); + % figure, imshow3D(coneOverlap); divZeroMask = (coneOverlap~=0); [ gaussKernel ] = gpuArray(BH_multi_gaussian3d(7, 1.5 )); for iBfact = 1:nBfactors bFactorFilter{iBfact}(divZeroMask) = bFactorFilter{iBfact}(divZeroMask) ./coneOverlap(divZeroMask); bFactorFilter{iBfact} = ifftshift(gather(convn(fftshift(bFactorFilter{iBfact}), gaussKernel, 'same'))); - % if iBfact == 1 - % bandFilter(divZeroMask) =bandFilter(divZeroMask) ./coneOverlap(divZeroMask); - % bandFilter = ifftshift(gather(convn(fftshift(bandFilter), gaussKernel, 'same'))); - % end + % if iBfact == 1 + % bandFilter(divZeroMask) =bandFilter(divZeroMask) ./coneOverlap(divZeroMask); + % bandFilter = ifftshift(gather(convn(fftshift(bandFilter), gaussKernel, 'same'))); + % end end - + end end diff --git a/statistics/BH_multi_cRef_VnormApply.m b/statistics/BH_multi_cRef_VnormApply.m index bd6e61cf..dcb6ea7b 100644 --- a/statistics/BH_multi_cRef_VnormApply.m +++ b/statistics/BH_multi_cRef_VnormApply.m @@ -1,6 +1,6 @@ function [ ] = BH_multi_cRef_VnormApply( matFile, CYCLE, outputPrefix, ... - imgBaseName, ... - idxVect, bFactor, varargin) + imgBaseName, ... + idxVect, bFactor, varargin) %UNTITLED Summary of this function goes here % Detailed explanation goes here @@ -47,7 +47,7 @@ else fscArgs = 1; end - + try fscParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('REF%d',fscArgs)); aliParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('ResampleREF%d',fscArgs)); @@ -65,21 +65,21 @@ mskParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('MaskRaw%d',1)); end end - + pixelSize = 0.5/fscParams{4}(end); if mFactor % Override default mtf filter fscParams{3}{3} = mFactor; end - + [w] = BH_multi_cRef_Vnorm(fscParams, aliParams, mskParams, ... - {img1{i},img2{i}},{wgt1{i},wgt2{i}}, ... - 1,0, pixelSize,bFactor); - + {img1{i},img2{i}},{wgt1{i},wgt2{i}}, ... + 1,0, pixelSize,bFactor); + for iBfact = 1:length(bFactor) SAVE_IMG(MRCImage(gather(w{iBfact})), ... - sprintf('%s-n%0.3u-bf-%d.mrc',outputPrefix,i,bFactor(iBfact)),pixelSize); + sprintf('%s-n%0.3u-bf-%d.mrc',outputPrefix,i,bFactor(iBfact)),pixelSize); end end diff --git a/statistics/BH_multi_cRef_wgtCritical.m b/statistics/BH_multi_cRef_wgtCritical.m index 45cd9e71..fdf56e2e 100644 --- a/statistics/BH_multi_cRef_wgtCritical.m +++ b/statistics/BH_multi_cRef_wgtCritical.m @@ -3,29 +3,29 @@ %problem. % Detailed explanation goes here - startingMax = max(sf3d(:)); - valAtZero = max(10,0.1*startingMax); % ~ value at zero sampling (a bit less after the subtraction to keep the - % value at minNumSampled unchanged with a smooth transition. - - wienerThreshold = (1.5.*(median(sf3d(sf3d(:)>valAtZero))-valAtZero)); +startingMax = max(sf3d(:)); +valAtZero = max(10,0.1*startingMax); % ~ value at zero sampling (a bit less after the subtraction to keep the +% value at minNumSampled unchanged with a smooth transition. +wienerThreshold = (1.5.*(median(sf3d(sf3d(:)>valAtZero))-valAtZero)); - minNumSampled = 0.2.*median(sf3d(sf3d(:)>10)); % value below where a penalty is add (very little until low numbers) - minFactor = 75/minNumSampled; - minWeight = gpuArray(10); % decreasing this increase the downweighting as you move from 0 to minNumSampled - % There shouldb't be any less than zero but due to the quality weighting - % there could be - sf3d(sf3d < 1) = 1; - m = (sf3d < minNumSampled); +minNumSampled = 0.2.*median(sf3d(sf3d(:)>10)); % value below where a penalty is add (very little until low numbers) +minFactor = 75/minNumSampled; +minWeight = gpuArray(10); % decreasing this increase the downweighting as you move from 0 to minNumSampled + +% There shouldb't be any less than zero but due to the quality weighting +% there could be +sf3d(sf3d < 1) = 1; +m = (sf3d < minNumSampled); + +sf3d(m) = (sf3d(m)+1) + valAtZero.^(minWeight.^((minFactor.*sf3d(m)+1).^-1)); +sf3d(m) = sf3d(m)-valAtZero.^(minWeight.^(minFactor*minNumSampled+1).^-1)+1; +m = sf3d > startingMax; +sf3d(m) = startingMax + log(sf3d(m)); + - sf3d(m) = (sf3d(m)+1) + valAtZero.^(minWeight.^((minFactor.*sf3d(m)+1).^-1)); - sf3d(m) = sf3d(m)-valAtZero.^(minWeight.^(minFactor*minNumSampled+1).^-1)+1; - m = sf3d > startingMax; - sf3d(m) = startingMax + log(sf3d(m)); - - end diff --git a/statistics/BH_multi_loadAndMaskStack.m b/statistics/BH_multi_loadAndMaskStack.m index a73ea074..d5e74e63 100644 --- a/statistics/BH_multi_loadAndMaskStack.m +++ b/statistics/BH_multi_loadAndMaskStack.m @@ -51,22 +51,22 @@ % calculate it based on subTomogram coordinates in 3d is not correct THICKNESS = 75; if ~isnumeric(STACK) - + if samplingRate > 1 fullStack = sprintf('aliStacks/%s_ali%d.fixed', STACK,mapBackIter+1); inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed',STACK,mapBackIter+1,samplingRate); - + if ~exist(inputStack, 'file') BH_multi_loadOrBin(fullStack,-1.*samplingRate,2); end - + else inputStack = sprintf('aliStacks/%s_ali%d%s.fixed',STACK,mapBackIter+1,suffix); end - - + + STACK = single(getVolume(MRCImage(inputStack))); - + end @@ -84,10 +84,10 @@ bandNyquist = BH_bandpass3d([d1,d2,1],0,0,1,'GPU','nyquistHigh'); end - fractionOfDose = TLT(:,14)/mean(TLT(:,14)); - fractionOfElastics = exp(-1.*THICKNESS./( cosd(TLT(:,4)).*400 )); - fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics(:)); - +fractionOfDose = TLT(:,14)/mean(TLT(:,14)); +fractionOfElastics = exp(-1.*THICKNESS./( cosd(TLT(:,4)).*400 )); +fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics(:)); + for iPrj = 1:d3 if (justHighPass) @@ -96,14 +96,14 @@ iProjection = real(ifftn(fftn(iProjection).*bandNyquist)); STACK(:,:,iPrj) = gather(iProjection); continue - else + else if (flgOnDevice) iProjection = STACK(:,:,TLT(iPrj,1)); else iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); end end - + if (useMask) iProjection = iProjection - mean2(iProjection(varargin{1}(:,:,TLT(iPrj,1))>0)); iProjection = iProjection ./ rms(rms(iProjection(varargin{1}(:,:,TLT(iPrj,1))>0))); @@ -111,7 +111,7 @@ maxEval = cosd(TLT(iPrj,4)).*(d1/2) + (THICKNESS*10./(PIXEL_SIZE))./2*abs(sind(TLT(iPrj,4))); oX = ceil((d1+1)./2); iEvalMask = max(EDGE_PAD,floor(oX-maxEval)):min(d1-EDGE_PAD,ceil(oX+maxEval)); - + % Remove gradients % Center under mask iProjection = iProjection - mean2(iProjection(iEvalMask,EDGE_PAD:end-EDGE_PAD)); @@ -119,17 +119,17 @@ iProjection = iProjection ./ rms(rms(iProjection(iEvalMask,EDGE_PAD:end-EDGE_PAD))); end - + iProjection = iProjection .* (fractionOfDose(iPrj)*fractionOfElastics(iPrj)); - -% if (useMask) -% meanVariance = meanVariance + rms(rms(iProjection(varargin{1}(:,:,TLT(iPrj,1))>0))); -% else -% meanVariance = meanVariance + rms(rms(iProjection(iEvalMask,EDGE_PAD:end-EDGE_PAD))); -% end - - + + % if (useMask) + % meanVariance = meanVariance + rms(rms(iProjection(varargin{1}(:,:,TLT(iPrj,1))>0))); + % else + % meanVariance = meanVariance + rms(rms(iProjection(iEvalMask,EDGE_PAD:end-EDGE_PAD))); + % end + + if (flgOnDevice) STACK(:,:,TLT(iPrj,1)) = iProjection; diff --git a/statistics/BH_multi_statScale.m b/statistics/BH_multi_statScale.m index 0997eb76..3a507ba0 100644 --- a/statistics/BH_multi_statScale.m +++ b/statistics/BH_multi_statScale.m @@ -1,7 +1,7 @@ function [IMG] = BH_multi_statScale(IMG,DATATYPE) %Scale to zero mean, R variance, and cast as data type % Detailed explanation goes here - + m = mean(IMG(:)); @@ -20,34 +20,34 @@ % number of standard deviations above which data values are compressed % (rather than truncated) maxNonCompressed = 4; - -if ~strcmp(class(IMG), DATATYPE) +if ~strcmp(class(IMG), DATATYPE) + % This should only be used with "raw" data like image stacks or tomograms - % where no individual set of pixels should have extremely large values. + % where no individual set of pixels should have extremely large values. switch DATATYPE case 'int16' - + compressMask = IMG > maxNonCompressed; IMG(compressMask) = maxNonCompressed + ... - (sqrt(single(IMG(compressMask))) - sqrt(maxNonCompressed)); + (sqrt(single(IMG(compressMask))) - sqrt(maxNonCompressed)); compressMask = IMG < -1*maxNonCompressed; IMG(compressMask) = maxNonCompressed - ... - (sqrt(abs(single(IMG(compressMask)))) - sqrt(maxNonCompressed)); + (sqrt(abs(single(IMG(compressMask)))) - sqrt(maxNonCompressed)); IMG = int16(IMG .* (2^16 / (2*max(abs(IMG(:)))))); - + case 'uint16' - + compressMask = IMG > maxNonCompressed; IMG(compressMask) = maxNonCompressed + ... - (sqrt(IMG(compressMask)) - sqrt(maxNonCompressed)); + (sqrt(IMG(compressMask)) - sqrt(maxNonCompressed)); compressMask = IMG < -1*maxNonCompressed; IMG(compressMask) = maxNonCompressed - ... - (sqrt(abs(IMG(compressMask))) - sqrt(maxNonCompressed)); - + (sqrt(abs(IMG(compressMask))) - sqrt(maxNonCompressed)); + IMG = IMG - min(IMG(:)); IMG = uint16(IMG .* (2^16 / (max(abs(IMG(:)))))); diff --git a/statistics/BH_multi_xcf_Rotational.m b/statistics/BH_multi_xcf_Rotational.m index b019f853..81542c5b 100755 --- a/statistics/BH_multi_xcf_Rotational.m +++ b/statistics/BH_multi_xcf_Rotational.m @@ -1,9 +1,9 @@ function [ iCCC, iWeight ] = BH_multi_xcf_Rotational( rotPART_FT, REF_FT, ... - wdgMask, refWdg, ... - wCCC) - + wdgMask, refWdg, ... + wCCC) + %Consolodating function, calculate wedge weight and cross correlation -% +% % % Called by: % @@ -11,7 +11,7 @@ % % BH_alignClass3d % -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % @@ -22,17 +22,17 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % TODO: Add something to report when no valid peak is found. -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - iWeight = 1; - +iWeight = 1; + % Testing, might be better later to just make a separate logical peak mask % once at the begining and pass it in. For now this means far fewer % changes. if isnan(gather(refWdg(1))) - refWdg = 1; + refWdg = 1; end if isnan(gather(wdgMask(1))) @@ -46,39 +46,39 @@ end + +if (flgWeightCCC) + iCCC = 0; - if (flgWeightCCC) - iCCC = 0; + for iShell = 3:length(wCCC); - for iShell = 3:length(wCCC); - - % reference is the conjugate of filtered fft - numerator = real(refWdg(wCCC{iShell}).*rotPART_FT(wCCC{iShell}) .* ... - REF_FT(wCCC{iShell}) .* wdgMask(wCCC{iShell})); - denominator = ( sum(sum(sum( abs(REF_FT(wCCC{iShell}).* ... - wdgMask(wCCC{iShell})).^2 ))) .* ... - sum(sum(sum( abs(rotPART_FT(wCCC{iShell}).*... - refWdg(wCCC{iShell})).^2))) ); - - - iCCCshell = real(sum(sum(sum(numerator))) ./ sqrt(denominator)); - if wCCC{1} - iCCC = iCCC + abs((iCCCshell^2 / (iCCCshell + wCCC{2}^-1.5))^3); - else - iCCC = iCCC + (iCCCshell^2 / (iCCCshell + wCCC{2}^-1.5))^3; - end - end - else % reference is the conjugate of filtered fft - numerator = real(refWdg.*rotPART_FT .* REF_FT .* wdgMask); - denominator = ( sum(sum(sum( abs(REF_FT.*wdgMask).^2 ))) .* ... - sum(sum(sum( abs(rotPART_FT.*refWdg).^2))) ); - - - iCCC = sum(sum(sum(numerator))) ./ sqrt(denominator); - + numerator = real(refWdg(wCCC{iShell}).*rotPART_FT(wCCC{iShell}) .* ... + REF_FT(wCCC{iShell}) .* wdgMask(wCCC{iShell})); + denominator = ( sum(sum(sum( abs(REF_FT(wCCC{iShell}).* ... + wdgMask(wCCC{iShell})).^2 ))) .* ... + sum(sum(sum( abs(rotPART_FT(wCCC{iShell}).*... + refWdg(wCCC{iShell})).^2))) ); + + + iCCCshell = real(sum(sum(sum(numerator))) ./ sqrt(denominator)); + if wCCC{1} + iCCC = iCCC + abs((iCCCshell^2 / (iCCCshell + wCCC{2}^-1.5))^3); + else + iCCC = iCCC + (iCCCshell^2 / (iCCCshell + wCCC{2}^-1.5))^3; + end end - clear rotPART_FT REF_FT REF_WDG iWedgeMask peakMask numerator denominator +else + % reference is the conjugate of filtered fft + numerator = real(refWdg.*rotPART_FT .* REF_FT .* wdgMask); + denominator = ( sum(sum(sum( abs(REF_FT.*wdgMask).^2 ))) .* ... + sum(sum(sum( abs(rotPART_FT.*refWdg).^2))) ); + + + iCCC = sum(sum(sum(numerator))) ./ sqrt(denominator); + +end +clear rotPART_FT REF_FT REF_WDG iWedgeMask peakMask numerator denominator end % end of multi_xcf diff --git a/statistics/BH_multi_xcf_Translational.m b/statistics/BH_multi_xcf_Translational.m index 96edef9f..25474c43 100755 --- a/statistics/BH_multi_xcf_Translational.m +++ b/statistics/BH_multi_xcf_Translational.m @@ -1,8 +1,8 @@ function [ PEAK_COORD ] = BH_multi_xcf_Translational( rotPART_FT, REF_FT, ... - peakMask, PEAK_COM) - + peakMask, PEAK_COM) + %Consolodating function, calculate wedge weight and cross correlation -% +% % % Called by: % @@ -10,7 +10,7 @@ % % BH_alignClass3d % -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % @@ -21,17 +21,17 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % TODO: Add something to report when no valid peak is found. -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % regular cross-correlation +% regular cross-correlation iCCCmap = fftshift(real(ifftn(rotPART_FT.*REF_FT))); minVal = min(iCCCmap,[],'all'); -iCCCmap(peakMask < 0.95) = minVal; +iCCCmap(peakMask < 0.95) = minVal; peakCOM = PEAK_COM; @@ -42,8 +42,8 @@ PEAK_COORD = [maxX, maxY, maxZ]; % Sometimes there is an "empty class", ignore this in xfc -if (~isnan(gather(maxVal)) && (maxVal ~= 0)) - +if (~isnan(gather(maxVal)) && (maxVal ~= 0)) + for iCOM = 1:2 if (peakCOM) @@ -53,19 +53,19 @@ % from max within peak window peakCoord = PEAK_COORD; end - + peakLOW = peakCoord - peakCOM; peakTOP = peakCoord + peakCOM; - + [cmX, cmY, cmZ] = ndgrid(gpuArray(-1*peakCOM(1):peakCOM(1)), ... - gpuArray(-1*peakCOM(2):peakCOM(2)), ... - gpuArray(-1*peakCOM(3):peakCOM(3)) ); - + gpuArray(-1*peakCOM(2):peakCOM(2)), ... + gpuArray(-1*peakCOM(3):peakCOM(3)) ); + try boX = iCCCmap(peakLOW(1):peakTOP(1), ... - peakLOW(2):peakTOP(2), ... - peakLOW(3):peakTOP(3)); - + peakLOW(2):peakTOP(2), ... + peakLOW(3):peakTOP(3)); + % Center of mass calc only makes sense for positive values boX = boX - minVal; catch @@ -76,30 +76,30 @@ iCCCmap = gather(iCCCmap); peakMask = gather(peakMask); save('xfcCalcErr.mat','peakCOM','peakCoord','peakLOW','peakTOP',... - 'iCCCmap','peakMask'); + 'iCCCmap','peakMask'); error('failed to box out COM calc, saving troubleshooting variables in xfcClacErr.mat') end - - - cMass = [ sum(sum(sum(boX.*cmX))) ; ... - sum(sum(sum(boX.*cmY))) ; ... - sum(sum(sum(boX.*cmZ))) ] ./ sum(boX(:)); - - PEAK_COORD = peakCoord + cMass'; - + + + cMass = [ sum(sum(sum(boX.*cmX))) ; ... + sum(sum(sum(boX.*cmY))) ; ... + sum(sum(sum(boX.*cmZ))) ] ./ sum(boX(:)); + + PEAK_COORD = peakCoord + cMass'; + end end else - fprintf('maxVal in iCCCmap is %f\n', maxVal); - - PEAK_COORD = [0,0,0]; + fprintf('maxVal in iCCCmap is %f\n', maxVal); + + PEAK_COORD = [0,0,0]; end PEAK_COORD = PEAK_COORD - ceil((size(iCCCmap)+1)./2); if any(isnan(PEAK_COORD)) - fprintf('%f %f %f\n', PEAK_COORD); - fprintf('Setting nans to 0\n'); - PEAK_COORD = PEAK_COORD.* ~isnan(PEAK_COORD); + fprintf('%f %f %f\n', PEAK_COORD); + fprintf('Setting nans to 0\n'); + PEAK_COORD = PEAK_COORD.* ~isnan(PEAK_COORD); end diff --git a/statistics/BH_multi_xcf_Translational_2.m b/statistics/BH_multi_xcf_Translational_2.m index 93192a50..9580ffd6 100644 --- a/statistics/BH_multi_xcf_Translational_2.m +++ b/statistics/BH_multi_xcf_Translational_2.m @@ -1,10 +1,10 @@ function [ PEAK_COORD, mip ] = BH_multi_xcf_Translational_2( rotPART_FT, REF_FT, ... - wdgMask, refWdg,... - peakMask, PEAK_COM, ... - mip) - + wdgMask, refWdg,... + peakMask, PEAK_COM, ... + mip) + %Consolodating function, calculate wedge weight and cross correlation -% +% % % Called by: % @@ -12,7 +12,7 @@ % % BH_alignClass3d % -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % @@ -23,7 +23,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % TODO: Add something to report when no valid peak is found. -% +% % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -32,15 +32,15 @@ % fixed, but only here (not rotational try -iCCCmap = fftshift(real(ifftn( ... - (refWdg.*rotPART_FT .* REF_FT .* wdgMask) ./ ... - (sqrt( ( sum( abs(REF_FT .*wdgMask).^2,'all' ) .* ... - sum( abs(rotPART_FT.*refWdg).^2,'all')))./numel(wdgMask))))); + iCCCmap = fftshift(real(ifftn( ... + (refWdg.*rotPART_FT .* REF_FT .* wdgMask) ./ ... + (sqrt( ( sum( abs(REF_FT .*wdgMask).^2,'all' ) .* ... + sum( abs(rotPART_FT.*refWdg).^2,'all')))./numel(wdgMask))))); catch save('xcf_probs.mat'); error('asdf') end - + if isnumeric(mip.x) updateVals = mip.x < iCCCmap(mip.mask); @@ -69,33 +69,33 @@ PEAK_COORD = [maxX, maxY, maxZ]; % Sometimes there is an "empty class", ignore this in xfc -if (~isnan(gather(maxVal)) && (maxVal ~= 0)) - +if (~isnan(gather(maxVal)) && (maxVal ~= 0)) + for iCOM = 1:2 if (peakCOM) if iCOM == 2 - - % figure, imshow3D(gather(iCCCmap)); pause(20) - - + + % figure, imshow3D(gather(iCCCmap)); pause(20) + + peakCoord = round(PEAK_COORD); else % from max within peak window peakCoord = PEAK_COORD; end - + peakLOW = peakCoord - peakCOM; peakTOP = peakCoord + peakCOM; - + [cmX, cmY, cmZ] = ndgrid(gpuArray(-1*peakCOM(1):peakCOM(1)), ... - gpuArray(-1*peakCOM(2):peakCOM(2)), ... - gpuArray(-1*peakCOM(3):peakCOM(3)) ); - + gpuArray(-1*peakCOM(2):peakCOM(2)), ... + gpuArray(-1*peakCOM(3):peakCOM(3)) ); + try boX = iCCCmap(peakLOW(1):peakTOP(1), ... - peakLOW(2):peakTOP(2), ... - peakLOW(3):peakTOP(3)); + peakLOW(2):peakTOP(2), ... + peakLOW(3):peakTOP(3)); catch peakCOM = gather(peakCOM); peakCoord = gather(peakCoord); @@ -104,30 +104,30 @@ iCCCmap = gather(iCCCmap); peakMask = gather(peakMask); save('xfcCalcErr.mat','peakCOM','peakCoord','peakLOW','peakTOP',... - 'iCCCmap','peakMask'); + 'iCCCmap','peakMask'); error('failed to box out COM calc, saving troubleshooting variables in xfcClacErr.mat') end - - - cMass = [ sum(sum(sum(boX.*cmX))) ; ... - sum(sum(sum(boX.*cmY))) ; ... - sum(sum(sum(boX.*cmZ))) ] ./ sum(boX(:)); - - PEAK_COORD = peakCoord + cMass'; - + + + cMass = [ sum(sum(sum(boX.*cmX))) ; ... + sum(sum(sum(boX.*cmY))) ; ... + sum(sum(sum(boX.*cmZ))) ] ./ sum(boX(:)); + + PEAK_COORD = peakCoord + cMass'; + end end else - fprintf('maxVal in iCCCmap is %f\n', maxVal); - - PEAK_COORD = [0,0,0]; + fprintf('maxVal in iCCCmap is %f\n', maxVal); + + PEAK_COORD = [0,0,0]; end PEAK_COORD = PEAK_COORD - ceil((size(iCCCmap)+1)./2); if any(isnan(PEAK_COORD)) - fprintf('%f %f %f\n', PEAK_COORD); - fprintf('Setting nans to 0\n'); - PEAK_COORD = PEAK_COORD.* ~isnan(PEAK_COORD); + fprintf('%f %f %f\n', PEAK_COORD); + fprintf('Setting nans to 0\n'); + PEAK_COORD = PEAK_COORD.* ~isnan(PEAK_COORD); end diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 4334e699..4b1d9522 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -41,7 +41,7 @@ % Output variables: % % None = files are written to disk in the current directory. This will be a -% mat file that has the +% mat file that has the % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % @@ -61,7 +61,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % TODO -% - Error checking for memory limitations +% - Error checking for memory limitations % - In testing verify "implicit" gpu arrays are actually gpu arrays % - Check binning % - Confirm position 7 is where I want to keep FSC value @@ -80,7 +80,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% if (nargin ~= 3) -% error('PARAMETER_FILE, CYCLE, PREVIOUS_PCA') + % error('PARAMETER_FILE, CYCLE, PREVIOUS_PCA') end @@ -104,7 +104,7 @@ % calculated from a random subset of the data to project the full data set % onto each of the selected principle components. when true and < 0 run % with the same parameters as before, but this time load in the adjusted -% variance maps to use as a mask for each scale space. +% variance maps to use as a mask for each scale space. % -2 use variance map, -1 use stdDev instead switch PREVIOUS_PCA @@ -127,7 +127,7 @@ flgVarianceMap = 0; flgStdDev = 0; flgLoadMask = 0; - case 1 + case 1 % case 0 and 1 same except value of PREVIOUS_PCA flgVarianceMap = 0; flgStdDev = 0; @@ -135,7 +135,7 @@ otherwise error('PREVIOUS_PCA should be 1,0,-1,-2,-3') end - + flgWMDs = 3; cycleNumber = sprintf('cycle%0.3u', CYCLE); @@ -191,7 +191,7 @@ try shape_mask_lowpass = emc.('shape_mask_lowpass'); catch - shape_mask_lowpass = 14 + 10; + shape_mask_lowpass = 14 + 10; end try @@ -200,7 +200,7 @@ shape_mask_threshold = 2.4 - 0.4; end -try +try tmpVal = emc.('whitenPS'); if (numel(tmpVal) == 3) wiener_constant = tmpVal(3); @@ -227,7 +227,7 @@ flgClassify = emc.('flgClassify'); % Removed flgGold everywhere else, but keep ability to classify full data set at -% the end (after all alignment is finished.) +% the end (after all alignment is finished.) %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of %%% these features are re-introduced, this will need to be reverted. @@ -256,8 +256,8 @@ geom_name='' if (flgMultiRefAlignment ) - geom_name='ClusterClsGeom'; - + geom_name='ClusterClsGeom'; + else geom_name='Avg_geometry'; end @@ -277,7 +277,7 @@ CUTPADDING=20 end -% % +% % % % pathList= subTomoMeta.mapPath; % % extList = subTomoMeta.mapExt; masterTM = subTomoMeta; clear subTomoMeta @@ -290,12 +290,12 @@ tomoList = fieldnames(geometry); nTomograms = length(tomoList); - + [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Cls', pixelSize); + BH_multi_maskCheck(emc, 'Cls', pixelSize); [ preMaskType, preMaskSize, preMaskRadius, preMaskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', refPixelSize); + BH_multi_maskCheck(emc, 'Ali', refPixelSize); % This is prob not a good way to make sure the mask size matches::w maskSize=preMaskSize; @@ -304,14 +304,14 @@ cpuVols = struct; [ preSizeWindow, preSizeCalc, preSizeMask, prePadWindow, prePadCalc ] = ... - BH_multi_validArea(preMaskSize,preMaskRadius, scaleCalcSize ) + BH_multi_validArea(preMaskSize,preMaskRadius, scaleCalcSize ) - [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) +[ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... + BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) -if (test_multi_ref_diffmap) +if (test_multi_ref_diffmap) refName = emc.('Raw_className'); else refName = 0; @@ -333,7 +333,7 @@ fprintf('\nReverting from %s to Raw in loading fitFSC\n','REF'); aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)) oddRot = reshape(aliParams(1,:),3,3)'; - % refine the translation per particle. + % refine the translation per particle. end clear iRefPrev end @@ -370,7 +370,7 @@ nScaleSpace = nReferences(1); pause(3); else - nReferences = [1,1]; + nReferences = [1,1]; end @@ -382,20 +382,20 @@ else halfSet = 'EVE'; end - - - - - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet); - - - [ averageMotif{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - preSizeMask); - - - + + + + + imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet); + + + [ averageMotif{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... + masterTM.(cycleNumber).(imgNAME){1}, ... + masterTM.(cycleNumber).(imgNAME){2},... + preSizeMask); + + + if ~(test_multi_ref_diffmap) averageMotif{iGold} = averageMotif{iGold}{1}; end @@ -413,29 +413,29 @@ size(averageMotif) for iRef = 1:nReferences(1) averageMotif{1}{iRef} = averageMotif{2}{iRef} + ... - BH_resample3d(gather(averageMotif{1}{iRef}), ... - oddRot, ... - aliParams(2,1:3), ... - {'Bah',1,'spline'}, 'cpu', ... - 'forward'); + BH_resample3d(gather(averageMotif{1}{iRef}), ... + oddRot, ... + aliParams(2,1:3), ... + {'Bah',1,'spline'}, 'cpu', ... + 'forward'); averageMotif{2}{iRef} = []; end end - + %%% incomplete, the idea is to generate an antialiased scaled volume for PCA -if ( refSamplingRate ~= samplingRate ) - fprintf('Resampling from %d refSampling to %d pcaSampling\n',refSamplingRate,samplingRate); - for iGold = 1:1+flgGold - averageMotif{iGold} = BH_reScale3d(averageMotif{iGold},'',sprintf('%f',1/samplingRate),'GPU'); - end - - if (flgLoadMask) - externalMask = BH_reScale3d(externalMask,'',sprintf('%f',1/samplingRate),'GPU'); - end -end - - +if ( refSamplingRate ~= samplingRate ) + fprintf('Resampling from %d refSampling to %d pcaSampling\n',refSamplingRate,samplingRate); + for iGold = 1:1+flgGold + averageMotif{iGold} = BH_reScale3d(averageMotif{iGold},'',sprintf('%f',1/samplingRate),'GPU'); + end + if (flgLoadMask) + externalMask = BH_reScale3d(externalMask,'',sprintf('%f',1/samplingRate),'GPU'); + end +end + + + prevVarianceMaps = struct(); if (flgVarianceMap) @@ -449,11 +449,11 @@ end else halfSet = 'STD'; - end - + end + % For randomsubset (PREVIOUS_PCA = 0) the suffix is *_pcaPart.mat) but % presumably we could have also just done full, so try that first - + try load(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet)) catch @@ -465,12 +465,12 @@ % parameter file, but in some data not even this may non-zero singluar % values are found, so the number could be different (lower) for iScale = 1:nScaleSpace - eigsFound = size(coeffs{iScale},1); + eigsFound = size(coeffs{iScale},1); fname = sprintf('%s_varianceMap%d-%s-%d.mrc', ... - outputPrefix, eigsFound, halfSet, iScale); + outputPrefix, eigsFound, halfSet, iScale); prevVarianceMaps.(sprintf('h%d',iGold)).(sprintf('s%d',iScale)) = ... - getVolume(MRCImage(fname)).^flgStdDev; + getVolume(MRCImage(fname)).^flgStdDev; end clear v coeffs eigsFound idxList end @@ -489,30 +489,30 @@ constrain_symmetry = false; end -if (PREVIOUS_PCA) +if (PREVIOUS_PCA) volumeMask = gpuArray(getVolume(MRCImage( ... - sprintf('%s_pcaVolMask.mrc',outputPrefix)))); + sprintf('%s_pcaVolMask.mrc',outputPrefix)))); else - + if (constrain_symmetry) gridSearch = eulerSearch(symmetry,180,5,360,5,0.0,1,true); [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter, ... - '3d', gridSearch.number_of_asymmetric_units); + '3d', gridSearch.number_of_asymmetric_units); else [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); end - + if ( flgPcaShapeMask ) % For testing we won't handle this block if (test_multi_ref_diffmap) error('test_multi_ref_diffmap is incompatible with flgPcaShapeMask') end - % when combining the addition is harmless, but is a convenient way to - % include when sets are left 100% separate. - % volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); - volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, ... - {'pca', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); - + % when combining the addition is harmless, but is a convenient way to + % include when sets are left 100% separate. + % volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); + volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, ... + {'pca', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); + end if (flgLoadMask) @@ -537,16 +537,16 @@ error('test_multi_ref_diffmap is incompatible with flgVarianceMap') end volTMP = gather(volumeMask.*prevVarianceMaps.(stHALF).(stSCALE)); - else + else volTMP = gather(volumeMask); end masks.('volMask').(stHALF).(stSCALE) = (volTMP); masks.('binary').(stHALF).(stSCALE) = (volTMP >= bh_global_binary_pcaMask_threshold); masks.('binary').(stHALF).(stSCALE) = ... - masks.('binary').(stHALF).(stSCALE)(:); + masks.('binary').(stHALF).(stSCALE)(:); masks.('binaryApply').(stHALF).(stSCALE) = (volTMP >= 0.01); - + nPixels(iGold,iScale) = gather(sum(masks.('binary').(stHALF).(stSCALE))); clear volTMP stHALF stSCALE end @@ -554,19 +554,19 @@ clear volumeMask - + % radius, convert Ang to pix , denom = equiv stdv from normal to include, e.g. % for 95% use 1/sig = 1/2 %stdDev = 1/2 .* (pcaScaleSpace ./ pixelSize - 1) .* 3.0./log(pcaScaleSpace) threeSigma = 1/3 .* (pcaScaleSpace ./ pixelSize) for iScale = 1:nScaleSpace - + kernelSize = ceil(threeSigma(iScale)) + 3; kernelSize = kernelSize + (1-mod(kernelSize,2)); masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); masks.('scaleMask').(sprintf('s%d',iScale)) - + end avgMotif_FT = cell(1+flgGold,nScaleSpace); @@ -574,14 +574,14 @@ % Here always read in both, combine if flgGold = 0 for iGold = 1:1+flgGold for iScale = 1:nScaleSpace - + if (test_multi_ref_diffmap) tmp_avg = averageMotif{iGold}{iScale}; else tmp_avg = averageMotif{iGold}; end - - + + tmp_avg = tmp_avg - mean(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); tmp_avg = tmp_avg ./ rms(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); tmp_avg = tmp_avg .* masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)); @@ -590,10 +590,10 @@ tmp_avg = EMC_convn(single(gpuArray(tmp_avg)) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); end avgMotif_FT{iGold, iScale} = ... - BH_bandLimitCenterNormalize(tmp_avg,... - BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... - masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)),... - [0,0,0;0,0,0],'single'); + BH_bandLimitCenterNormalize(tmp_avg,... + BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... + masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)),... + [0,0,0;0,0,0],'single'); % This reproduces the orginal behavior, which wrote over averageMotif. This is a bug, but who knows, it may be beneficial, so lets for now make it optional. if (test_scale_space_bug_fix) if (test_multi_ref_diffmap) @@ -603,10 +603,10 @@ end end avgFiltered{iGold, iScale} = real(ifftn(avgMotif_FT{iGold, iScale})); - + avgFiltered{iGold, iScale} = avgFiltered{iGold, iScale} - mean(avgFiltered{iGold, iScale}(masks.('binary').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); avgFiltered{iGold, iScale} = gather(avgFiltered{iGold, iScale} ./rms(avgFiltered{iGold, iScale}(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))) .* ... - masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale))); + masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale))); end end @@ -618,8 +618,8 @@ -% If randomSubset is string with a previous matfile use this, without any -% decomposition. +% If randomSubset is string with a previous matfile use this, without any +% decomposition. for iGold = 1:1+flgGold flgRefIsPadded = 0; @@ -663,82 +663,82 @@ nTOTAL = nTOTAL*emc.nPeaks; nSUBSET = nSUBSET*emc.nPeaks; end - + % Initialize array in main memory for pca - clear dataMatrix tempDataMatrix + clear dataMatrix tempDataMatrix dataMatrix = cell(3,1); tempDataMatrix = cell(3,1); for iScale = 1:nScaleSpace dataMatrix{iScale} = zeros(nPixels(iGold,iScale), nSUBSET, 'single'); tempDataMatrix{iScale} = zeros(nPixels(iGold,iScale), nTempParticles, 'single', 'gpuArray'); end - + % Pull masks onto GPU (which are cleared along with everything else when % the device is reset at the end of each loop.) gpuMasks = struct(); for iScale = 1:nScaleSpace - stSCALE = sprintf('s%d',iScale); - - gpuMasks.('volMask').(stSCALE) = ... - gpuArray(masks.('volMask').(stHALF).(stSCALE)); - gpuMasks.('binary').(stSCALE) = ... - gpuArray(masks.('binary').(stHALF).(stSCALE)); - gpuMasks.('binaryApply').(stSCALE) = ... - gpuArray(masks.('binaryApply').(stHALF).(stSCALE)); - gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); - - - gpuMasks.('highPass').(stSCALE) = BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize); + stSCALE = sprintf('s%d',iScale); + + gpuMasks.('volMask').(stSCALE) = ... + gpuArray(masks.('volMask').(stHALF).(stSCALE)); + gpuMasks.('binary').(stSCALE) = ... + gpuArray(masks.('binary').(stHALF).(stSCALE)); + gpuMasks.('binaryApply').(stSCALE) = ... + gpuArray(masks.('binaryApply').(stHALF).(stSCALE)); + gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); + + + gpuMasks.('highPass').(stSCALE) = BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize); end -% % % for iGold_inner = 1:1+flgGold -% % % for iScale = 1:nScaleSpace -% % % avgMotif_FT{iGold_inner, iScale} = ... -% % % gpuArray(cpuVols.('avgMotif_FT').(sprintf('g%d_%d',iGold_inner,iScale))); -% % % end -% % % end - - + % % % for iGold_inner = 1:1+flgGold + % % % for iScale = 1:nScaleSpace + % % % avgMotif_FT{iGold_inner, iScale} = ... + % % % gpuArray(cpuVols.('avgMotif_FT').(sprintf('g%d_%d',iGold_inner,iScale))); + % % % end + % % % end + + nExtracted = 1; nTemp = 1; nTempPrev = 0; idxList = zeros(1,nSUBSET); peakList = zeros(1,nSUBSET); - + firstLoop = true; sI = 1; nIgnored = 0; for iTomo = 1:nTomograms - - + + tomoName = tomoList{iTomo}; iGPU = 1; - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); - - + tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); + TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); + + if (flgCutOutVolumes) - volumeData = []; + volumeData = []; else [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate, iGPU, reconScaling,0); - volHeader = getHeader(volumeData); + reconCoords, mapBackIter, ... + samplingRate, iGPU, reconScaling,0); + volHeader = getHeader(volumeData); end - - nCtfGroups = masterTM.('ctfGroupSize').(tomoList{iTomo})(1); - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; - wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); -% wgtName = sprintf('cache/%s_bin%d.wgt', tomoList{iTomo},... - - - - + + nCtfGroups = masterTM.('ctfGroupSize').(tomoList{iTomo})(1); + iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; + wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); + % wgtName = sprintf('cache/%s_bin%d.wgt', tomoList{iTomo},... + + + + tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); - + fprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); @@ -746,53 +746,53 @@ % Loop over peaks inside each tomo to limit wedge mask xfer positionList = positionList(positionList(:,26) ~= -9999,:); nSubTomos = size(positionList,1); - - + + if (flgWMDs == 0) % Make a wedge mask that can be interpolated with no extrapolation for - % calculating wedge weighting in class average alignment. - + % calculating wedge weighting in class average alignment. + % COMMMMMMENT - + % make a binary wedge [ wedgeMask ]= BH_weightMask3d(sizeMask, tiltGeometry, ... - 'binaryWedgeGPU',2*maskRadius,1, 1, samplingRate); + 'binaryWedgeGPU',2*maskRadius,1, 1, samplingRate); + - error('do not do it man'); - end + end % reset for each tomogram wdgIDX = 0; radialMask = ''; - if (flgNorm) - - - % bins = 1./[1000,800,600,400,300,200,150,100,80,60,50,40,35,30,28,26,24,22,20,18,16,14,12,10,8,6,4,2]; - % bins = [0, bins]; - - [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(size(avgMotif_FT{iGold, iScale}),'Cartesian',... - 'GPU',{'none'},1,0,1); - - bins = radialGrid(1:floor(size(avgMotif_FT{iGold, iScale},1)/2),1,1); - bins = bins(bins < 0.5); - - radialMask = cell(length(bins)-1,1); - - for iBin = 1:length(bins)-1 - radialMask{iBin} = find(radialGrid >= bins(iBin) & radialGrid < bins(iBin+1)); - end - - radialGrid = ''; - + if (flgNorm) + + + % bins = 1./[1000,800,600,400,300,200,150,100,80,60,50,40,35,30,28,26,24,22,20,18,16,14,12,10,8,6,4,2]; + % bins = [0, bins]; + + [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(size(avgMotif_FT{iGold, iScale}),'Cartesian',... + 'GPU',{'none'},1,0,1); + + bins = radialGrid(1:floor(size(avgMotif_FT{iGold, iScale},1)/2),1,1); + bins = bins(bins < 0.5); + + radialMask = cell(length(bins)-1,1); + + for iBin = 1:length(bins)-1 + radialMask{iBin} = find(radialGrid >= bins(iBin) & radialGrid < bins(iBin+1)); end - + + radialGrid = ''; + + end + wdgBP = ifftshift(gpuMasks.('highPass').(sprintf('s%d',iScale))); for iSubTomo = 1:nSubTomos %%%%% %%%%% - - + + % Check that the given subTomo is not to be ignored - for now, treat % all peaks as included. The assumption is that using this will be % for initializing the project to get a good starting model. "True" @@ -801,199 +801,199 @@ includeParticle = positionList(iSubTomo, 8); iPeak=0; % make sure this exists if we are no including the particle - if (includeParticle) + if (includeParticle) make_sf3d = true; for iPeak = 0:emc.nPeaks-1 - - % Get position and rotation info, angles stored as e1,e3,e2 as in AV3 - % and PEET. This also makes inplane shifts easier to see. - - center = positionList(iSubTomo,[11:13]+26*iPeak)./samplingRate; - angles = positionList(iSubTomo,[17:25]+26*iPeak); - - - if ( make_sf3d ) - make_sf3d = false; - radialGrid = ''; - padWdg = [0,0,0;0,0,0]; - [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, ... - TLT, center,reconGeometry, wiener_constant); - end - - % If flgGold there is no change, otherwise temporarily resample the - % eve halfset to minimize differences due to orientaiton - if positionList(iSubTomo,7) == 1 % This is true for all peaks - % TODO FIXME should this be the transpose of oddRot? - angles = reshape(angles,3,3) * oddRot; - end - wedgeMask = wedgeMask .* wdgBP; - - % Find range to extract, and check for domain error. - if (flgCutOutVolumes) - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(2*CUTPADDING+sizeWindow, ... - sizeWindow, maskRadius, center); - else - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... - sizeWindow, maskRadius, center); - end - - if ~(flgGold) - shiftVAL = shiftVAL + aliParams(2,1:3)./samplingRate; - end - - if ~ischar(indVAL) - % Read in and interpolate at single precision as the local values - % in the interpolant suffer from any significant round off errors. - particleIDX = positionList(iSubTomo, 4); % Same for all peaks - - + + % Get position and rotation info, angles stored as e1,e3,e2 as in AV3 + % and PEET. This also makes inplane shifts easier to see. + + center = positionList(iSubTomo,[11:13]+26*iPeak)./samplingRate; + angles = positionList(iSubTomo,[17:25]+26*iPeak); + + + if ( make_sf3d ) + make_sf3d = false; + radialGrid = ''; + padWdg = [0,0,0;0,0,0]; + [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, ... + TLT, center,reconGeometry, wiener_constant); + end + + % If flgGold there is no change, otherwise temporarily resample the + % eve halfset to minimize differences due to orientaiton + if positionList(iSubTomo,7) == 1 % This is true for all peaks + % TODO FIXME should this be the transpose of oddRot? + angles = reshape(angles,3,3) * oddRot; + end + wedgeMask = wedgeMask .* wdgBP; + + % Find range to extract, and check for domain error. if (flgCutOutVolumes) - - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak+1); - iParticle = gpuArray(getVolume(MRCImage(particleOUT_name),... - [indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid(2*CUTPADDING+sizeWindow, ... + sizeWindow, maskRadius, center); else - - iParticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... + sizeWindow, maskRadius, center); end - - - if any(padVAL(:)) - [ iParticle ] = BH_padZeros3d(iParticle, padVAL(1,1:3), ... - padVAL(2,1:3), 'GPU', 'single'); + + if ~(flgGold) + shiftVAL = shiftVAL + aliParams(2,1:3)./samplingRate; end - - if ( use_new_interpolator ) - % Pulling in the newer interpolater from alignRaw3d_v2. There, I instantiate a new interpolator every subtomo, but it is generally - % being used many times, over the angle loop. It may be more efficient to do this outside the for subtomo loop here, but - % to start, just do it the same way. - use_only_once = true; - [ ~, iParticle ] = interpolator(gpuArray(iParticle),angles, shiftVAL, 'Bah', 'inv', symmetry, use_only_once); + + if ~ischar(indVAL) + % Read in and interpolate at single precision as the local values + % in the interpolant suffer from any significant round off errors. + particleIDX = positionList(iSubTomo, 4); % Same for all peaks + + + if (flgCutOutVolumes) + + particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak+1); + iParticle = gpuArray(getVolume(MRCImage(particleOUT_name),... + [indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + + else + + iParticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + end + + + if any(padVAL(:)) + [ iParticle ] = BH_padZeros3d(iParticle, padVAL(1,1:3), ... + padVAL(2,1:3), 'GPU', 'single'); + end + + if ( use_new_interpolator ) + % Pulling in the newer interpolater from alignRaw3d_v2. There, I instantiate a new interpolator every subtomo, but it is generally + % being used many times, over the angle loop. It may be more efficient to do this outside the for subtomo loop here, but + % to start, just do it the same way. + use_only_once = true; + [ ~, iParticle ] = interpolator(gpuArray(iParticle),angles, shiftVAL, 'Bah', 'inv', symmetry, use_only_once); + + [ ~, iWedge ] = interpolator(gpuArray(wedgeMask),angles,[0,0,0], 'Bah', 'inv', symmetry, use_only_once); + + else + % Transform the particle, and then trim to motif size + + [ iParticle ] = BH_resample3d(iParticle, angles, shiftVAL, ... + 'Bah', 'GPU', 'inv'); + + [ iWedge ] = BH_resample3d(wedgeMask, angles, [0,0,0], ... + 'Bah', 'GPU', 'inv'); + end + + + + iTrimParticle = iParticle(padWindow(1,1)+1 : end - padWindow(2,1), ... + padWindow(1,2)+1 : end - padWindow(2,2), ... + padWindow(1,3)+1 : end - padWindow(2,3)); + + - [ ~, iWedge ] = interpolator(gpuArray(wedgeMask),angles,[0,0,0], 'Bah', 'inv', symmetry, use_only_once); - - else - % Transform the particle, and then trim to motif size - - [ iParticle ] = BH_resample3d(iParticle, angles, shiftVAL, ... - 'Bah', 'GPU', 'inv'); - - [ iWedge ] = BH_resample3d(wedgeMask, angles, [0,0,0], ... - 'Bah', 'GPU', 'inv'); - end - - - - iTrimParticle = iParticle(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3)); - - - - - for iScale = 1:nScaleSpace - iPrt = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); - - iPrt = BH_bandLimitCenterNormalize( ... - iPrt .* ... - gpuMasks.('volMask').(sprintf('s%d',iScale)), ... - gpuMasks.('highPass').(sprintf('s%d',iScale)),... - gpuMasks.('binary').(sprintf('s%d',iScale)),... - [0,0,0;0,0,0],'single'); - - - - - - [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iPrt,ifftshift(iWedge),... - flgNorm,pixelSize,radialMask, padWdg); - - - - if all(isfinite(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale))))) - keepTomo = 1; - tempDataMatrix{iScale}(:,nTemp) = single(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale)))); - else - fprintf('inf or nan in subtomo %d scalePace %d',iSubTomo,iScale); - keepTomo = 0; - end - - end - clear iAvg iWmd iTrimParticle - - - - if (keepTomo) - idxList(1, nExtracted) = particleIDX; - peakList(1,nExtracted) = iPeak+1; - nExtracted = nExtracted + 1; - nTemp = nTemp + 1; - - % pull data of the gpu every 1000 particls (adjust this to max mem) - if nTemp == nTempParticles - 1 for iScale = 1:nScaleSpace - dataMatrix{iScale}(:,1+nTempPrev:nTemp+nTempPrev-1) = ... - gather(tempDataMatrix{iScale}(:,1:nTemp-1)); + + iPrt = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); + + iPrt = BH_bandLimitCenterNormalize( ... + iPrt .* ... + gpuMasks.('volMask').(sprintf('s%d',iScale)), ... + gpuMasks.('highPass').(sprintf('s%d',iScale)),... + gpuMasks.('binary').(sprintf('s%d',iScale)),... + [0,0,0;0,0,0],'single'); + + + + + + [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iPrt,ifftshift(iWedge),... + flgNorm,pixelSize,radialMask, padWdg); + + + + if all(isfinite(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale))))) + keepTomo = 1; + tempDataMatrix{iScale}(:,nTemp) = single(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale)))); + else + fprintf('inf or nan in subtomo %d scalePace %d',iSubTomo,iScale); + keepTomo = 0; + end + end - - nTempPrev = nTempPrev + nTemp - 1; - nTemp = 1; - end - else - nIgnored = nIgnored + 1; - fprintf('Ignoring subtomo %d from %s\n',iSubTomo, tomoList{iTomo}); - masterTM.(cycleNumber).(geom_name).(tomoList{iTomo})(iSubTomo, 26+iPeak*26) = -9999; - end - - - else - nIgnored = nIgnored + 1; - fprintf('Ignoring subtomo %d from %s\n',iSubTomo, tomoList{iTomo}); - masterTM.(cycleNumber).(geom_name).(tomoList{iTomo})(iSubTomo, 26+iPeak*26) = -9999; - - end % end of ignore new particles - + clear iAvg iWmd iTrimParticle + + + + if (keepTomo) + idxList(1, nExtracted) = particleIDX; + peakList(1,nExtracted) = iPeak+1; + nExtracted = nExtracted + 1; + nTemp = nTemp + 1; + + % pull data of the gpu every 1000 particls (adjust this to max mem) + if nTemp == nTempParticles - 1 + for iScale = 1:nScaleSpace + dataMatrix{iScale}(:,1+nTempPrev:nTemp+nTempPrev-1) = ... + gather(tempDataMatrix{iScale}(:,1:nTemp-1)); + end + + nTempPrev = nTempPrev + nTemp - 1; + nTemp = 1; + end + else + nIgnored = nIgnored + 1; + fprintf('Ignoring subtomo %d from %s\n',iSubTomo, tomoList{iTomo}); + masterTM.(cycleNumber).(geom_name).(tomoList{iTomo})(iSubTomo, 26+iPeak*26) = -9999; + end + + + else + nIgnored = nIgnored + 1; + fprintf('Ignoring subtomo %d from %s\n',iSubTomo, tomoList{iTomo}); + masterTM.(cycleNumber).(geom_name).(tomoList{iTomo})(iSubTomo, 26+iPeak*26) = -9999; + + end % end of ignore new particles + end % end of loop over peaks end % end of ignore if statment if ~rem(iSubTomo,100) fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... - iSubTomo, nSubTomos,iPeak+1,emc.nPeaks, iTomo,nTomograms); - + iSubTomo, nSubTomos,iPeak+1,emc.nPeaks, iTomo,nTomograms); + fprintf('Total nExtracted = %d\n', nExtracted-1); fprintf('Total nIgnored = %d\n', nIgnored); - + end end % end of the loop over subTomos - - clear volumeData + + clear volumeData end % end of the loop over Tomograms, -% % % volBinaryMask = reshape(gather(volBinaryMask),sizeMask); + % % % volBinaryMask = reshape(gather(volBinaryMask),sizeMask); for iScale = 1:nScaleSpace masks.('binary').(stHALF).(sprintf('s%d',iScale)) = ... reshape(masks.('binary').(stHALF).(sprintf('s%d',iScale)),sizeMask); end - + masterTM.(cycleNumber).('newIgnored_PCA').(halfSet) = gather(nIgnored); subTomoMeta = masterTM; save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - + for iScale = 1:nScaleSpace dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... - gather(tempDataMatrix{iScale}(:,1:nTemp-1)); + gather(tempDataMatrix{iScale}(:,1:nTemp-1)); end - + clear tempDataMatrix % Get rid of any zero vals from newly ignored particles which are there due to % pre-allocation. Assuming no zeros have found their way in anywhere else which @@ -1006,13 +1006,13 @@ dataMatrix{iScale} = dataMatrix{iScale}(:,1:size(idxList,2)); % Center the rows for row = 1:size(dataMatrix{iScale},1) - dataMatrix{iScale}(row,:) = dataMatrix{iScale}(row,:) - mean(double(dataMatrix{iScale}(row,:))); + dataMatrix{iScale}(row,:) = dataMatrix{iScale}(row,:) - mean(double(dataMatrix{iScale}(row,:))); end end - - + + %save('preparpoolSave.mat'); try EMC_parpool(nCores); @@ -1020,7 +1020,7 @@ delete(gcp('nocreate')); EMC_parpool(nCores); end - + if (previousPCA) % Read the matrix of eigenvectors from the prior PCA. oldPca = load(previousPCA); @@ -1028,15 +1028,15 @@ clear oldPca; sDiag = cell(nScaleSpace,1); coeffs = cell(nScaleSpace,1); - + for iScale = 1:nScaleSpace % Sanity checks on the dimensionality numEigs = size(U{iScale}, 2); if nPixels(iGold,iScale) ~= size(U{iScale}, 1) error('Image size %d does not match that of previous PCA %d!', nPixels(iGold,iScale), size(U{iScale},1)); end - - coeffs{iScale} = U{iScale}' * dataMatrix{iScale}; + + coeffs{iScale} = U{iScale}' * dataMatrix{iScale}; end else @@ -1047,54 +1047,54 @@ coeffs = cell(nScaleSpace,1); varianceMap = cell(nScaleSpace,1); for iScale = 1:nScaleSpace - - krylovScalar = 5; + + krylovScalar = 5; % Calculate the decomposition [ U{iScale},S{iScale},V{iScale}, convergenceFlag ] = svds(double(dataMatrix{iScale}), ... - maxEigs, 'largest', ... - 'MaxIterations',1000, ... % default 300 - 'SubspaceDimension',max(krylovScalar*maxEigs,30),... % default max(3*maxEigs,15) - 'Display',true); % Diagnostics default false (will this work in compiled?) - U{iScale} = single(U{iScale}); - S{iScale} = single(S{iScale}); - V{iScale} = single(V{iScale}); - -% [U{iScale},S{iScale},V{iScale}] = svd(dataMatrix{iScale}, 0); - + maxEigs, 'largest', ... + 'MaxIterations',1000, ... % default 300 + 'SubspaceDimension',max(krylovScalar*maxEigs,30),... % default max(3*maxEigs,15) + 'Display',true); % Diagnostics default false (will this work in compiled?) + U{iScale} = single(U{iScale}); + S{iScale} = single(S{iScale}); + V{iScale} = single(V{iScale}); + + % [U{iScale},S{iScale},V{iScale}] = svd(dataMatrix{iScale}, 0); + sDiag{iScale} = diag(S{iScale}); numNonZero = find(( sDiag{iScale} ~= 0 ), 1, 'last'); % For Method 1, save eigenvectors 1-4 (or user-specified max) as images eigsFound = min(maxEigs, numNonZero); - + fprintf('Found %d / %d non-zero eigenvalues sum = %4.4f, in set %s.\n All singular values converged is t/f ( %d ) ', ... - numNonZero, size(S{iScale}, 1), sum(sDiag{iScale}), halfSet, convergenceFlag); - - coeffs{iScale} = S{iScale} * V{iScale}' - + numNonZero, size(S{iScale}, 1), sum(sDiag{iScale}), halfSet, convergenceFlag); + + coeffs{iScale} = S{iScale} * V{iScale}' + % Can be GB-TB if calculated full %varianceMap{iScale} = (U{iScale}*S{iScale}.^2*V{iScale} ./ numel(U{iScale}-1)); fprintf('Size S, %d %d Size U %d %d \n', size(S{iScale},1),size(S{iScale},2), size(U{iScale},1),size(U{iScale},2)); - + % We want the diagnol of US^2U'/ n-1 % This will be maxEigs * Nvoxels matrix (U is Nvoxels * maxEigs) rightSide = S{iScale}(1:numNonZero,1:numNonZero).^2*U{iScale}'; varianceMap = zeros(nPixels(iGold,iScale),1); for k = 1:nPixels(iGold,iScale) -% varianceMap(k) = U{iScale}(k,1)*rightSide(1,k) + ... -% U{iScale}(k,2)*rightSide(2,k) + ... -% U{iScale}(k,3)*rightSide(3,k); + % varianceMap(k) = U{iScale}(k,1)*rightSide(1,k) + ... + % U{iScale}(k,2)*rightSide(2,k) + ... + % U{iScale}(k,3)*rightSide(3,k); varianceMap(k) = U{iScale}(k,:)*rightSide(:,k); end varianceMap = varianceMap ./ (numel(varianceMap) - 1); tmpReshape = zeros(prod(sizeMask),1); tmpReshape(masks.('binary').(stHALF).(sprintf('s%d',iScale)) ) = varianceMap(:); - - fname = sprintf('%s_varianceMap%d-%s-%d.mrc',outputPrefix, eigsFound, halfSet, iScale); + + fname = sprintf('%s_varianceMap%d-%s-%d.mrc',outputPrefix, eigsFound, halfSet, iScale); SAVE_IMG(MRCImage(single(gather(reshape(tmpReshape, sizeMask)))), fname,pixelSize); - - - + + + eigList = cell(eigsFound,1); eigList_SUM = cell(eigsFound,1); @@ -1107,44 +1107,44 @@ eigList{iEig,1} = gather(eigenImage); eigList_SUM{iEig,1} = gather((eigenImage + avgFiltered{iGold, iScale} )./2); end - - + + [ eigMont ] = BH_montage4d(eigList, 'eigMont'); [ eigMont_SUM ] = BH_montage4d(eigList_SUM, 'eigMont_SUM'); - fname = sprintf('%s_eigenImage%d-%s-mont_%d.mrc',outputPrefix, eigsFound, halfSet, iScale); - fname_SUM = sprintf('%s_eigenImage%d-SUM-%s-mont_%d.mrc',outputPrefix, eigsFound, halfSet, iScale); - SAVE_IMG(MRCImage(single(gather(eigMont))), fname,pixelSize); - SAVE_IMG(MRCImage(single(gather(eigMont_SUM))), fname_SUM,pixelSize); - - + fname = sprintf('%s_eigenImage%d-%s-mont_%d.mrc',outputPrefix, eigsFound, halfSet, iScale); + fname_SUM = sprintf('%s_eigenImage%d-SUM-%s-mont_%d.mrc',outputPrefix, eigsFound, halfSet, iScale); + SAVE_IMG(MRCImage(single(gather(eigMont))), fname,pixelSize); + SAVE_IMG(MRCImage(single(gather(eigMont_SUM))), fname_SUM,pixelSize); + + % If requested, limit the number of principal components and coeffs saved if maxEigs < size(S{iScale}, 1) fprintf('Saving only the first %d principal components.\n', ... maxEigs); if ~isempty(U{iScale}) % U will not exist for pcaMethods 2 or 3 - U{iScale} = U{iScale}(:, 1:maxEigs); + U{iScale} = U{iScale}(:, 1:maxEigs); end if ~(previousPCA) - S{iScale} = S{iScale}(1:maxEigs, 1:maxEigs); - V{iScale} = V{iScale}(:, 1:maxEigs); + S{iScale} = S{iScale}(1:maxEigs, 1:maxEigs); + V{iScale} = V{iScale}(:, 1:maxEigs); end coeffs{iScale} = coeffs{iScale}(1:maxEigs, :); % end end end - - + + % Only U is needed for further analysis, so save only this, unless % troubleshooting. if (previousPCA) for iScale = 1:nScaleSpace - + % If requested, limit the number of principal components and coeffs saved. if maxEigs < size(U{iScale}, 2) fprintf('Saving only the first %d principal components.\n', ... p.pcaMaxNumComponents); - U{iScale} = U{iScale}(:, 1:maxEigs); - coeffs{iScale} = coeffs{iScale}(1:maxEigs, :); + U{iScale} = U{iScale}(:, 1:maxEigs); + coeffs{iScale} = coeffs{iScale}(1:maxEigs, :); end end @@ -1158,17 +1158,17 @@ end fprintf('Total execution time on %s set: %f seconds\n', halfSet, etime(clock, startTime)); - - close all force; + + close all force; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -clear dataMatrix U S V coeffs eigMont eigMontSum - -delete(gcp('nocreate')); - -% after resetting the device, bring back masks etc. - - + + clear dataMatrix U S V coeffs eigMont eigMontSum + + delete(gcp('nocreate')); + + % after resetting the device, bring back masks etc. + + end % end of loop over halfsets gpuDevice(1); delete(gcp('nocreate')); diff --git a/statistics/BH_rocCurve.m b/statistics/BH_rocCurve.m index c6295d65..345d0023 100644 --- a/statistics/BH_rocCurve.m +++ b/statistics/BH_rocCurve.m @@ -63,14 +63,14 @@ [bx, by] = BH_multi_gridCoordinates(2.*[com_radius,com_radius]+1,'Cartesian','cpu',{'none'},0,1,0); for iPeak = 1:nPeaks - + % First scan the area for a maximum dx = gt(iPeak,1)./pixelSize + ox; dy = gt(iPeak,2)./pixelSize + oy; - + ix = floor(dx); iy = floor(dy); - + % TODO out of bounds heck cSq = mip(ix-thr_radius:ix+thr_radius, iy - thr_radius:iy+thr_radius); [m,c] = max(cSq(:)); @@ -78,27 +78,27 @@ [i,j] = ind2sub(2.*[thr_radius,thr_radius]+1,c); i = i - thr_radius - 1; j = j - thr_radius - 1; - + comSq = gather(mip(ix-com_radius + i:ix+com_radius + i, iy - com_radius + j:iy+com_radius+ j)); - + com = [sum(bx(:).*comSq(:)),sum(by(:).*comSq(:))]./sum(comSq(:)); m2 = interpn(bx,by,comSq,com(1),com(2),'cubic'); fprintf('found a max of %3.3f, %3.3f ,for peak %d\n',m2,m2/m,iPeak); - + peakList(iPeak) = m2; - + mip(ix-com_radius + i:ix+com_radius + i, iy - com_radius + j:iy+com_radius+ j) = 0; - + end eraseNeighbors=0; for iVal = 1:nVals - - + + if (eraseNeighbors) - + kernel = ones([3,3],'single','gpuArray'); % Remove all values that have more than two neighbors m1 = mip >= snr(iVal); @@ -106,35 +106,35 @@ m2 = convn((m1),kernel,'same'); m1 = m1 .* (m1 < iN); end - + nSum = sum(m1(:)); % Now add in the central pixel and take diff kernel(5) = 1; FP = sum(sum(convn(single(m1),kernel,'same'))) - nSum; - + fprintf('snr %2.2f, m1 %d, m2 %d, FP %d \n', snr(iVal),sum(mip(:) > snr(iVal)), sum(m1(:)), FP); else FP = sum(mip(:) > snr(iVal)); end - % True peaks are already zeroed) + % True peaks are already zeroed) TP = sum(peakList >= snr(iVal)); FN = nPeaks - TP; TN = d1*d2 - TP - FN - FP; - + MatthewsCCC = (TP * TN - FP * FN) ./ sqrt((TP+FP).*(TP+FN).*(TN+FP).*(TN+FN)); - - + + RECALL = TP ./ nPeaks; PRECISION = TP ./ (TP + FP); F1 = 2. * (RECALL*PRECISION)/(RECALL+PRECISION); - + roc(iVal,:) = gather( ... - [ snr(iVal), ... - RECALL, ... - PRECISION, ... - F1,... - MatthewsCCC]); - + [ snr(iVal), ... + RECALL, ... + PRECISION, ... + F1,... + MatthewsCCC]); + end fout = fopen(sprintf('%s_roc.txt',mip_name),'w'); @@ -152,21 +152,21 @@ snrMax = max(find(a(:,3)>0,1,'last'),find(a(:,4)>0,1,'last')) + 1; snrMax = a(snrMax,1); -figure('visible','off'), +figure('visible','off'), - subplot(1,3,[1:2]); - plot(roc(:,1),roc(:,2),'r',roc(:,1),roc(:,3),'b', ... - snr(cM),roc(cM,5),'c*',... - ones(length(0:0.1:1),1).*roc(onefalsePos,1),0:0.1:1,'k--'); - title({sprintf('1 FP @ %2.2f (%2.2f calc) SNR and %2.2f recall\nmCCC,rec,prec,snr\n [%2.2f %2.2f %2.2f %2.2f]',roc(onefalsePos,[1]),gaussSNR,roc(onefalsePos,[2]),roc(cM,5),snr(cM),roc(cM,[2:3]))}); - xlabel('SNR'); ylabel('Recall (red) Precision (blue)'); +subplot(1,3,[1:2]); +plot(roc(:,1),roc(:,2),'r',roc(:,1),roc(:,3),'b', ... + snr(cM),roc(cM,5),'c*',... + ones(length(0:0.1:1),1).*roc(onefalsePos,1),0:0.1:1,'k--'); +title({sprintf('1 FP @ %2.2f (%2.2f calc) SNR and %2.2f recall\nmCCC,rec,prec,snr\n [%2.2f %2.2f %2.2f %2.2f]',roc(onefalsePos,[1]),gaussSNR,roc(onefalsePos,[2]),roc(cM,5),snr(cM),roc(cM,[2:3]))}); +xlabel('SNR'); ylabel('Recall (red) Precision (blue)'); + +subplot(1,3,3); +plot(a(:,1),log(a(:,3)),'k--',a(:,1),log(a(:,4)),'b'); +title({'Survival histogram'}) +xlabel('SNR'); ylabel('log(counts)'); +xlim([0,snrMax]); ylim([0.9,inf]); - subplot(1,3,3); - plot(a(:,1),log(a(:,3)),'k--',a(:,1),log(a(:,4)),'b'); - title({'Survival histogram'}) - xlabel('SNR'); ylabel('log(counts)'); - xlim([0,snrMax]); ylim([0.9,inf]); - saveas(gcf,sprintf('%s_roc.pdf',mip_name),'pdf'); diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index ebc3d3aa..c3d53aa9 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -22,7 +22,7 @@ try flgColorMap = emc.('flgColorMap'); -catch +catch flgColorMap = 0; end @@ -57,7 +57,7 @@ end - + CYCLE = EMC_str2double(CYCLE); cycle_numerator = ''; cycle_denominator =''; @@ -76,7 +76,7 @@ cycle_denominator = CYCLE(3); CYCLE = CYCLE(1); flgAltRun = 1; - + else flgAltRun = 0; % Could just use one flag for RunAlignments and ALt Run end @@ -88,9 +88,9 @@ reconScaling = 1; samplingRate = emc.('Ali_samplingRate'); -% used to determine the number of fiducials/patch for local area. +% used to determine the number of fiducials/patch for local area. MOL_MASS = emc.('particleMass'); -molMass = MOL_MASS.*(25/samplingRate); +molMass = MOL_MASS.*(25/samplingRate); try tomoCPR_random_subset = emc.('tomoCPR_randomSubset') @@ -100,18 +100,18 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%% Parameters I am currently experimenting with as of Jan 2018 - try - rmsScale = emc.('rmsScale'); - catch - % Larger RMS downweights the contribution of the tomogram. The use of - % the molecular mass in MDa fits with experiments from the ribosome that - % seem to be best ~ 4 and with HIV ~ 0.5 however it may be that the - % number of subtomograms contributing to the average should also be - % considered to then work back to an estimate of the SNR in the particle - % (tomogram) volume. - rmsScale = sqrt(MOL_MASS); - end - +try + rmsScale = emc.('rmsScale'); +catch + % Larger RMS downweights the contribution of the tomogram. The use of + % the molecular mass in MDa fits with experiments from the ribosome that + % seem to be best ~ 4 and with HIV ~ 0.5 however it may be that the + % number of subtomograms contributing to the average should also be + % considered to then work back to an estimate of the SNR in the particle + % (tomogram) volume. + rmsScale = sqrt(MOL_MASS); +end + try probabilityPeakiness = emc.('probPeakiness'); catch @@ -123,7 +123,7 @@ catch % While it seems like using the per fiducial defocus max for the % refinement makes sense, have the default be the average of the - % projection + % projection useAverageDefocus = 0; end @@ -135,27 +135,27 @@ % Used to calc defocus values using tilt instead of manually. Convention % diff. - flgInvertTiltAngles = 0; +flgInvertTiltAngles = 0; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Playing around with the model n_surfaces=2; -try - rot_option_global = emc.('rot_option_global'); +try + rot_option_global = emc.('rot_option_global'); catch - rot_option_global = 1; + rot_option_global = 1; end -try - rot_option_local = emc.('rot_option_local'); +try + rot_option_local = emc.('rot_option_local'); catch rot_option_local = 1; end -try - rot_default_grouping_global = emc.('rot_default_grouping_global'); +try + rot_default_grouping_global = emc.('rot_default_grouping_global'); catch rot_default_grouping_global = 3; end -try +try rot_default_grouping_local = emc.('rot_default_grouping_local'); catch rot_default_grouping_local = 3; @@ -242,7 +242,7 @@ end if (use_PCF) - error('The PCF scaling is not working correctly, please set use_PCF=0'); + error('The PCF scaling is not working correctly, please set use_PCF=0'); end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -252,10 +252,10 @@ % so override the tmpCache. tmpCache = ''; else - tmpCache= emc.('fastScratchDisk'); + tmpCache= emc.('fastScratchDisk'); end -if strcmpi(tmpCache, 'ram') +if strcmpi(tmpCache, 'ram') if isempty(getenv('EMC_CACHE_MEM')) fprintf('Did not find a variable for EMC_CACHE_MEM\nSkipping ram\n'); tmpCache= ''; @@ -277,14 +277,14 @@ gpuScale=3*samplingRate; nWorkers = min(nGPUs*gpuScale,emc.('nCpuCores')); % 18 fprintf('Using %d workers as max of %d %d*nGPUs and %d nWorkers visible\n', ... - nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); - + nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); + % Check to make sure it even exists if isempty(dir(tmpCache)) - fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); - tmpCache = ''; + fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); + tmpCache = ''; end -if isempty(tmpCache) +if isempty(tmpCache) tmpCache='cache/'; flgCleanCache = 0; CWD = ''; @@ -311,7 +311,7 @@ % Add error check onrange for reasonable values. -ctfRange = emc.('tomoCprDefocusRange')*10^10; +ctfRange = emc.('tomoCprDefocusRange')*10^10; ctfInc = emc.('tomoCprDefocusStep')*10^10; calcCTF = emc.('tomoCprDefocusRefine'); @@ -319,7 +319,7 @@ [tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); - + if (flgAltRun && ~skip_to_the_end_and_run) nParts = ceil(nTiltSeries ./ cycle_denominator); tiltStart = 1+(cycle_numerator - 1)*nParts; @@ -331,7 +331,7 @@ % Cycle 0 is named differently - I'll be deleting this in an overhaul of the way % the subTomoMeta is written. if (CYCLE) - try + try fprintf('Using Alignment geometry %s\n',cycleNumber); geometry = subTomoMeta.(cycleNumber).RawAlign; catch @@ -339,7 +339,7 @@ geometry = subTomoMeta.(cycleNumber).Avg_geometry; end else - try + try fprintf('Using Alignment geometry %s\n',cycleNumber); geometry = subTomoMeta.(cycleNumber).RawAlign; catch @@ -352,7 +352,7 @@ % Load in the reference images. refVol = cell(2,1); - + refName = emc.('Raw_className'); try @@ -371,23 +371,23 @@ particleMask = cell(nRefs,1); for iGold = 1:2 - + if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - - - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) - + + + imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) + iHeader = getHeader(MRCImage(subTomoMeta.(cycleNumber).(imgNAME){1},0)); sizeWindow = iHeader.nZ.*[1,1,1]; [ refVol{iGold} ] = BH_unStackMontage4d(1:nRefs, ... - subTomoMeta.(cycleNumber).(imgNAME){1}, ... - subTomoMeta.(cycleNumber).(imgNAME){2},... - sizeWindow); - + subTomoMeta.(cycleNumber).(imgNAME){1}, ... + subTomoMeta.(cycleNumber).(imgNAME){2},... + sizeWindow); + end @@ -405,17 +405,17 @@ % refNameODD = sprintf('%s_%s_class0_REF_ODD.mrc', ... % cycleNumber,emc.('subTomoMeta')); % refNameEVE = sprintf('%s_%s_class0_REF_EVE.mrc', ... -% cycleNumber,emc.('subTomoMeta')); +% cycleNumber,emc.('subTomoMeta')); % refODD = getVolume(MRCImage(refNameODD)); -% refEVE = getVolume(MRCImage(refNameEVE)); +% refEVE = getVolume(MRCImage(refNameEVE)); % catch % fprintf('\nDid not find either %s or %s, trying Raw prefix\n',refNameODD,refNameEVE); % try % refNameODD = sprintf('%s_%s_class0_Raw_ODD.mrc', ... % cycleNumber,emc.('subTomoMeta')); % refNameEVE = sprintf('%s_%s_class0_Raw_EVE.mrc', ... -% cycleNumber,emc.('subTomoMeta')); -% +% cycleNumber,emc.('subTomoMeta')); +% % refODD = getVolume(MRCImage(refNameODD)); % refEVE = getVolume(MRCImage(refNameEVE)); % catch @@ -435,15 +435,15 @@ mbOUT = {[tmpCache],[mapBackIter+1],'dummy'}; fprintf('\nmBOUT name is %smapBack%d/%s\n',mbOUT{1:3}); - + for iTiltSeries = tiltStart:nTiltSeries if (skip_to_the_end_and_run) continue; end - + tiltNameList mapBackRePrjSize = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize'); -% % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); + % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos if nTomograms == 0 % No points were saved after template matching so skip this tilt series @@ -452,46 +452,46 @@ end - + tiltList = cell(nTomograms,1); - % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); + % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); tomoList = {}; tomoIDX = 1; for iTomo = 1:size(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords,1) % This is dumb, fix it to be explicit. if any(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords(iTomo,:)) tomoList{tomoIDX} = sprintf('%s_%d',tiltNameList{iTiltSeries},iTomo); - - - - tiltList{tomoIDX} = sprintf('%saliStacks/%s_ali%d.fixed',... - CWD,tiltNameList{iTiltSeries},mapBackIter+1); - outCTF='_ctf'; - - % Only increment if values found. - tomoIDX = tomoIDX + 1; + + + + tiltList{tomoIDX} = sprintf('%saliStacks/%s_ali%d.fixed',... + CWD,tiltNameList{iTiltSeries},mapBackIter+1); + outCTF='_ctf'; + + % Only increment if values found. + tomoIDX = tomoIDX + 1; end end [~,tiltBaseName,~] = fileparts(tiltList{1}); mbOUT{3} = tiltBaseName; - - + + if (mapBackIter) localFile = sprintf('%smapBack%d/%s_ali%d_ctf.local', ... - CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter) + CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter) else localFile = sprintf('%sfixedStacks/%s.local',CWD,tiltNameList{iTiltSeries}) end - + if exist(localFile,'file') fprintf('Found local file\n.'); else fprintf('No local transforms requested.\n'); localFile = 0; end - + % For now assume that all of these are the same - this is a shitty way to % handle it, but keeps things general and simple. reconRotation = zeros(nTomograms,3); @@ -507,20 +507,20 @@ end pixelSize = fullPixelSize.*samplingRate; -try - eraseMaskType = emc.('Peak_mType'); - eraseMaskRadius = emc.('Peak_mRadius')./pixelSize; - fprintf('Further restricting peak search to radius %f %f %f\n',... - eraseMaskRadius); - eraseMask = 1; -catch - eraseMask = 0; - fprintf('\n'); -end - - + try + eraseMaskType = emc.('Peak_mType'); + eraseMaskRadius = emc.('Peak_mRadius')./pixelSize; + fprintf('Further restricting peak search to radius %f %f %f\n',... + eraseMaskRadius); + eraseMask = 1; + catch + eraseMask = 0; + fprintf('\n'); + end + + [ ~,~,maskRadius,~ ] = BH_multi_maskCheck(emc,'Ali',pixelSize) - PARTICLE_RADIUS = floor(max(emc.('particleRadius')./pixelSize)); + PARTICLE_RADIUS = floor(max(emc.('particleRadius')./pixelSize)); %PARTICLE_RADIUS = floor(mean(emc.('particleRadius')./pixelSize)); % TODO, is this too restricted? @@ -528,7 +528,7 @@ try lowPassCutoff = emc.('tomoCprLowPass'); fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n.',... - lowPassCutoff); + lowPassCutoff); catch % TODO are these range limits okay? lowPassCutoff = 1.5.*mean(subTomoMeta.currentResForDefocusError); @@ -538,16 +538,16 @@ lowPassCutoff = 24; end fprintf('Using an internatlly determined lowpass cutoff of %3.3f Ang\n.',... - lowPassCutoff); + lowPassCutoff); end if lowPassCutoff < 2* pixelSize fprintf('Psych, the cutoff is being set to Nyquist'); lowPassCutoff = 2*pixelSize; end - min_res_for_ctf_fitting = 10.0; + min_res_for_ctf_fitting = 10.0; if (calcCTF) - try + try min_res_for_ctf_fitting = emc.('min_res_for_ctf_fitting'); catch end @@ -557,42 +557,42 @@ calcCTF = false; end end - -% % % % % targetPatchSize = max(500, ceil(2.*(PARTICLE_RADIUS).*sqrt(nFiducialsPerPatch))) - + + % % % % % targetPatchSize = max(500, ceil(2.*(PARTICLE_RADIUS).*sqrt(nFiducialsPerPatch))) + nFiducialsPerPatch = ceil(100./sqrt(molMass)) targetPatchSize = max(500, ceil(2.*(PARTICLE_RADIUS).*sqrt(nFiducialsPerPatch))) - -% % % -% % % % Check to see if this tilt has already been worked on, if so skip -% % % aliCmdFileCheck = sprintf('%smapBack%d/%s.align',mbOUT{1:3}); -% % % if exist(aliCmdFileCheck,'file') -% % % fprintf('\n\nFound aliCmdFileCheck, skipping rather than overwrite.\n'); -% % % continue -% % % end - - - - + + % % % + % % % % Check to see if this tilt has already been worked on, if so skip + % % % aliCmdFileCheck = sprintf('%smapBack%d/%s.align',mbOUT{1:3}); + % % % if exist(aliCmdFileCheck,'file') + % % % fprintf('\n\nFound aliCmdFileCheck, skipping rather than overwrite.\n'); + % % % continue + % % % end + + + + if (samplingRate > 1) - + for iTomo = 1:nTomograms - [~, tltName, tltExt] = fileparts(tiltList{iTomo}); - + [~, tltName, tltExt] = fileparts(tiltList{iTomo}); + % Resample the tilt if necessary, then modify the tilt list - - BH_multi_loadOrBin(tiltList{iTomo},-1.*samplingRate, 2); - tiltList{iTomo} = sprintf('%scache/%s_bin%d%s', ... - CWD,tltName, samplingRate,tltExt); - - - - - + + BH_multi_loadOrBin(tiltList{iTomo},-1.*samplingRate, 2); + tiltList{iTomo} = sprintf('%scache/%s_bin%d%s', ... + CWD,tltName, samplingRate,tltExt); + + + + + end end - + % Check that an existing mapBack dir doen't exist, if so move to backup if exist(sprintf('mapBack%d',mapBackIter+1), 'dir') [y,m,d] = ymd(datetime); @@ -601,10 +601,10 @@ clear y m d h mi s end system(sprintf('mkdir -p %smapBack%d',tmpCache,mapBackIter+1)); - - - - + + + + % re-initialize the parpool for each tilt series to free up mem. if ~isempty(gcp('nocreate')) @@ -614,22 +614,22 @@ EMC_parpool(nWorkers); end fprintf('init with %d workers\n',nWorkers); - + outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); - - - - - - + + + + + + % Get the thickest for recon maxZ = 0; overSampleZforProjection = 1.0; tiltHeader = getHeader(MRCImage(tiltList{1},0)); - + for iTomo = 1:nTomograms - + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; nZdZ = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,[4,6])./samplingRate @@ -639,103 +639,103 @@ if sZneeded > maxZ maxZ = sZneeded; end - + clear tomoNumber nZdZ end maxZ = maxZ + (samplingRate*2); fprintf('combining thickness and shift, found a maxZ of %d\n',maxZ); - + % xyzproj assumes centered in Z, so add extra height for z offsets to create % the true "in microsope" dimension - - reconstructionSize = [tiltHeader.nX,tiltHeader.nY,maxZ] - originRec = ceil((reconstructionSize+1)./2) - avgTomo = cell(3,1); - - - avgSampling = zeros(reconstructionSize,'uint8'); - % These two are mutually exclusive for now, but not enforced. - if (flgClassAvg) - avgColor = zeros(reconstructionSize, 'int16'); - end - - if (flgColorMap) - avgColor = zeros(reconstructionSize, 'int16'); - end - + + reconstructionSize = [tiltHeader.nX,tiltHeader.nY,maxZ] + originRec = ceil((reconstructionSize+1)./2) + avgTomo = cell(3,1); + + + avgSampling = zeros(reconstructionSize,'uint8'); + % These two are mutually exclusive for now, but not enforced. + if (flgClassAvg) + avgColor = zeros(reconstructionSize, 'int16'); + end + + if (flgColorMap) + avgColor = zeros(reconstructionSize, 'int16'); + end + % as the projection of the 3dModel with tilt will use this file and it % must match the zCoords in the defAng file. tomoList{1} pause(3) TLT = tiltGeometry.(tomoList{1}); - - + + iRawTltName = sprintf('%smapBack%d/%s_align.rawtlt',mbOUT{1:3}) iTiltFile = fopen(iRawTltName, 'w'); rawTLT = sortrows(TLT(:,[1,4]),1); fprintf(iTiltFile,'%f\n',rawTLT(:,2)'); - fclose(iTiltFile); + fclose(iTiltFile); % Test this out with the full reconstruction, should enforce zeroing % past the first CTF zero. For now just flip blindly. No Offsets should % be needed. % There is a gpu clear inside that is a prob. Not sure how to handle. % I could run outside the loop, but that would be disk space heave - for iRef = 1:nRefs - refVol{1}{iRef} = gather(refVol{1}{iRef}); - refVol{2}{iRef} = gather(refVol{2}{iRef}); - particleMask{iRef} = gather(particleMask{iRef}); - end - - sprintf('[%d,%d]',maxZ,samplingRate) - tiltNameList{iTiltSeries} - - + for iRef = 1:nRefs + refVol{1}{iRef} = gather(refVol{1}{iRef}); + refVol{2}{iRef} = gather(refVol{2}{iRef}); + particleMask{iRef} = gather(particleMask{iRef}); + end + + sprintf('[%d,%d]',maxZ,samplingRate) + tiltNameList{iTiltSeries} + + backgroundName = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec',CWD,tiltNameList{iTiltSeries},1, samplingRate); -% emClarity('internal','ctf','3d',PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries},'dummy'); + % emClarity('internal','ctf','3d',PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries},'dummy'); BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries},'dummy'); - % re-initialize the parpool for each tilt series to free up mem. - delete(gcp('nocreate')) - EMC_parpool(nWorkers); - + % re-initialize the parpool for each tilt series to free up mem. + delete(gcp('nocreate')) + EMC_parpool(nWorkers); + avgTomo{1} = getVolume(MRCImage(backgroundName)); - system(sprintf('rm %s',backgroundName)); - - for iRef = 1:nRefs - refVol{1}{iRef} = gpuArray(refVol{1}{iRef}); - refVol{2}{iRef} = gpuArray(refVol{2}{iRef}); - particleMask{iRef} = gpuArray(particleMask{iRef}); - end - avgTomo{1} = avgTomo{1} ./ (overSampleZforProjection.*rmsScale*rms(avgTomo{1}(:))); - -% % % % Now reset the binned tilt to the non-ctf corrected. Could probably -% % % % just temporarily rename, but for testing do this. -% % % if (samplingRate > 1) -% % % rmTiltName = sprintf('%scache/%s_ali%d_bin%d.fixed', ... -% % % CWD,tiltNameList{iTiltSeries}, mapBackIter+1, samplingRate); -% % % % Force removal so that a binned version of the ctf stack will be -% % % % created -% % % system(sprintf('rm %s',rmTiltName)); -% % % -% % % % Resample the tilt if necessary, then modify the tilt list -% % % BH_multi_loadOrBin(sprintf('aliStacks/%s_ali%d.fixed',tiltNameList{iTiltSeries}, mapBackIter+1),-1.*samplingRate, 2); -% % % end - -% avgTomo{1} = zeros(reconstructionSize,'single'); - - - + system(sprintf('rm %s',backgroundName)); + + for iRef = 1:nRefs + refVol{1}{iRef} = gpuArray(refVol{1}{iRef}); + refVol{2}{iRef} = gpuArray(refVol{2}{iRef}); + particleMask{iRef} = gpuArray(particleMask{iRef}); + end + avgTomo{1} = avgTomo{1} ./ (overSampleZforProjection.*rmsScale*rms(avgTomo{1}(:))); + + % % % % Now reset the binned tilt to the non-ctf corrected. Could probably + % % % % just temporarily rename, but for testing do this. + % % % if (samplingRate > 1) + % % % rmTiltName = sprintf('%scache/%s_ali%d_bin%d.fixed', ... + % % % CWD,tiltNameList{iTiltSeries}, mapBackIter+1, samplingRate); + % % % % Force removal so that a binned version of the ctf stack will be + % % % % created + % % % system(sprintf('rm %s',rmTiltName)); + % % % + % % % % Resample the tilt if necessary, then modify the tilt list + % % % BH_multi_loadOrBin(sprintf('aliStacks/%s_ali%d.fixed',tiltNameList{iTiltSeries}, mapBackIter+1),-1.*samplingRate, 2); + % % % end + + % avgTomo{1} = zeros(reconstructionSize,'single'); + + + if (flgColorMap) avgColor = zeros(reconstructionSize, 'int16'); end - + if (buildTomo) coordOUT = fopen(sprintf('%smapBack%d/%s.coord',mbOUT{1:3}),'w'); coordSTART = fopen(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3}),'w'); - + defOUT = fopen(sprintf('%smapBack%d/%s.defAng',mbOUT{1:3}),'w'); end @@ -743,9 +743,9 @@ % aggressivley downweight outliers in the alignment nFidsTotal = 0; for iTomo = 1:nTomograms - + TLT = tiltGeometry.(tomoList{iTomo}); - + doseList = TLT(:,[1,11]); postExposure = doseList(:,2)'; @@ -756,7 +756,7 @@ % Extract a "defocus file" for tilt to calculate the defocus for each % fiducial also considering the local alignment. If this works, I can - % get rid of defAng + % get rid of defAng iDefocusFileName = sprintf('%smapBack%d/%s_align.defocus',mbOUT{1:3}); iDefocusFile = fopen(iDefocusFileName,'w'); defTLT = sortrows(TLT(:,[1,15]),1); @@ -769,1434 +769,1434 @@ % We also need the transform from the microscope frame in order to % get an accurate defocus value. Not sure if I should be binning? % Additionally, we do NOT want the model for alignment in the - % microscope frame, + % microscope frame, iXFName = sprintf('%smapBack%d/%s_align.XF',mbOUT{1:3}); iXF = fopen(iXFName,'w'); -% % 20190509 - I think this is royally screwing things up FIXME -% % Commenting this out invalidates the defocus vals -% xfTLT = sortrows(TLT(:,[1,7:10,2,3],1)); -% fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT(:,2:7)'); -% fclose(iXF); + % % 20190509 - I think this is royally screwing things up FIXME + % % Commenting this out invalidates the defocus vals + % xfTLT = sortrows(TLT(:,[1,7:10,2,3],1)); + % fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT(:,2:7)'); + % fclose(iXF); + + xfTLT = zeros(size(TLT,1),6); + xfTLT(:,[1,4]) = 1.0; + fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT'); + fclose(iXF); - xfTLT = zeros(size(TLT,1),6); - xfTLT(:,[1,4]) = 1.0; - fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT'); - fclose(iXF); - positionList = geometry.(tomoList{iTomo}); tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); -% [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); + % [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); binShift = [0,0,0]; positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); - + % Need to store tilt name/path explicity in meta deta tiltName = tiltList{iTomo}; - + tiltHeader = getHeader(MRCImage(tiltName,0)); - + fullTiltSizeXandY = [tiltHeader.nX,tiltHeader.nY].*samplingRate; - - + + sTX = floor(tiltHeader.nX ); sTY = floor(tiltHeader.nY ); iTLT = floor(tiltHeader.nZ); - % FIXME the z-dimension should be 1 right? + % FIXME the z-dimension should be 1 right? originPrj = ceil(([sTX,sTY,0]+1)./2); - + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - - -% iGPU=1; - + + + % iGPU=1; + if (buildTomo) -% [tomo,tomoReconCoords] = BH_multi_loadOrBuild(tomoList{iTomo}, ... -% reconCoords, mapBackIter, ... -% samplingRate, iGPU,reconScaling,1); - - doRecon = 0; - doLoad = false; - reconCoords - [~,tomoReconCoords] = BH_multi_loadOrBuild(tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate, doRecon,reconScaling,... - doLoad, 'tomoCPR'); - - - + % [tomo,tomoReconCoords] = BH_multi_loadOrBuild(tomoList{iTomo}, ... + % reconCoords, mapBackIter, ... + % samplingRate, iGPU,reconScaling,1); + + doRecon = 0; + doLoad = false; + reconCoords + [~,tomoReconCoords] = BH_multi_loadOrBuild(tomoList{iTomo}, ... + reconCoords, mapBackIter, ... + samplingRate, doRecon,reconScaling,... + doLoad, 'tomoCPR'); + + + originVol = ceil((tomoReconCoords(1,1:3)+1)./2); - + reconShift = tomoReconCoords(2,1:3); - + end - - - + + + nPrjs = size(TLT,1) nSubTomos = size(positionList,1); - + % TODO need to update this. if (flgColorMap) colorMap = single(getVolume(MRCImage(COLOR_MAP))); % should be the same size as the average - + if any(size(refVol{1})-size(colorMap)) error('Color map and average vol must be the same size.\n'); end - colorMap = colorMap(avgOrigin(1)-maxRad:avgOrigin(1)+maxRad,... - avgOrigin(2)-maxRad:avgOrigin(2)+maxRad,... - avgOrigin(3)-maxRad:avgOrigin(3)+maxRad); - end - - - + colorMap = colorMap(avgOrigin(1)-maxRad:avgOrigin(1)+maxRad,... + avgOrigin(2)-maxRad:avgOrigin(2)+maxRad,... + avgOrigin(3)-maxRad:avgOrigin(3)+maxRad); + end + + + % Switch from maskRadius to particleRadius 20180129 sizeAvgVol = size(refVol{1}{1}); - - - - + + + + for iRef = 1:nRefs - - % FIXME change to EMC_maskreference - refVol{1}{iRef} = gpuArray(refVol{1}{iRef}); - refVol{2}{iRef} = gpuArray(refVol{2}{iRef}); - particleMask{iRef} = BH_mask3d('sphere',sizeAvgVol,PARTICLE_RADIUS.*[1,1,1],[0,0,0]).* ... - BH_mask3d(refVol{1}{iRef} + refVol{2}{iRef} ,pixelSize,'',''); - -% binaryMask = particleMask{iRef} > 0.01; -% for rV = 1:2 -% refVol{rV}{iRef} = refVol{rV}{iRef} - mean(refVol{rV}{iRef}(binaryMask)); -% refVol{rV}{iRef} = refVol{rV}{iRef} ./ (0.5.*rms(refVol{rV}{iRef}(binaryMask))); -% refVol{rV}{iRef} = refVol{rV}{iRef} .* particleMask{iRef}; -% end + + % FIXME change to EMC_maskreference + refVol{1}{iRef} = gpuArray(refVol{1}{iRef}); + refVol{2}{iRef} = gpuArray(refVol{2}{iRef}); + particleMask{iRef} = BH_mask3d('sphere',sizeAvgVol,PARTICLE_RADIUS.*[1,1,1],[0,0,0]).* ... + BH_mask3d(refVol{1}{iRef} + refVol{2}{iRef} ,pixelSize,'',''); + + % binaryMask = particleMask{iRef} > 0.01; + % for rV = 1:2 + % refVol{rV}{iRef} = refVol{rV}{iRef} - mean(refVol{rV}{iRef}(binaryMask)); + % refVol{rV}{iRef} = refVol{rV}{iRef} ./ (0.5.*rms(refVol{rV}{iRef}(binaryMask))); + % refVol{rV}{iRef} = refVol{rV}{iRef} .* particleMask{iRef}; + % end end - - - - + + + + if (iTomo == 1) fidIDX = 0; end - + %%%%%%%%%% if (buildTomo) - + modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector'); - - for iSubTomo = 1:nSubTomos - - -% -% prjVector = (positionList(iSubTomo,11:13)./samplingRate + binShift) - ... -% originVol + reconShift; -% - rSubTomo = reshape(positionList(iSubTomo,17:25),3,3); - prjVector = (positionList(iSubTomo,11:13)./samplingRate) - originVol + reconShift; - - iRefIDX = 1; - iClassIDX = 1; - if (nRefs > 1) - % Assuming generally there are fewer classes seleceted as references than there are total classes - % For those that aren't on of the select ones, we could try to track the best matched reference from the most recent - % alignment - iClassIDX = positionList(iSubTomo,26); - if ~(ismember(iClassIDX,classVector{1}) || ismember(iClassIDX,classVector{2})) - iClassIDX = datasample(classVector{1},1); - end - iRefIDX = find(classVector{1} == iClassIDX); - end - - -% % % prjVector = prjVector + [0.5,0.0,-0.5]; -% prjVector = prjVector + [0.0,0.0,1.0]; - prjVector = prjVector - preShift; - recVector = (originPrj + [0,0,ceil((reconstructionSize(3)+1)/2)] + prjVector); % subTomo origin relative to reconLowerLeft - - %Resample a copy of the average to match the position in the tomogram - % The third entry is a dummy, normally used to make sure at least the - % particle was being extracted even if the surrounding density (where - % some delocalized values may be located) are not. - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(reconstructionSize, sizeAvgVol, sizeAvgVol./5, recVector); - - - - if ~ischar(indVAL) - - - if positionList(iSubTomo,7) == 1 - iAvgResamp = BH_resample3d(refVol{1}{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); - elseif positionList(iSubTomo,7) ==2 - iAvgResamp = BH_resample3d(refVol{2}{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); - else - error('positionList iSubtomo %d col 7 is %d',iSubTomo,positionList(iSubTomo,7)); - end - iMaskResamp = BH_resample3d(particleMask{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); - - - - - iAvgResamp = gather(iMaskResamp.*iAvgResamp); - - if (flgColorMap || flgClassAvg) - if (flgColorMap) - iColorMap = gather(int16(iMaskResamp.* BH_resample3d(colorMap, ... - rSubTomo',shiftVAL,'Bah',METHOD,'forward'))); + for iSubTomo = 1:nSubTomos + + + % + % prjVector = (positionList(iSubTomo,11:13)./samplingRate + binShift) - ... + % originVol + reconShift; + % + rSubTomo = reshape(positionList(iSubTomo,17:25),3,3); + prjVector = (positionList(iSubTomo,11:13)./samplingRate) - originVol + reconShift; + + iRefIDX = 1; + iClassIDX = 1; + if (nRefs > 1) + % Assuming generally there are fewer classes seleceted as references than there are total classes + % For those that aren't on of the select ones, we could try to track the best matched reference from the most recent + % alignment + iClassIDX = positionList(iSubTomo,26); + if ~(ismember(iClassIDX,classVector{1}) || ismember(iClassIDX,classVector{2})) + iClassIDX = datasample(classVector{1},1); + end + iRefIDX = find(classVector{1} == iClassIDX); + end + + + + % % % prjVector = prjVector + [0.5,0.0,-0.5]; + % prjVector = prjVector + [0.0,0.0,1.0]; + prjVector = prjVector - preShift; + recVector = (originPrj + [0,0,ceil((reconstructionSize(3)+1)/2)] + prjVector); % subTomo origin relative to reconLowerLeft + + %Resample a copy of the average to match the position in the tomogram + % The third entry is a dummy, normally used to make sure at least the + % particle was being extracted even if the surrounding density (where + % some delocalized values may be located) are not. + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid(reconstructionSize, sizeAvgVol, sizeAvgVol./5, recVector); + + + + if ~ischar(indVAL) + + + if positionList(iSubTomo,7) == 1 + iAvgResamp = BH_resample3d(refVol{1}{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); + elseif positionList(iSubTomo,7) ==2 + iAvgResamp = BH_resample3d(refVol{2}{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); else - % Set value to class average number - iColorMap = iMaskResamp; - iColorMap(iColorMap < 0.05) = 0; - iColorMap(iColorMap >= 0.05) = iRefIDX; - iColorMap = gather(int16(iColorMap)); + error('positionList iSubtomo %d col 7 is %d',iSubTomo,positionList(iSubTomo,7)); end - - - if ( flgClassAvg ) - + iMaskResamp = BH_resample3d(particleMask{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); + + + + + iAvgResamp = gather(iMaskResamp.*iAvgResamp); + + if (flgColorMap || flgClassAvg) + if (flgColorMap) + iColorMap = gather(int16(iMaskResamp.* BH_resample3d(colorMap, ... + rSubTomo',shiftVAL,'Bah',METHOD,'forward'))); + else + % Set value to class average number + iColorMap = iMaskResamp; + iColorMap(iColorMap < 0.05) = 0; + iColorMap(iColorMap >= 0.05) = iRefIDX; + iColorMap = gather(int16(iColorMap)); + end + + + if ( flgClassAvg ) + + end + + avgColor(indVAL(1,1):indVAL(2,1), ... + indVAL(1,2):indVAL(2,2), ... + indVAL(1,3):indVAL(2,3)) = avgColor(indVAL(1,1):indVAL(2,1), ... + indVAL(1,2):indVAL(2,2), ... + indVAL(1,3):indVAL(2,3)) + ... + iColorMap(1+padVAL(1,1):end-padVAL(2,1),... + 1+padVAL(1,2):end-padVAL(2,2),... + 1+padVAL(1,3):end-padVAL(2,3)); + + end - avgColor(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3)) = avgColor(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3)) + ... - iColorMap(1+padVAL(1,1):end-padVAL(2,1),... - 1+padVAL(1,2):end-padVAL(2,2),... - 1+padVAL(1,3):end-padVAL(2,3)); - - end - - - try - avgTomo{1}(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3)) = ... - avgTomo{1}(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3)) .* ... - gather((1 - iMaskResamp(1+padVAL(1,1):end-padVAL(2,1),... % zeros out region being replaced - 1+padVAL(1,2):end-padVAL(2,2),... - 1+padVAL(1,3):end-padVAL(2,3)))) + ... - iAvgResamp(1+padVAL(1,1):end-padVAL(2,1),... - 1+padVAL(1,2):end-padVAL(2,2),... - 1+padVAL(1,3):end-padVAL(2,3)); - catch - fprintf('Warning, subTomo %d appears to be out of bounds in mapBack?\n'); - continue - end + try + avgTomo{1}(indVAL(1,1):indVAL(2,1), ... + indVAL(1,2):indVAL(2,2), ... + indVAL(1,3):indVAL(2,3)) = ... + avgTomo{1}(indVAL(1,1):indVAL(2,1), ... + indVAL(1,2):indVAL(2,2), ... + indVAL(1,3):indVAL(2,3)) .* ... + gather((1 - iMaskResamp(1+padVAL(1,1):end-padVAL(2,1),... % zeros out region being replaced + 1+padVAL(1,2):end-padVAL(2,2),... + 1+padVAL(1,3):end-padVAL(2,3)))) + ... + iAvgResamp(1+padVAL(1,1):end-padVAL(2,1),... + 1+padVAL(1,2):end-padVAL(2,2),... + 1+padVAL(1,3):end-padVAL(2,3)); + catch + fprintf('Warning, subTomo %d appears to be out of bounds in mapBack?\n'); + continue + end - - - - % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot*prjVector' + [originRec(1),originRec(3),originRec(2)]'- prjVectorShift([1,3,2]), fidIDX); - - for iPrj = 1:nPrjs - iPrj_nat = find(TLT(:,1) == iPrj); - % imod is indexing from zero - % imod is indexing from zero - zCoord = iPrj_nat; - - rTilt = BH_defineMatrix([90,1.*TLT(iPrj_nat,4),-90],'Bah','forwardVector'); - prjCoords = rTilt*prjVector'; - - fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)); -% d1 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; -% d2 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; - d1 = -1.*(samplingRate.*prjVector(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15))*10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm - d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value - - fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',fidIDX, tomoNumber,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(rSubTomo,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); - - % These shifts are a record of transformation from the raw data, but here - % we are comparing with [CTF] corrected data, from which the - % reconstructino was made directly - - end % loop over tilt projections - - - fidIDX = fidIDX + 1; - else - fprintf('ignoring subTomo %d for out of bounds conditions.\n', iSubTomo); - reconstructionSize - sizeAvgVol - sizeAvgVol./5 - recVector - end - end % loop over subtomos + % Reproject using tilt, so just save the 3d coords. + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot*prjVector' + [originRec(1),originRec(3),originRec(2)]'- prjVectorShift([1,3,2]), fidIDX); + + for iPrj = 1:nPrjs + + iPrj_nat = find(TLT(:,1) == iPrj); + % imod is indexing from zero + % imod is indexing from zero + zCoord = iPrj_nat; + + rTilt = BH_defineMatrix([90,1.*TLT(iPrj_nat,4),-90],'Bah','forwardVector'); + + prjCoords = rTilt*prjVector'; + + fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)); + % d1 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; + % d2 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; + + d1 = -1.*(samplingRate.*prjVector(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15))*10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm + d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value + + fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',fidIDX, tomoNumber,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(rSubTomo,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); + + % These shifts are a record of transformation from the raw data, but here + % we are comparing with [CTF] corrected data, from which the + % reconstructino was made directly + + end % loop over tilt projections + + + fidIDX = fidIDX + 1; + else + fprintf('ignoring subTomo %d for out of bounds conditions.\n', iSubTomo); + reconstructionSize + sizeAvgVol + sizeAvgVol./5 + recVector + end + end % loop over subtomos end %%%% temp condition to skip building full tomo - + end - - + + if (buildTomo) - fclose(coordOUT); - fclose(coordSTART); - - + fclose(coordOUT); + fclose(coordSTART); + + p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 ',... - '%smapBack%d/%s.coord %smapBack%d/%s.3dfid'], ... - mbOUT{1:3},mbOUT{1:3}) + '%smapBack%d/%s.coord %smapBack%d/%s.3dfid'], ... + mbOUT{1:3},mbOUT{1:3}) system(p2m); - + for iSave = 1 SAVE_IMG(MRCImage(gather(avgTomo{iSave})),sprintf('%smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave),pixelSize); - avgTomo{iSave} = []; + avgTomo{iSave} = []; end clear avgTomo if (flgColorMap || flgClassAvg) SAVE_IMG(MRCImage(gather(avgColor)),sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),4.0); - clear avgColor + clear avgColor end % If not planning on visualization, save only a binned copy of the synthetic % tomo. -% tmpTomoBin = floor(1/samplingRate*6); -% TODO make this an adjustable parameter -% tmpTomoBin = ceil(6/pixelSize); -% -% for iSave = 1:1+(3*testSubtraction) -% system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomo%d ',... -% '%smapBack%d/%s.bin%dTomo%d.mrc'], ... -% tmpTomoBin,mbOUT{1:3},iSave,mbOUT{1:3},tmpTomoBin,iSave)); -% end - tmpTomoBin = 2; - if (flgColorMap || flgClassAvg) - system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomoColor ',... - '%smapBack%d/%s.bin%dTomoColor.mrc'], ... - tmpTomoBin,mbOUT{1:3},mbOUT{1:3},tmpTomoBin)); - system(sprintf('rm %smapBack%d/%s.tmpTomoColor ', mbOUT{1:3})); - - end + % tmpTomoBin = floor(1/samplingRate*6); + % TODO make this an adjustable parameter + % tmpTomoBin = ceil(6/pixelSize); + % + % for iSave = 1:1+(3*testSubtraction) + % system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomo%d ',... + % '%smapBack%d/%s.bin%dTomo%d.mrc'], ... + % tmpTomoBin,mbOUT{1:3},iSave,mbOUT{1:3},tmpTomoBin,iSave)); + % end + tmpTomoBin = 2; + if (flgColorMap || flgClassAvg) + system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomoColor ',... + '%smapBack%d/%s.bin%dTomoColor.mrc'], ... + tmpTomoBin,mbOUT{1:3},mbOUT{1:3},tmpTomoBin)); + system(sprintf('rm %smapBack%d/%s.tmpTomoColor ', mbOUT{1:3})); - - - - rotSize = [tiltHeader.nX,maxZ,tiltHeader.nY] - - for iSave = 1 - rotCMD = sprintf(['rotatevol -angles 0,0,90 -size %d,%d,%d ',... - '%smapBack%d/%s.tmpTomo%d %smapBack%d/%s.tmpRot%d'], ... - rotSize, mbOUT{1:3},iSave,mbOUT{1:3},iSave); - - system(rotCMD); - - system(sprintf('rm %smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave)); - - - end - - - - - % % % % if (flgColorMap) - % % % % SAVE_IMG(MRCImage(avgColor),sprintf('mapBack/%s_colorMap.mrc',tiltBaseName)); - % % % % % -90 is assumed for trim vol, so if rotate vol is used add 90 - % % % % if (rotateVol) - % % % % system(sprintf('rotatevol -angles 0,0,%d mapBack/%s_colorMap.mrc mapBack/%s_colorMap.rot',reconRotation(iTomo,3)+90.0,tiltBaseName,tiltBaseName)); - % % % % system(sprintf('mv mapBack/%s_colorMap.rot mapBack/%s_colorMap.mrc',tiltBaseName,tiltBaseName)); - % % % % system(sprintf('rm mapBack/%s_colorMap.rot',tiltBaseName)); - % % % % end - % % % % end - - + end + + + + + rotSize = [tiltHeader.nX,maxZ,tiltHeader.nY] + + for iSave = 1 + rotCMD = sprintf(['rotatevol -angles 0,0,90 -size %d,%d,%d ',... + '%smapBack%d/%s.tmpTomo%d %smapBack%d/%s.tmpRot%d'], ... + rotSize, mbOUT{1:3},iSave,mbOUT{1:3},iSave); + + system(rotCMD); + + system(sprintf('rm %smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave)); + + + end + + + + + % % % % if (flgColorMap) + % % % % SAVE_IMG(MRCImage(avgColor),sprintf('mapBack/%s_colorMap.mrc',tiltBaseName)); + % % % % % -90 is assumed for trim vol, so if rotate vol is used add 90 + % % % % if (rotateVol) + % % % % system(sprintf('rotatevol -angles 0,0,%d mapBack/%s_colorMap.mrc mapBack/%s_colorMap.rot',reconRotation(iTomo,3)+90.0,tiltBaseName,tiltBaseName)); + % % % % system(sprintf('mv mapBack/%s_colorMap.rot mapBack/%s_colorMap.mrc',tiltBaseName,tiltBaseName)); + % % % % system(sprintf('rm mapBack/%s_colorMap.rot',tiltBaseName)); + % % % % end + % % % % end + + clear avgTomo{1} wgt end % % % % It may be faster to work with a rotated vol since the reading in may cause % % % % problems, but the projection is so slow, that this isn't worth dealing with % % % % now. - if (nWorkers > 1) - chunkSize = ceil(sTY./nWorkers); - chunkInc = zeros(nWorkers,3); - for iWorker = 1:nWorkers-1 - chunkInc(iWorker,:) = [iWorker,(iWorker-1)*chunkSize+1,iWorker*chunkSize]; - end - iWorker = iWorker +1 ; - chunkInc(end, :) = [iWorker, (iWorker-1)*chunkSize+1, sTY]; - else - chunkSize = sTY; - chunkInc = [1,1,sTY]; + if (nWorkers > 1) + chunkSize = ceil(sTY./nWorkers); + chunkInc = zeros(nWorkers,3); + for iWorker = 1:nWorkers-1 + chunkInc(iWorker,:) = [iWorker,(iWorker-1)*chunkSize+1,iWorker*chunkSize]; end - - + iWorker = iWorker +1 ; + chunkInc(end, :) = [iWorker, (iWorker-1)*chunkSize+1, sTY]; + else + chunkSize = sTY; + chunkInc = [1,1,sTY]; + end + + % % % % % % % if (buildTomo) - - - taStr = [sprintf('%f',rawTLT(1,2))]; - for iTa = 2:length(rawTLT(:,2)) - taStr = [taStr sprintf(',%f',rawTLT(iTa,2))]; - end - - - - if (localFile) - lastLine1 = sprintf('LOCALFILE %s', localFile) - % Used if GPU fails - cpuLastLine = lastLine1; - else - lastLine1 = ''; - cpuLastLine = ''; - end - - if strcmpi(METHOD, 'GPU') - if (lastLine1) - lastLine2 = 'UseGPU 0'; - lastLine3 = 'ActionIfGPUFails 2,2'; - else - lastLine1 = 'UseGPU 0'; - lastLine2 = 'ActionIfGPUFails 2,2'; - lastLine3 = ''; - end + + + taStr = [sprintf('%f',rawTLT(1,2))]; + for iTa = 2:length(rawTLT(:,2)) + taStr = [taStr sprintf(',%f',rawTLT(iTa,2))]; + end + + + + if (localFile) + lastLine1 = sprintf('LOCALFILE %s', localFile) + % Used if GPU fails + cpuLastLine = lastLine1; + else + lastLine1 = ''; + cpuLastLine = ''; + end + + if strcmpi(METHOD, 'GPU') + if (lastLine1) + lastLine2 = 'UseGPU 0'; + lastLine3 = 'ActionIfGPUFails 2,2'; else - lastLine2 = ''; + lastLine1 = 'UseGPU 0'; + lastLine2 = 'ActionIfGPUFails 2,2'; lastLine3 = ''; end - + else + lastLine2 = ''; + lastLine3 = ''; + end + + + % Break this up into chunks since things hang even with the + % ActionIfGPUFails option. Try 3 times 512,256,128 + % refPrj = zeros(sTX,sTY,iTLT, 'single'); + + for iSave = 1 + keepItRunning = 1; + outputStackName = sprintf('%smapBack%d/%s_%d_mapBack.st',mbOUT{1:3},iSave); - % Break this up into chunks since things hang even with the - % ActionIfGPUFails option. Try 3 times 512,256,128 -% refPrj = zeros(sTX,sTY,iTLT, 'single'); + while (keepItRunning) - for iSave = 1 - keepItRunning = 1; - outputStackName = sprintf('%smapBack%d/%s_%d_mapBack.st',mbOUT{1:3},iSave); - while (keepItRunning) - - - inc = 0:mapBackRePrjSize:sTY-1; - if inc(end) < sTY-1 - inc = [inc,sTY-1]; - end - inc - nChunks = length(inc)-1; - - for iChunk = 1:nChunks + inc = 0:mapBackRePrjSize:sTY-1; + if inc(end) < sTY-1 + inc = [inc,sTY-1]; + end + inc + nChunks = length(inc)-1; + + for iChunk = 1:nChunks + + if iChunk == 1 - if iChunk == 1 - - if exist(outputStackName,'file') - fprintf('removing %s\n',outputStackName); - system(sprintf('rm %s',outputStackName)); - end - % Special case, initialize the full sized volume and the - % header but don't actually reproject anything. - fprintf('Initializing volume %d/%d with size %d\n',... - iChunk,nChunks,mapBackRePrjSize); - rePrjFileName = sprintf('%smapBack%d/%s_%d_rePrj.sh',mbOUT{1:3},iSave); - reModFileName = sprintf('%smapBack%d/%s_%d_reMod.sh',mbOUT{1:3},iSave); - reProjFile = fopen(rePrjFileName,'w'); - reModFile = fopen(reModFileName,'w'); - fprintf(reProjFile,['#!/bin/bash\n\n',... - 'tilt -StandardInput << EOF\n',... - 'input %s\n', ... - 'output %s\n', ... - 'COSINTERP 0\n', ... - 'THICKNESS %d\n', ... - 'TILTFILE %smapBack%d/%s_align.rawtlt\n', ... - 'REPROJECT %s\n', ... - 'RecFileToReproject %smapBack%d/%s.tmpRot%d\n',... - 'TOTALSLICES %d,%d\n',... - 'ZMinAndMaxReproj %d,%d\n',... - '%s\n', ... - '%s\n', ... - '%s\n',... - 'EOF'],tiltList{1} ,outputStackName, maxZ, ... - mbOUT{1:3},... - taStr, mbOUT{1:3},iSave,... - 0,sTY-1,... - -1,-1,... - lastLine1,lastLine2,... - lastLine3); - - fclose(reProjFile); - system(sprintf('chmod a=wrx %s',rePrjFileName)); - [failedToRun,~] = system(sprintf('%s',rePrjFileName)); - - if (failedToRun) - error('failed to initialize reporojection %s\n',outputStackName); - end + if exist(outputStackName,'file') + fprintf('removing %s\n',outputStackName); + system(sprintf('rm %s',outputStackName)); end - - % Special case, initialize the full sized volume and the - % header but don't actually reproject anything. - fprintf('Reprojecting volume %d/%d with size %d\n',... - iChunk,nChunks,mapBackRePrjSize); - rePrjFileName = sprintf('%smapBack%d/%s_%d_rePrj.sh',mbOUT{1:3},iSave); - reModFileName = sprintf('%smapBack%d/%s_%d_reMod.sh',mbOUT{1:3},iSave); - reProjFile = fopen(rePrjFileName,'w'); - reModFile = fopen(reModFileName,'w'); - fprintf(reProjFile,['#!/bin/bash\n\n',... - 'tilt -StandardInput << EOF\n',... - 'input %s\n', ... - 'output %s\n', ... - 'COSINTERP 0\n', ... - 'THICKNESS %d\n', ... - 'TILTFILE %smapBack%d/%s_align.rawtlt\n', ... - 'REPROJECT %s\n', ... - 'RecFileToReproject %smapBack%d/%s.tmpRot%d\n',... - 'TOTALSLICES %d,%d\n',... - 'ZMinAndMaxReproj %d,%d\n',... - '%s\n', ... - '%s\n', ... - '%s\n',... - 'EOF'],tiltList{1} ,outputStackName, maxZ, ... - mbOUT{1:3},... - taStr, mbOUT{1:3},iSave,... - 0,sTY-1,... - inc(iChunk),inc(iChunk+1),... - lastLine1,lastLine2,... - lastLine3); - - fclose(reProjFile); - system(sprintf('chmod a=wrx %s',rePrjFileName)); - + % Special case, initialize the full sized volume and the + % header but don't actually reproject anything. + fprintf('Initializing volume %d/%d with size %d\n',... + iChunk,nChunks,mapBackRePrjSize); + rePrjFileName = sprintf('%smapBack%d/%s_%d_rePrj.sh',mbOUT{1:3},iSave); + reModFileName = sprintf('%smapBack%d/%s_%d_reMod.sh',mbOUT{1:3},iSave); + reProjFile = fopen(rePrjFileName,'w'); + reModFile = fopen(reModFileName,'w'); + fprintf(reProjFile,['#!/bin/bash\n\n',... + 'tilt -StandardInput << EOF\n',... + 'input %s\n', ... + 'output %s\n', ... + 'COSINTERP 0\n', ... + 'THICKNESS %d\n', ... + 'TILTFILE %smapBack%d/%s_align.rawtlt\n', ... + 'REPROJECT %s\n', ... + 'RecFileToReproject %smapBack%d/%s.tmpRot%d\n',... + 'TOTALSLICES %d,%d\n',... + 'ZMinAndMaxReproj %d,%d\n',... + '%s\n', ... + '%s\n', ... + '%s\n',... + 'EOF'],tiltList{1} ,outputStackName, maxZ, ... + mbOUT{1:3},... + taStr, mbOUT{1:3},iSave,... + 0,sTY-1,... + -1,-1,... + lastLine1,lastLine2,... + lastLine3); - - + fclose(reProjFile); + system(sprintf('chmod a=wrx %s',rePrjFileName)); [failedToRun,~] = system(sprintf('%s',rePrjFileName)); - + if (failedToRun) - % Reduce size - switch mapBackRePrjSize - case 512 - mapBackRePrjSize = 384; - case 384 - mapBackRePrjSize = 256; - case 256 - mapBackRePrjSize = 192; - case 192 - mapBackRePrjSize = 128; - case 128 - mapBackRePrjSize = 96; - case 96 + error('failed to initialize reporojection %s\n',outputStackName); + end + end + + % Special case, initialize the full sized volume and the + % header but don't actually reproject anything. + fprintf('Reprojecting volume %d/%d with size %d\n',... + iChunk,nChunks,mapBackRePrjSize); + rePrjFileName = sprintf('%smapBack%d/%s_%d_rePrj.sh',mbOUT{1:3},iSave); + reModFileName = sprintf('%smapBack%d/%s_%d_reMod.sh',mbOUT{1:3},iSave); + reProjFile = fopen(rePrjFileName,'w'); + reModFile = fopen(reModFileName,'w'); + fprintf(reProjFile,['#!/bin/bash\n\n',... + 'tilt -StandardInput << EOF\n',... + 'input %s\n', ... + 'output %s\n', ... + 'COSINTERP 0\n', ... + 'THICKNESS %d\n', ... + 'TILTFILE %smapBack%d/%s_align.rawtlt\n', ... + 'REPROJECT %s\n', ... + 'RecFileToReproject %smapBack%d/%s.tmpRot%d\n',... + 'TOTALSLICES %d,%d\n',... + 'ZMinAndMaxReproj %d,%d\n',... + '%s\n', ... + '%s\n', ... + '%s\n',... + 'EOF'],tiltList{1} ,outputStackName, maxZ, ... + mbOUT{1:3},... + taStr, mbOUT{1:3},iSave,... + 0,sTY-1,... + inc(iChunk),inc(iChunk+1),... + lastLine1,lastLine2,... + lastLine3); + + fclose(reProjFile); + system(sprintf('chmod a=wrx %s',rePrjFileName)); + + + + + [failedToRun,~] = system(sprintf('%s',rePrjFileName)); + + if (failedToRun) + % Reduce size + switch mapBackRePrjSize + case 512 + mapBackRePrjSize = 384; + case 384 + mapBackRePrjSize = 256; + case 256 + mapBackRePrjSize = 192; + case 192 + mapBackRePrjSize = 128; + case 128 + mapBackRePrjSize = 96; + case 96 % header but don't actually reproject anyth - mapBackRePrjSize = 64; - case 64 - mapBackRePrjSize = 32; - case 32 - mapBackRePrjSize = 16; - case 16 - mapBackRePrjSize = 8; - case 8 - mapBackRePrjSize = 4; - case 4 - system(sprintf('%s > failReProj.log',rePrjFileName)) - error(['mapBackRePrjSize = 4 is still too much for the'],... - ['GPU which probably means your local alignments'],... - ['are too large']) - end - % Update the stored size - subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize') = mapBackRePrjSize; - - system(sprintf('mv %s %s.tmp',rePrjFileName,rePrjFileName)); - system(sprintf('cp %s.tmp %s',rePrjFileName,rePrjFileName)); - system(sprintf('rm %s.tmp %s',rePrjFileName,rePrjFileName)); -% system(sprintf('%s',rePrjFileName)); - - % Break out to next iter of while loop, recalculating the - % chunk size at reduced depth. - - break - - end - - - if iChunk == nChunks - keepItRunning = 0; - end - + mapBackRePrjSize = 64; + case 64 + mapBackRePrjSize = 32; + case 32 + mapBackRePrjSize = 16; + case 16 + mapBackRePrjSize = 8; + case 8 + mapBackRePrjSize = 4; + case 4 + system(sprintf('%s > failReProj.log',rePrjFileName)) + error(['mapBackRePrjSize = 4 is still too much for the'],... + ['GPU which probably means your local alignments'],... + ['are too large']) + end + % Update the stored size + subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize') = mapBackRePrjSize; - end % end of for loop over chunks - end % end of while loop - end % loop over error and masked tomo - - - - fprintf(reModFile,['#!/bin/bash\n\n',... - 'tilt -StandardInput << EOF\n',... - 'input %s\n', ... - 'output %smapBack%d/%s.fid\n', ... - 'COSINTERP 0\n', ... - 'THICKNESS %d\n', ... - 'TILTFILE %smapBack%d/%s_align.rawtlt \n', ... - 'DefocusFile %smapBack%d/%s_align.defocus \n', ... - 'PixelForDefocus %f,%f\n', ... - 'AngleOutputFile %smapBack%d/%s.defAngTilt\n', ... - 'AlignTransformFile %smapBack%d/%s_align.XF\n', ... - 'ProjectModel %smapBack%d/%s.3dfid\n', ... - '%s\n',... - '%s\n',... - '%s\n',... - 'EOF'],tiltList{1}, mbOUT{1:3}, maxZ, ... - mbOUT{1:3},... - mbOUT{1:3},... - pixelSize./10, flgInvertTiltAngles,... % Ang --> nm - mbOUT{1:3},... - mbOUT{1:3},... - mbOUT{1:3},... - lastLine1,lastLine2,... - lastLine3); - - fclose(reModFile); - system(sprintf('chmod a=wrx %s',reModFileName)); - - - [failedToRun,~] = system(sprintf('%s',reModFileName)); - % Sometimes the file is busy - if (failedToRun) - system(sprintf('mv %s %s.tmp',reModFileName,reModFileName)); - system(sprintf('cp %s.tmp %s',reModFileName,reModFileName)); - system(sprintf('rm %s.tmp',reModFileName)); - system(sprintf('%s',reModFileName)); - end - - % re-write the projected coords - system(sprintf(['model2point -float -contour -zero ',... - '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'],... - mbOUT{1:3}, mbOUT{1:3})) - - - - - - end - - - for iSave = 1 - % Remove the full size tomo - system(sprintf('rm %smapBack%d/%s.tmpRot%d',mbOUT{1:3},iSave)); + system(sprintf('mv %s %s.tmp',rePrjFileName,rePrjFileName)); + system(sprintf('cp %s.tmp %s',rePrjFileName,rePrjFileName)); + system(sprintf('rm %s.tmp %s',rePrjFileName,rePrjFileName)); + % system(sprintf('%s',rePrjFileName)); + + % Break out to next iter of while loop, recalculating the + % chunk size at reduced depth. + + break + + end + + + if iChunk == nChunks + keepItRunning = 0; + end + + + end % end of for loop over chunks + end % end of while loop + end % loop over error and masked tomo + + + + fprintf(reModFile,['#!/bin/bash\n\n',... + 'tilt -StandardInput << EOF\n',... + 'input %s\n', ... + 'output %smapBack%d/%s.fid\n', ... + 'COSINTERP 0\n', ... + 'THICKNESS %d\n', ... + 'TILTFILE %smapBack%d/%s_align.rawtlt \n', ... + 'DefocusFile %smapBack%d/%s_align.defocus \n', ... + 'PixelForDefocus %f,%f\n', ... + 'AngleOutputFile %smapBack%d/%s.defAngTilt\n', ... + 'AlignTransformFile %smapBack%d/%s_align.XF\n', ... + 'ProjectModel %smapBack%d/%s.3dfid\n', ... + '%s\n',... + '%s\n',... + '%s\n',... + 'EOF'],tiltList{1}, mbOUT{1:3}, maxZ, ... + mbOUT{1:3},... + mbOUT{1:3},... + pixelSize./10, flgInvertTiltAngles,... % Ang --> nm + mbOUT{1:3},... + mbOUT{1:3},... + mbOUT{1:3},... + lastLine1,lastLine2,... + lastLine3); + + fclose(reModFile); + system(sprintf('chmod a=wrx %s',reModFileName)); + + + [failedToRun,~] = system(sprintf('%s',reModFileName)); + % Sometimes the file is busy + if (failedToRun) + system(sprintf('mv %s %s.tmp',reModFileName,reModFileName)); + system(sprintf('cp %s.tmp %s',reModFileName,reModFileName)); + system(sprintf('rm %s.tmp',reModFileName)); + system(sprintf('%s',reModFileName)); end - + % re-write the projected coords + system(sprintf(['model2point -float -contour -zero ',... + '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'],... + mbOUT{1:3}, mbOUT{1:3})) + + + + + + end + + + for iSave = 1 + % Remove the full size tomo + system(sprintf('rm %smapBack%d/%s.tmpRot%d',mbOUT{1:3},iSave)); + end + + fidList = load(sprintf('%smapBack%d/%s.coordPrj',mbOUT{1:3})); parList = load(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3})); defList = load(sprintf('%smapBack%d/%s.defAngTilt',mbOUT{1:3})); - + % Need to shift again from the model coordinate system - fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(prjVectorShift(1:2)', size(fidList,1),1); + fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(prjVectorShift(1:2)', size(fidList,1),1); foundNans = sum(isnan(fidList(:,3))); if (foundNans) fprintf('\n\t\tThere are %d NaNs in the projected fiducial list %3.3f\n\n',foundNans, foundNans/size(fidList,1)*100); fprintf('The only confirmed case that produced this were NaNs in the fixedStacks/tiltN.local file.\n'); error("Exiting"); end - + % Give every instance of each fiducial a unique identifier. fidList = [1:size(fidList,1);fidList']'; - - - % for center of mass - COM = 3; - [bx,by] = ndgrid(-COM:COM,-COM:COM) - % add optional half radius for edge case and make the padding more - % logical, twice the particle radius, and then CTF size using mulit_iter - % with an optimization step - particlePad = 1.5; - tileRadius = floor(particlePad.*PARTICLE_RADIUS); - tileSize = (2.*tileRadius + 1).*[1,1]; + + + % for center of mass + COM = 3; + [bx,by] = ndgrid(-COM:COM,-COM:COM) + % add optional half radius for edge case and make the padding more + % logical, twice the particle radius, and then CTF size using mulit_iter + % with an optimization step + particlePad = 1.5; + tileRadius = floor(particlePad.*PARTICLE_RADIUS); + tileSize = (2.*tileRadius + 1).*[1,1]; + + CTFSIZE = BH_multi_iterator([2.*tileSize,1], 'fourier'); + + CTFSIZE = CTFSIZE(1:2); + ctfOrigin = floor(CTFSIZE./2) + 1; + + padCTF = BH_multi_padVal(tileSize,CTFSIZE); + ctfMask = BH_mask3d('sphere',CTFSIZE,ctfOrigin-7,[0,0],'2d'); + + if (eraseMask) + peakMask = BH_mask3d(eraseMaskType,CTFSIZE,eraseMaskRadius,[0,0],'2d'); + else + peakMask = BH_mask3d('sphere',CTFSIZE,peakSearchRad,[0,0],'2d'); + end + + peakMask(peakMask < 0.99) = 0; + + + + bandPassPrj = BH_bandpass3d([sTX,sTY,1],0,0,lowPassCutoff,'cpu',pixelSize); + + diagnosticCell = cell(nPrjs,1); + evalMaskCell = cell(nPrjs,1); + + for iPrj = 1:nPrjs + evalMaskCell{iPrj} = zeros(gather([sTX,sTY,1]),'uint8'); + end + + % Any large shifts should be obvious in the original alignment, so only + % look around +/- this value + globalPeak = max(2,ceil(10/pixelSize)); + globalPeak = globalPeak + mod(globalPeak,2); + + globalPeakMask = zeros([sTX,sTY,1],'single'); + + globalPeakMask(originPrj(1) -globalPeak : originPrj(1) + globalPeak,... + originPrj(2) -globalPeak : originPrj(2) + globalPeak) = 1; + + globalBinary = ( globalPeakMask > 0 ); + % Zero and only changed if CTF is refined. + defocusShifts = cell(nPrjs,1); + + if (calcCTF) + nToCheck = floor(ctfRange./ctfInc); + defShiftVect = ctfInc.*[-nToCheck:nToCheck]'; + else + nToCheck = 1; + defShiftVect = 0; + end + nDefTotal = length(defShiftVect); + + defocusCCC = cell(nPrjs,1); + expectedDefocusPerFiducial=cell(nPrjs,1); + + + if samplingRate > 1 + tiltSeries = sprintf('%scache/%s_ali%d_bin%d.fixed',CWD,tiltName,mapBackIter+1,samplingRate); + else + tiltSeries = sprintf('%saliStacks/%s_ali%d.fixed',CWD,tiltName,mapBackIter+1); + end + + + + % Optionally restrict the search to a given number of fiducials: + nUniqueFids = numel(unique(fidList(:,2))); % I think the max val of this column should also be okay (+1) + nFidsTotal = nUniqueFids; + % nFidsTotal = sum(fidList(:,5) == 1 ); + if tomoCPR_random_subset == -1 || tomoCPR_random_subset > nUniqueFids + fprintf('Using all of the %d available fiducials\n',nUniqueFids); + else + fprintf('Using a random subset of %d fiducials from the %d available\n',... + tomoCPR_random_subset, nUniqueFids); - CTFSIZE = BH_multi_iterator([2.*tileSize,1], 'fourier'); - - CTFSIZE = CTFSIZE(1:2); - ctfOrigin = floor(CTFSIZE./2) + 1; - - padCTF = BH_multi_padVal(tileSize,CTFSIZE); - ctfMask = BH_mask3d('sphere',CTFSIZE,ctfOrigin-7,[0,0],'2d'); + keepFids = datasample(0:nUniqueFids-1,tomoCPR_random_subset,'Replace',false); + fidList(~ismember(fidList(:,2),keepFids),2) = -9999; + nFidsTotal = tomoCPR_random_subset; + end + + + for iPrj = 1:nPrjs + % I must specify the number of fiducials somehwere else, replace the + % unique when there is time. + defocusShifts{iPrj} = 0; + defocusCCC{iPrj} = zeros(nDefTotal, nFidsTotal,'single','gpuArray'); + expectedDefocusPerFiducial{iPrj} = zeros(nDefTotal,nFidsTotal,'single'); + end + + %Put back into a natural order + TLT = sortrows(TLT,1); + + if isnan(k_factor_scaling) + k_factor_scaling = 10 / sqrt(nFidsTotal); + end + + parfor iPrj = 1:nPrjs + + % % For some reason if these mrc objects are created before the parfor + % loop begins, they fail to load. It is fine as a regular for loop + % though - annoying, but very little overhead. It would be nice + % to know what is going on here. + bhF = fourierTransformer(randn(CTFSIZE,'single','gpuArray'),'OddSizeOversampled'); + + iMrcObj = MRCImage(tiltSeries,0); + iMrcObjRef = MRCImage(sprintf('%smapBack%d/%s_1_mapBack.st',mbOUT{1:3}),0); + iMrcObjSamplingMask = MRCImage(sprintf('%saliStacks/%s_ali%d.fixed.samplingMask',CWD,tiltName,mapBackIter+1),0); + + + tic + while toc < 300 + try + dataPrj = single(getVolume(iMrcObj,[1,sTX],[1,sTY],iPrj,'keep')); + break + catch + pause(1e-1) + if ~(mod(toc,10)) + fprintf('Waiting for %d sec for dataPrj to be available\n',toc); + end + end + end + if toc == 300 + error('failed to load dataPrj %s at %d after %d tries\n.', ... + tiltSeries,iPrj,3000); + else + % fprintf('loaded dataPrj %d on try %d\n',iPrj,floor(toc./0.1)); + end + tic + while toc < 300 + try + refPrj = single(getVolume(iMrcObjRef,[1,sTX],[1,sTY],iPrj,'keep')); + break + catch + pause(1e-1) + if ~(mod(toc,10)) + fprintf('Waiting for %d sec for refPrj to be available\n',toc); + end + end + end + if toc == 300 + error('failed to load refPrj %s at %d after %d tries\n.', ... + tiltSeries,iPrj,3000); + else + % fprintf('loaded refPrj %d on try %d\n',iPrj,floor(toc./0.1)); + end + tic + while toc < 300 + try + samplingMask = gpuArray(single(getVolume(iMrcObjSamplingMask,[],[],iPrj,'keep'))); + break + catch + pause(1e-1) + if ~(mod(toc,10)) + fprintf('Waiting for %d sec for samplingMask to be available\n',toc); + end + end + end + if toc == 300 + error('failed to load samplingMask %s at %d after %d tries\n.', ... + tiltSeries,iPrj,3000); + else + % fprintf('loaded refPrj %d on try %d\n',iPrj,floor(toc./0.1)); + end - if (eraseMask) - peakMask = BH_mask3d(eraseMaskType,CTFSIZE,eraseMaskRadius,[0,0],'2d'); + if (samplingRate > 1) + samplingMask = BH_resample2d(samplingMask,[0,0,0],[0,0],'Bah','GPU','forward',1/samplingRate,[sTX,sTY]); else - peakMask = BH_mask3d('sphere',CTFSIZE,peakSearchRad,[0,0],'2d'); + end - peakMask(peakMask < 0.99) = 0; + % stored as uint8, it comes out 128,129 instead of 0,1.FIXME + samplingMask = ( samplingMask == max(samplingMask(:)) ); + % In case there is any carbon or other bright shit in the periphery + normSize = floor([256,256]./samplingRate); + % set to even dimension + normSize = normSize + mod(normSize,2); + eSize = floor([64,64]./samplingRate); + eSize = eSize + mod(eSize,2); + % subtract the global mean and global center so an unsampled area mask can be made + dataPrj = dataPrj - mean(dataPrj(:)); + dataPrj = dataPrj ./ rms(dataPrj(:)); - bandPassPrj = BH_bandpass3d([sTX,sTY,1],0,0,lowPassCutoff,'cpu',pixelSize); - - diagnosticCell = cell(nPrjs,1); - evalMaskCell = cell(nPrjs,1); - - for iPrj = 1:nPrjs - evalMaskCell{iPrj} = zeros(gather([sTX,sTY,1]),'uint8'); - end - - % Any large shifts should be obvious in the original alignment, so only - % look around +/- this value - globalPeak = max(2,ceil(10/pixelSize)); - globalPeak = globalPeak + mod(globalPeak,2); + % % % dataPrj = dataPrj - BH_movingAverage(dataPrj,normSize); + dataAVG = BH_movingAverage(dataPrj,normSize); + % This needs to be calculated prior to normalizing the dataPrj + dataRMS = BH_movingRMS(dataPrj-dataAVG,eSize); + mRms = mean(dataRMS(:)); + sRms = rms(dataRMS(:)-mRms); - globalPeakMask = zeros([sTX,sTY,1],'single'); - - globalPeakMask(originPrj(1) -globalPeak : originPrj(1) + globalPeak,... - originPrj(2) -globalPeak : originPrj(2) + globalPeak) = 1; - - globalBinary = ( globalPeakMask > 0 ); - % Zero and only changed if CTF is refined. - defocusShifts = cell(nPrjs,1); - - if (calcCTF) - nToCheck = floor(ctfRange./ctfInc); - defShiftVect = ctfInc.*[-nToCheck:nToCheck]'; - else - nToCheck = 1; - defShiftVect = 0; - end - nDefTotal = length(defShiftVect); - - defocusCCC = cell(nPrjs,1); - expectedDefocusPerFiducial=cell(nPrjs,1); - - - if samplingRate > 1 - tiltSeries = sprintf('%scache/%s_ali%d_bin%d.fixed',CWD,tiltName,mapBackIter+1,samplingRate); + % FIXME + if (whitenProjections) + whitenBP = [2*PARTICLE_RADIUS,lowPassCutoff,pixelSize,PARTICLE_RADIUS]; + [dataPrj,NPS] = BH_whitenNoiseSpectrum(dataPrj,'',whitenBP,1); + % Create a matched filter. + refPrj = refPrj ./ NPS; NPS = []; else - tiltSeries = sprintf('%saliStacks/%s_ali%d.fixed',CWD,tiltName,mapBackIter+1); + % Local scaling + dataPrj = dataPrj - dataAVG; + dataPrj = gather(dataPrj ./ BH_movingRMS(dataPrj,normSize)); end - - - - % Optionally restrict the search to a given number of fiducials: - nUniqueFids = numel(unique(fidList(:,2))); % I think the max val of this column should also be okay (+1) - nFidsTotal = nUniqueFids; -% nFidsTotal = sum(fidList(:,5) == 1 ); - if tomoCPR_random_subset == -1 || tomoCPR_random_subset > nUniqueFids - fprintf('Using all of the %d available fiducials\n',nUniqueFids); - else - fprintf('Using a random subset of %d fiducials from the %d available\n',... - tomoCPR_random_subset, nUniqueFids); - - keepFids = datasample(0:nUniqueFids-1,tomoCPR_random_subset,'Replace',false); - fidList(~ismember(fidList(:,2),keepFids),2) = -9999; - nFidsTotal = tomoCPR_random_subset; + + + % Global scaling + dataPrj = dataPrj - mean(dataPrj(:)); + dataPrj = dataPrj ./ rms(dataPrj(:)); + refPrj = refPrj - mean(refPrj(:)); + refPrj = refPrj ./ rms(refPrj(:)); + + + % In principle the robust fitting in imod tilt should get rid of + % outliers, however, with a systematic error in tracking this helps + % to avoid those outliers in the first place. + evalMask = ( dataRMS > (mRms - 2*sRms) ); + dataRMS = []; + + minEval = 0.9 * (2*tileRadius)^2; + evalMask(~samplingMask) = false; + samplingMask = []; + + mean_defocus = TLT(iPrj,15); + half_astigmatism = TLT(iPrj,12); + angle_astigmatism = TLT(iPrj,13); + defVect = [mean_defocus - half_astigmatism, mean_defocus + half_astigmatism, angle_astigmatism]; + + [Hqz, HqzUnMod] = BH_ctfCalc(TLT(iPrj,16).*samplingRate,TLT(iPrj,17), ... + TLT(iPrj,18),defVect,size(refPrj), ... + 1.*TLT(iPrj,19),-0.15); + + Hqz = gather(Hqz); + + HqzUnMod = gather(HqzUnMod); + + + + + cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).* abs(HqzUnMod).*... + conj(fftn(refPrj).*Hqz)))); + % cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).*... + % conj(fftn(refPrj).*Hqz)))); + + cccPrj = (cccPrj-min(cccPrj(:))) .* globalPeakMask; + [~,maxPRJ] = max(cccPrj(:)); + [mRx, mRy] = ind2sub(size(cccPrj), maxPRJ); + try + cccPRJ = cccPrj(mRx-COM:mRx+COM, mRy-COM:mRy+COM); + catch + mRx + mRy + COM + SAVE_IMG(MRCImage(gather(cccPrj)),'err.mrc'); + error('failed to box out the cross-correlation for image\n%s\non Projection %d\n', ... + tiltName, iPrj); end + cccPRJ = cccPRJ - min(cccPRJ(:)); + comPRJX = sum(sum(bx.*cccPRJ))./sum(cccPRJ(:)); + comPRJY = sum(sum(by.*cccPRJ))./sum(cccPRJ(:)); + + + + + estPeak = [mRx, mRy] - originPrj(1:2) + [comPRJX, comPRJY]; + glbList = fopen(sprintf('%smapBack%d/%s_%03d.global',mbOUT{1:3},iPrj),'w'); + % Add unique indicies to prevent ambiquity when comparing with paral + fprintf(glbList,'%f degree tilt at %f %f\n', TLT(iPrj,4),estPeak); + dataPrj = BH_resample2d(dataPrj,[0,0,0],[estPeak,0],'Bah',METHOD,'inv',1,size(dataPrj)); + %mapBack%d, imshow3D(gather(fftshift(real(ifftn(fftn(dataPrj).*conj(fftn(refPrj))))))); + + % figure, imshow3D(gather(cccPrj)); + + % Sanity check for geometry + cccPrj = cccPrj - mean(cccPrj(:)); + + diagnosticCell{iPrj} = reshape(cccPrj(globalBinary),2*globalPeak+1,2*globalPeak+1); + cccPrj = []; + + % Both are ordered by fiducial (imod contour number) but are not + % explicitly checked to correspond. Should this be done? + + % fid list is produced by projection of the 3dmodel using tilt with + % the angles supplied sorted from (-) --> (+) + wrkPrjIDX = ( fidList(:,5) == iPrj - 1 ); + wrkFid = fidList(wrkPrjIDX,:); + wrkPar = parList(wrkPrjIDX,:); + + wrkDefAngTilt = defList(wrkPrjIDX,[7,6,5]); + + coordOUT = fopen(sprintf('%smapBack%d/%s_%03d.coordFIT',mbOUT{1:3},iPrj),'w'); + + + + for iFid = 1:size(wrkFid,1) - for iPrj = 1:nPrjs - % I must specify the number of fiducials somehwere else, replace the - % unique when there is time. - defocusShifts{iPrj} = 0; - defocusCCC{iPrj} = zeros(nDefTotal, nFidsTotal,'single','gpuArray'); - expectedDefocusPerFiducial{iPrj} = zeros(nDefTotal,nFidsTotal,'single'); - end - - %Put back into a natural order - TLT = sortrows(TLT,1); - - if isnan(k_factor_scaling) - k_factor_scaling = 10 / sqrt(nFidsTotal); - end - -parfor iPrj = 1:nPrjs - -% % For some reason if these mrc objects are created before the parfor - % loop begins, they fail to load. It is fine as a regular for loop - % though - annoying, but very little overhead. It would be nice - % to know what is going on here. - bhF = fourierTransformer(randn(CTFSIZE,'single','gpuArray'),'OddSizeOversampled'); - - iMrcObj = MRCImage(tiltSeries,0); - iMrcObjRef = MRCImage(sprintf('%smapBack%d/%s_1_mapBack.st',mbOUT{1:3}),0); - iMrcObjSamplingMask = MRCImage(sprintf('%saliStacks/%s_ali%d.fixed.samplingMask',CWD,tiltName,mapBackIter+1),0); - - - tic - while toc < 300 - try - dataPrj = single(getVolume(iMrcObj,[1,sTX],[1,sTY],iPrj,'keep')); - break - catch - pause(1e-1) - if ~(mod(toc,10)) - fprintf('Waiting for %d sec for dataPrj to be available\n',toc); - end - end - end - if toc == 300 - error('failed to load dataPrj %s at %d after %d tries\n.', ... - tiltSeries,iPrj,3000); - else -% fprintf('loaded dataPrj %d on try %d\n',iPrj,floor(toc./0.1)); + if wrkFid(iFid,2) == -9999 + fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), [-4,-4], -9999); + continue end - tic - while toc < 300 - try - refPrj = single(getVolume(iMrcObjRef,[1,sTX],[1,sTY],iPrj,'keep')); - break - catch - pause(1e-1) - if ~(mod(toc,10)) - fprintf('Waiting for %d sec for refPrj to be available\n',toc); - end - end + + pixelX = wrkFid(iFid,3) - pixelShift + postShift(1); + pixelY = wrkFid(iFid,4) - pixelShift + postShift(2); + + ox = floor(pixelX) - tileRadius; + oy = floor(pixelY) - tileRadius; + + sx = pixelMultiplier*(pixelX - floor(pixelX)); + sy = pixelMultiplier*(pixelY - floor(pixelY)); + + % ox = floor(wrkFid(iFid,3)) - tileRadius; + % oy = floor(wrkFid(iFid,4)) - tileRadius; + oxEval = [floor(wrkFid(iFid,3) - PARTICLE_RADIUS),floor(wrkFid(iFid,3) + PARTICLE_RADIUS)]; + oyEval = [floor(wrkFid(iFid,4) - PARTICLE_RADIUS),floor(wrkFid(iFid,4) + PARTICLE_RADIUS)]; + % it would be good to try a smaller tile. + % First check that the data are found in this given projection + + try + % If any zeros values within the particle radius, do not evaluate + iSkipEval = any(any(evalMask(oxEval(1):oxEval(2),oyEval(1):oyEval(2)) == 0)); + % iSkipEval = sum(evalMask(oxEval(1):oxEval(2),oyEval(1):oyEval(2)),'all') < minEval; + catch + % If the particle was outof bounds, do not evaluate + iSkipEval = 1; end - if toc == 300 - error('failed to load refPrj %s at %d after %d tries\n.', ... - tiltSeries,iPrj,3000); - else -% fprintf('loaded refPrj %d on try %d\n',iPrj,floor(toc./0.1)); - end - tic - while toc < 300 - try - samplingMask = gpuArray(single(getVolume(iMrcObjSamplingMask,[],[],iPrj,'keep'))); - break - catch - pause(1e-1) - if ~(mod(toc,10)) - fprintf('Waiting for %d sec for samplingMask to be available\n',toc); - end - end + + if ( iSkipEval ) + % fprintf('\nThe current fiducial %d is not sampled in this projection %d\n',iFid,iPrj); + fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), [2,-2], -9999); + continue end - if toc == 300 - error('failed to load samplingMask %s at %d after %d tries\n.', ... - tiltSeries,iPrj,3000); - else -% fprintf('loaded refPrj %d on try %d\n',iPrj,floor(toc./0.1)); - end - - if (samplingRate > 1) - samplingMask = BH_resample2d(samplingMask,[0,0,0],[0,0],'Bah','GPU','forward',1/samplingRate,[sTX,sTY]); - else - + + if (ox < 1 || oy < 1 || ox +2*tileRadius > sTX || oy +2*tileRadius > sTY ) + fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), [2,-2], -9999); + continue end - % stored as uint8, it comes out 128,129 instead of 0,1.FIXME - samplingMask = ( samplingMask == max(samplingMask(:)) ); + dataTile = dataPrj(ox:ox+2.*tileRadius,oy:oy+2.*tileRadius); + dataTile = dataTile - mean(dataTile(:)); - % In case there is any carbon or other bright shit in the periphery - normSize = floor([256,256]./samplingRate); - % set to even dimension - normSize = normSize + mod(normSize,2); - eSize = floor([64,64]./samplingRate); - eSize = eSize + mod(eSize,2); - % subtract the global mean and global center so an unsampled area mask can be made - dataPrj = dataPrj - mean(dataPrj(:)); - dataPrj = dataPrj ./ rms(dataPrj(:)); + refTile = refPrj(ox:ox+2.*tileRadius,oy:oy+2.*tileRadius); + refTile = refTile - mean(refTile(:)); -% % % dataPrj = dataPrj - BH_movingAverage(dataPrj,normSize); - dataAVG = BH_movingAverage(dataPrj,normSize); - % This needs to be calculated prior to normalizing the dataPrj - dataRMS = BH_movingRMS(dataPrj-dataAVG,eSize); - mRms = mean(dataRMS(:)); - sRms = rms(dataRMS(:)-mRms); - - % FIXME - if (whitenProjections) - whitenBP = [2*PARTICLE_RADIUS,lowPassCutoff,pixelSize,PARTICLE_RADIUS]; - [dataPrj,NPS] = BH_whitenNoiseSpectrum(dataPrj,'',whitenBP,1); - % Create a matched filter. - refPrj = refPrj ./ NPS; NPS = []; - else - % Local scaling - dataPrj = dataPrj - dataAVG; - dataPrj = gather(dataPrj ./ BH_movingRMS(dataPrj,normSize)); - end - - % Global scaling - dataPrj = dataPrj - mean(dataPrj(:)); - dataPrj = dataPrj ./ rms(dataPrj(:)); - refPrj = refPrj - mean(refPrj(:)); - refPrj = refPrj ./ rms(refPrj(:)); - - - % In principle the robust fitting in imod tilt should get rid of - % outliers, however, with a systematic error in tracking this helps - % to avoid those outliers in the first place. - evalMask = ( dataRMS > (mRms - 2*sRms) ); - dataRMS = []; - - minEval = 0.9 * (2*tileRadius)^2; - evalMask(~samplingMask) = false; - samplingMask = []; - - mean_defocus = TLT(iPrj,15); - half_astigmatism = TLT(iPrj,12); - angle_astigmatism = TLT(iPrj,13); - defVect = [mean_defocus - half_astigmatism, mean_defocus + half_astigmatism, angle_astigmatism]; + dataTile = dataTile./rms(dataTile(:)); + refTile = refTile ./ rms(refTile(:)); - [Hqz, HqzUnMod] = BH_ctfCalc(TLT(iPrj,16).*samplingRate,TLT(iPrj,17), ... - TLT(iPrj,18),defVect,size(refPrj), ... - 1.*TLT(iPrj,19),-0.15); - - Hqz = gather(Hqz); + dataTile = ctfMask.*BH_padZeros3d(dataTile,'fwd',padCTF, ... + 'GPU','singleTaper'); - HqzUnMod = gather(HqzUnMod); + refTile = ctfMask.*BH_padZeros3d(refTile,'fwd',padCTF, ... + 'GPU','singleTaper'); - - - - cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).* abs(HqzUnMod).*... - conj(fftn(refPrj).*Hqz)))); -% cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).*... -% conj(fftn(refPrj).*Hqz)))); - - cccPrj = (cccPrj-min(cccPrj(:))) .* globalPeakMask; - [~,maxPRJ] = max(cccPrj(:)); - [mRx, mRy] = ind2sub(size(cccPrj), maxPRJ); - try - cccPRJ = cccPrj(mRx-COM:mRx+COM, mRy-COM:mRy+COM); - catch - mRx - mRy - COM - SAVE_IMG(MRCImage(gather(cccPrj)),'err.mrc'); - error('failed to box out the cross-correlation for image\n%s\non Projection %d\n', ... - tiltName, iPrj); - end - cccPRJ = cccPRJ - min(cccPRJ(:)); - comPRJX = sum(sum(bx.*cccPRJ))./sum(cccPRJ(:)); - comPRJY = sum(sum(by.*cccPRJ))./sum(cccPRJ(:)); - - - - - estPeak = [mRx, mRy] - originPrj(1:2) + [comPRJX, comPRJY]; - glbList = fopen(sprintf('%smapBack%d/%s_%03d.global',mbOUT{1:3},iPrj),'w'); - % Add unique indicies to prevent ambiquity when comparing with paral - fprintf(glbList,'%f degree tilt at %f %f\n', TLT(iPrj,4),estPeak); - dataPrj = BH_resample2d(dataPrj,[0,0,0],[estPeak,0],'Bah',METHOD,'inv',1,size(dataPrj)); - %mapBack%d, imshow3D(gather(fftshift(real(ifftn(fftn(dataPrj).*conj(fftn(refPrj))))))); - - - % figure, imshow3D(gather(cccPrj)); - - % Sanity check for geometry - cccPrj = cccPrj - mean(cccPrj(:)); - - diagnosticCell{iPrj} = reshape(cccPrj(globalBinary),2*globalPeak+1,2*globalPeak+1); - cccPrj = []; - - % Both are ordered by fiducial (imod contour number) but are not - % explicitly checked to correspond. Should this be done? - % fid list is produced by projection of the 3dmodel using tilt with - % the angles supplied sorted from (-) --> (+) - wrkPrjIDX = ( fidList(:,5) == iPrj - 1 ); - wrkFid = fidList(wrkPrjIDX,:); - wrkPar = parList(wrkPrjIDX,:); - - wrkDefAngTilt = defList(wrkPrjIDX,[7,6,5]); - - coordOUT = fopen(sprintf('%smapBack%d/%s_%03d.coordFIT',mbOUT{1:3},iPrj),'w'); - - - - for iFid = 1:size(wrkFid,1) - - if wrkFid(iFid,2) == -9999 - fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), [-4,-4], -9999); - continue - end - - pixelX = wrkFid(iFid,3) - pixelShift + postShift(1); - pixelY = wrkFid(iFid,4) - pixelShift + postShift(2); - ox = floor(pixelX) - tileRadius; - oy = floor(pixelY) - tileRadius; - - sx = pixelMultiplier*(pixelX - floor(pixelX)); - sy = pixelMultiplier*(pixelY - floor(pixelY)); -% ox = floor(wrkFid(iFid,3)) - tileRadius; -% oy = floor(wrkFid(iFid,4)) - tileRadius; - oxEval = [floor(wrkFid(iFid,3) - PARTICLE_RADIUS),floor(wrkFid(iFid,3) + PARTICLE_RADIUS)]; - oyEval = [floor(wrkFid(iFid,4) - PARTICLE_RADIUS),floor(wrkFid(iFid,4) + PARTICLE_RADIUS)]; - % it would be good to try a smaller tile. - % First check that the data are found in this given projection - - try - % If any zeros values within the particle radius, do not evaluate - iSkipEval = any(any(evalMask(oxEval(1):oxEval(2),oyEval(1):oyEval(2)) == 0)); -% iSkipEval = sum(evalMask(oxEval(1):oxEval(2),oyEval(1):oyEval(2)),'all') < minEval; - catch - % If the particle was outof bounds, do not evaluate - iSkipEval = 1; - end + df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; + df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; + dfA = wrkPar(iFid,6); + + if (calcCTF) + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixelSize),pixelSize]),'fwd'); + refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixelSize),pixelSize])); + else + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixelSize]),'fwd'); + refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixelSize])); + end + + bestScore = -1000000; + bestCTF = 1; + for deltaCTF = 1:nDefTotal - if ( iSkipEval ) -% fprintf('\nThe current fiducial %d is not sampled in this projection %d\n',iFid,iPrj); - fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), [2,-2], -9999); - continue - end - - if (ox < 1 || oy < 1 || ox +2*tileRadius > sTX || oy +2*tileRadius > sTY ) - fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), [2,-2], -9999); - continue - end - - dataTile = dataPrj(ox:ox+2.*tileRadius,oy:oy+2.*tileRadius); - dataTile = dataTile - mean(dataTile(:)); - + iRefCTF = refFT .* ... + mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... + single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... + single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18))); - refTile = refPrj(ox:ox+2.*tileRadius,oy:oy+2.*tileRadius); - refTile = refTile - mean(refTile(:)); - + % try + iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')); - dataTile = dataTile./rms(dataTile(:)); - refTile = refTile ./ rms(refTile(:)); - - dataTile = ctfMask.*BH_padZeros3d(dataTile,'fwd',padCTF, ... - 'GPU','singleTaper'); - - refTile = ctfMask.*BH_padZeros3d(refTile,'fwd',padCTF, ... - 'GPU','singleTaper'); - - - - - df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; - df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; - dfA = wrkPar(iFid,6); - - if (calcCTF) - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixelSize),pixelSize]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixelSize),pixelSize])); + cccMap = dataFT .* iRefCTF; + if (use_PCF) + cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); else - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixelSize]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixelSize])); + % % % % % cccMap = peakMask.*real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(dataTile,1,0,[0,300,lowPassCutoff,pixelSize]).*conj(bhF.fwdFFT(refTile,1,0) .* iCTF),'fwd'))); + end - - bestScore = -1000000; - bestCTF = 1; - for deltaCTF = 1:nDefTotal - - iRefCTF = refFT .* ... - mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... - single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... - single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18))); - -% try - iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')); - - cccMap = dataFT .* iRefCTF; - if (use_PCF) - cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); - else -% % % % % cccMap = peakMask.*real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(dataTile,1,0,[0,300,lowPassCutoff,pixelSize]).*conj(bhF.fwdFFT(refTile,1,0) .* iCTF),'fwd'))); + cccMap = peakMask.*real(bhF.invFFT(cccMap)); - end - cccMap = peakMask.*real(bhF.invFFT(cccMap)); - - - - [maxVal,maxMap] = max(cccMap(:)); - defocusCCC{iPrj}(deltaCTF,iFid) = maxVal; - - - % It might be better to reorder the search, to reduce the - % number of times this loop is executed (assumming the value is - % closer to the center of the defocusVector) - if maxVal > bestScore - bestScore = maxVal; - bestCTF = deltaCTF; - if ~(calcCTF) - [mMx, mMy] = ind2sub(size(cccMap), maxMap); - - try - - % It would be good to know why this is out of bounds - % sometimes. FIXME - - cccMap = cccMap(mMx-COM:mMx+COM, mMy-COM:mMy+COM); - cccMap = cccMap - min(cccMap(:)); - - comMapX = sum(sum(bx.*cccMap))./sum(cccMap(:)); - comMapY = sum(sum(by.*cccMap))./sum(cccMap(:)); - - % peak in Map is where query is relative to ref, dXY then is the shift - % needed to move the predicted position to the measured. - % Data moved from a position of estPeak, so add this to dXY - - dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estPeak - [sx,sy]; - catch - dXY = estPeak - [sx,sy]; % TODO double check me - end - - end - end - end % End of loop over defocus values. - - if (calcCTF) - - [~,imDefC] = max(defocusCCC{iPrj}(:,iFid),[],1); - dCTF = imDefC; -% fprintf('New best score %3.6f for defocus shift %3.3eAng\n', bestScore, defShiftVect(dCTF)); - - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixelSize]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixelSize])); - - iRefCTF = refFT .* ... - mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... - single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... - single(df1 + defShiftVect(dCTF)),single(df2 + defShiftVect(dCTF)),single(dfA),single(TLT(iPrj,18))); - % Renormalize - dataFT = dataFT ./ (sum(abs(dataFT(:)).^2)./numel(dataFT)); - iRefCTF = iRefCTF ./ (sum(abs(iRefCTF(:)).^2)./numel(iRefCTF)); - - cccMap = dataFT .* iRefCTF; - if (use_PCF) - cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); - - end - cccMap = peakMask.*real(bhF.invFFT(cccMap)); - - - [~,maxMap] = max(cccMap(:)); - - [mMx, mMy] = ind2sub(size(cccMap), maxMap); - - cccMap = cccMap(mMx-COM:mMx+COM, mMy-COM:mMy+COM); - - cccMap = cccMap - min(cccMap(:)); - - comMapX = sum(sum(bx.*cccMap))./sum(cccMap(:)); - comMapY = sum(sum(by.*cccMap))./sum(cccMap(:)); - - % peak in Map is where query is relative to ref, dXY then is the shift - % needed to move the predicted position to the measured. - % Data moved from a position of estPeak, so add this to dXY - - dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estPeak - [sx,sy]; - end - + + [maxVal,maxMap] = max(cccMap(:)); + defocusCCC{iPrj}(deltaCTF,iFid) = maxVal; + + + % It might be better to reorder the search, to reduce the + % number of times this loop is executed (assumming the value is + % closer to the center of the defocusVector) + if maxVal > bestScore + bestScore = maxVal; + bestCTF = deltaCTF; + if ~(calcCTF) + [mMx, mMy] = ind2sub(size(cccMap), maxMap); + + try - fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), dXY, wrkFid(iFid,5)); - - + % It would be good to know why this is out of bounds + % sometimes. FIXME + + cccMap = cccMap(mMx-COM:mMx+COM, mMy-COM:mMy+COM); + cccMap = cccMap - min(cccMap(:)); + + comMapX = sum(sum(bx.*cccMap))./sum(cccMap(:)); + comMapY = sum(sum(by.*cccMap))./sum(cccMap(:)); + + % peak in Map is where query is relative to ref, dXY then is the shift + % needed to move the predicted position to the measured. + % Data moved from a position of estPeak, so add this to dXY + + dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estPeak - [sx,sy]; + catch + dXY = estPeak - [sx,sy]; % TODO double check me + end + end + end + end % End of loop over defocus values. + - end % end of loop over fiducials - if (calcCTF) - [~,imDefC] = max(defocusCCC{iPrj},[],1); - expectedDefocus = mean(defShiftVect(imDefC)); - defocusShifts{iPrj} = expectedDefocus; - fprintf('prj %d delDef %3.3e\n',expectedDefocus); - end - evalMaskCell{iPrj} = uint8(evalMask); evalMask = []; - fclose(coordOUT); - end % end of the parfor loop - - if ( calcCTF ) - - save(sprintf('%smapBack%d/%s%s.defShiftsMat',mbOUT{1:3},outCTF),'defocusShifts'); - defShifts = fopen(sprintf('%smapBack%d/%s%s.defShifts',mbOUT{1:3},outCTF),'w'); - defCCC = sprintf('%smapBack%d/%s%s_defCCC.mat',mbOUT{1:3},outCTF); - save(defCCC,'defocusCCC','expectedDefocusPerFiducial'); - for iPrj = 1:nPrjs - fprintf(defShifts,'%6.6e\n',defocusShifts{iPrj}); + + [~,imDefC] = max(defocusCCC{iPrj}(:,iFid),[],1); + dCTF = imDefC; + % fprintf('New best score %3.6f for defocus shift %3.3eAng\n', bestScore, defShiftVect(dCTF)); + + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixelSize]),'fwd'); + refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixelSize])); + + iRefCTF = refFT .* ... + mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... + single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... + single(df1 + defShiftVect(dCTF)),single(df2 + defShiftVect(dCTF)),single(dfA),single(TLT(iPrj,18))); + % Renormalize + dataFT = dataFT ./ (sum(abs(dataFT(:)).^2)./numel(dataFT)); + iRefCTF = iRefCTF ./ (sum(abs(iRefCTF(:)).^2)./numel(iRefCTF)); + + cccMap = dataFT .* iRefCTF; + if (use_PCF) + cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); + + end + cccMap = peakMask.*real(bhF.invFFT(cccMap)); + + + [~,maxMap] = max(cccMap(:)); + + [mMx, mMy] = ind2sub(size(cccMap), maxMap); + + cccMap = cccMap(mMx-COM:mMx+COM, mMy-COM:mMy+COM); + + cccMap = cccMap - min(cccMap(:)); + + comMapX = sum(sum(bx.*cccMap))./sum(cccMap(:)); + comMapY = sum(sum(by.*cccMap))./sum(cccMap(:)); + + % peak in Map is where query is relative to ref, dXY then is the shift + % needed to move the predicted position to the measured. + % Data moved from a position of estPeak, so add this to dXY + + dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estPeak - [sx,sy]; end - end -% evalMaskStack = zeros(sTX,sTY,nPrjs); -% diagnosticStack = zeros([(globalPeak.*2+1).*[1,1],nPrjs],'single'); -% for iPrj = 1:nPrjs -% diagnosticStack(:,:,iPrj) = gather(diagnosticCell{iPrj}); -% evalMaskStack(:,:,iPrj) = int16(gather(evalMaskCell{iPrj})); -% end -% if (bh_global_save_tomoCPR_diagnostics) -% diagnosticStack = zeros([(globalPeak.*2+1).*[1,1],nPrjs],'single'); -% for iPrj = 1:nPrjs -% diagnosticStack(:,:,iPrj) = gather(diagnosticCell{iPrj}); -% evalMaskStack(:,:,iPrj) = int16(gather(evalMaskCell{iPrj})); -% end -% end - - clear diagnosticCell evalMaskCell -% if ~(conserveDiskSpace) && bh_global_save_tomoCPR_diagnostics -% SAVE_IMG(MRCImage(gather(diagnosticStack)),sprintf('%smapBack%d/%s_diagnostic.mrc',mbOUT{1:3})); -% SAVE_IMG(MRCImage(gather(evalMaskStack)),sprintf('%smapBack%d/%s_evalMask.mrc',mbOUT{1:3})); -% end - system(sprintf('cat %smapBack%d/%s_???.coordFIT | sort -k 1 -g > %smapBack%d/%s.coordFIT',mbOUT{1:3},mbOUT{1:3})); - system(sprintf('rm %smapBack%d/%s_???.coordFIT',mbOUT{1:3})); - - system(sprintf('cat %smapBack%d/%s_???.global | sort -k 1 -g > %smapBack%d/%s.global',mbOUT{1:3},mbOUT{1:3})); - system(sprintf('rm %smapBack%d/%s_???.global ',mbOUT{1:3})); - - - % create model tomogram for cross correlation - fidShifts = load(sprintf('%smapBack%d/%s.coordFIT',mbOUT{1:3})); - fidShifts = fidShifts(:,2:end); - fidCombine = fopen(sprintf('%smapBack%d/%s.coordCombine',mbOUT{1:3}),'w'); - % Output at full sampling for tiltalign - fidFull = fopen(sprintf('%smapBack%d/%s.coordFull',mbOUT{1:3}),'w'); + + + fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), dXY, wrkFid(iFid,5)); + + + + + end % end of loop over fiducials + if (calcCTF) - fidDefFull = fopen(sprintf('%smapBack%d/%s%s.defFidFull',mbOUT{1:3},outCTF),'w'); + [~,imDefC] = max(defocusCCC{iPrj},[],1); + expectedDefocus = mean(defShiftVect(imDefC)); + defocusShifts{iPrj} = expectedDefocus; + fprintf('prj %d delDef %3.3e\n',expectedDefocus); end - fidBin = fopen(sprintf('%smapBack%d/%s.coordBin%d',mbOUT{1:3},samplingRate),'w'); - fidList = fidList(:,2:end); - - - % shifts/List col 1/4 should match - maybe add a check to be safe - size(fidShifts) - size(fidList) - - fCombine = [fidShifts(:,1),fidList(:,2:3)+fidShifts(:,2:3),fidShifts(:,4)]; - fprintf('\n\n%d/%d pts ignored\n\n',sum(fCombine(:,4)==-9999),size(fCombine,1)); - - + evalMaskCell{iPrj} = uint8(evalMask); evalMask = []; + fclose(coordOUT); + end % end of the parfor loop + + if ( calcCTF ) - fFull = fCombine; - fDefFull = [fCombine,zeros(size(fCombine,1),1)]; -% % % % % fDefFull(:,2:3) = fDefFull(:,2:3).*pixelSize; - fDefFull(:,2:3) = fDefFull(:,2:3).*samplingRate; + save(sprintf('%smapBack%d/%s%s.defShiftsMat',mbOUT{1:3},outCTF),'defocusShifts'); + defShifts = fopen(sprintf('%smapBack%d/%s%s.defShifts',mbOUT{1:3},outCTF),'w'); + defCCC = sprintf('%smapBack%d/%s%s_defCCC.mat',mbOUT{1:3},outCTF); + save(defCCC,'defocusCCC','expectedDefocusPerFiducial'); for iPrj = 1:nPrjs - % Create a file that has the X,Y,defocus positions for all fiducials - % in each tilt to use in ctf correction. - wrkFidIDX = ( fidList(:,4) == iPrj - 1 ); - fDefFull(wrkFidIDX,5) = defList(wrkFidIDX,7) + defocusShifts{iPrj}; + fprintf(defShifts,'%6.6e\n',defocusShifts{iPrj}); end - + end + % evalMaskStack = zeros(sTX,sTY,nPrjs); + % diagnosticStack = zeros([(globalPeak.*2+1).*[1,1],nPrjs],'single'); + % for iPrj = 1:nPrjs + % diagnosticStack(:,:,iPrj) = gather(diagnosticCell{iPrj}); + % evalMaskStack(:,:,iPrj) = int16(gather(evalMaskCell{iPrj})); + % end + % if (bh_global_save_tomoCPR_diagnostics) + % diagnosticStack = zeros([(globalPeak.*2+1).*[1,1],nPrjs],'single'); + % for iPrj = 1:nPrjs + % diagnosticStack(:,:,iPrj) = gather(diagnosticCell{iPrj}); + % evalMaskStack(:,:,iPrj) = int16(gather(evalMaskCell{iPrj})); + % end + % end - fDefFull = fDefFull(fDefFull(:,4)~=-9999,:); - fFull = fFull(fFull(:,4)~=-9999,:); - fCombine = fCombine(fCombine(:,4)~=-9999,:); - - if (calcCTF) - fprintf(fidDefFull,'%d %4.4f %4.4f %d %2.6e\n',fDefFull'); - fclose(fidDefFull); - end - - fprintf(fidBin,'%d %4.4f %4.4f %d\n',fFull'); - fclose(fidBin); - -% % % % % fFull(:,2:3) = fFull(:,2:3).*samplingRate; - % The model ends up seeing the pixel size as 1, so even though it loads - % properly on the full aligned stack, these coords need to be scaled by - % the pixel size since this is the input to tiltalign. - fFull(:,2:3) = fFull(:,2:3).*pixelSize; - - fprintf(fidCombine,'%d %4.4f %4.4f %d\n',fCombine'); - fclose(fidCombine); - - fprintf(fidFull,'%d %4.4f %4.4f %d\n',fFull'); - fclose(fidFull); - % convert to model - system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... - '%smapBack%d/%s.coordCombine %smapBack%d/%s_fit-comb.fid'],mbOUT{1:3},mbOUT{1:3})); - system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... - '%smapBack%d/%s.coordFull %smapBack%d/%s_fit-full.fid'],mbOUT{1:3},mbOUT{1:3})); - system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... - '%smapBack%d/%s.coordBin%d %smapBack%d/%s_fit-bin%d.fid'],mbOUT{1:3},samplingRate,mbOUT{1:3},samplingRate)); - % write the com script for running tiltalign - TN =tiltBaseName - RotDef = 5; - TltDef = 4; - aliCom = fopen(sprintf('%smapBack%d/%s.align',mbOUT{1:3}),'w'); -% % % % % fprintf(aliCom,['#!/bin/bash\n\n',... -% % % % % 'tiltalign -StandardInput << EOF\n',... -% % % % % 'ModelFile %smapBack%d/%s_fit-full.fid\n',... -% % % % % 'ImageSizeXandY %d,%d\n',... -% % % % % 'ImagePixelSizeXandY %f,%f\n',... -% % % % % 'ImagesAreBinned 1\n',... -% % % % % 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... -% % % % % 'OutputResidualFile %smapBack%d/%s%s.resid\n',... -% % % % % 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... -% % % % % 'OutputTiltFile %smapBack%d/%s%s.tlt\n',... -% % % % % 'OutputXAxisTiltFile %smapBack%d/%s%s.xtilt\n',... -% % % % % 'OutputTransformFile %smapBack%d/%s%s.tltxf\n',... -% % % % % 'RotationAngle 0.00\n',... % assumed to be rotated already -% % % % % 'TiltFile %s\n',... -% % % % % 'SurfacesToAnalyze 2\n',... -% % % % % 'RotOption 1\n',... % def solve all rotations -% % % % % 'RotDefaultGrouping 3\n',... % if rot option --> 5 use def group size -% % % % % 'TiltOption %d\n',... % Tilts are harder use automapping -% % % % % 'TiltDefaultGrouping %d\n',... -% % % % % 'MagOption 1\n',... % def solve all mags -% % % % % 'MagDefaultGrouping 3\n',... -% % % % % 'XStretchOption 0\n',... -% % % % % 'SkewOption 0\n',... -% % % % % 'BeamTiltOption 0\n',... -% % % % % 'XTiltOption 0\n',... -% % % % % 'ResidualReportCriterion 0.001\n',... -% % % % % 'ShiftZFromOriginal\n',... -% % % % % 'AxisZShift 0.0\n',... -% % % % % 'RobustFitting\n',... -% % % % % 'KFactorScaling %3.3f\n',... -% % % % % 'LocalAlignments\n',... -% % % % % 'LocalRotOption 1\n',... -% % % % % 'LocalRotDefaultGrouping 3\n',... -% % % % % 'LocalTiltOption %d\n',... -% % % % % 'LocalTiltDefaultGrouping %d\n',... -% % % % % 'LocalMagOption %d\n',... -% % % % % 'LocalMagDefaultGrouping 5\n',... -% % % % % 'OutputLocalFile %smapBack%d/%s%s.local\n',... -% % % % % 'TargetPatchSizeXandY %d,%d\n', ... -% % % % % 'MinFidsTotalAndEachSurface %d,%d\n',... -% % % % % 'MinSizeOrOverlapXandY 0.5,0.5\n',... -% % % % % 'LocalOutputOptions 1,1,1\n', ... -% % % % % 'EOF'],mbOUT{1:3},fullTiltSizeXandY,... -% % % % % fullPixelSize,fullPixelSize,... -% % % % % mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,... -% % % % % mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,mbOUT{1:3},outCTF, ... -% % % % % iRawTltName,tiltAliOption(1:2),... -% % % % % 10 / sqrt(nFidsTotal),tiltAliOption(3:4),flgLocalMag, ... -% % % % % mbOUT{1:3},outCTF,targetPatchSize,targetPatchSize,... -% % % % % nFiducialsPerPatch,floor(nFiducialsPerPatch/3)); - - if (shift_z_to_to_centroid) - final_line1 = 'ShiftZFromOriginal'; - final_line2 = 'AxisZShift 0.0'; - final_line3 = 'LocalOutputOptions 1,1,1'; + clear diagnosticCell evalMaskCell + % if ~(conserveDiskSpace) && bh_global_save_tomoCPR_diagnostics + % SAVE_IMG(MRCImage(gather(diagnosticStack)),sprintf('%smapBack%d/%s_diagnostic.mrc',mbOUT{1:3})); + % SAVE_IMG(MRCImage(gather(evalMaskStack)),sprintf('%smapBack%d/%s_evalMask.mrc',mbOUT{1:3})); + % end + system(sprintf('cat %smapBack%d/%s_???.coordFIT | sort -k 1 -g > %smapBack%d/%s.coordFIT',mbOUT{1:3},mbOUT{1:3})); + system(sprintf('rm %smapBack%d/%s_???.coordFIT',mbOUT{1:3})); + + system(sprintf('cat %smapBack%d/%s_???.global | sort -k 1 -g > %smapBack%d/%s.global',mbOUT{1:3},mbOUT{1:3})); + system(sprintf('rm %smapBack%d/%s_???.global ',mbOUT{1:3})); + + + % create model tomogram for cross correlation + fidShifts = load(sprintf('%smapBack%d/%s.coordFIT',mbOUT{1:3})); + fidShifts = fidShifts(:,2:end); + fidCombine = fopen(sprintf('%smapBack%d/%s.coordCombine',mbOUT{1:3}),'w'); + % Output at full sampling for tiltalign + fidFull = fopen(sprintf('%smapBack%d/%s.coordFull',mbOUT{1:3}),'w'); + if (calcCTF) + fidDefFull = fopen(sprintf('%smapBack%d/%s%s.defFidFull',mbOUT{1:3},outCTF),'w'); + end + fidBin = fopen(sprintf('%smapBack%d/%s.coordBin%d',mbOUT{1:3},samplingRate),'w'); + fidList = fidList(:,2:end); + + + % shifts/List col 1/4 should match - maybe add a check to be safe + size(fidShifts) + size(fidList) + + fCombine = [fidShifts(:,1),fidList(:,2:3)+fidShifts(:,2:3),fidShifts(:,4)]; + fprintf('\n\n%d/%d pts ignored\n\n',sum(fCombine(:,4)==-9999),size(fCombine,1)); + + + + fFull = fCombine; + fDefFull = [fCombine,zeros(size(fCombine,1),1)]; + % % % % % fDefFull(:,2:3) = fDefFull(:,2:3).*pixelSize; + fDefFull(:,2:3) = fDefFull(:,2:3).*samplingRate; + for iPrj = 1:nPrjs + % Create a file that has the X,Y,defocus positions for all fiducials + % in each tilt to use in ctf correction. + wrkFidIDX = ( fidList(:,4) == iPrj - 1 ); + fDefFull(wrkFidIDX,5) = defList(wrkFidIDX,7) + defocusShifts{iPrj}; + end + + + fDefFull = fDefFull(fDefFull(:,4)~=-9999,:); + fFull = fFull(fFull(:,4)~=-9999,:); + fCombine = fCombine(fCombine(:,4)~=-9999,:); + + if (calcCTF) + fprintf(fidDefFull,'%d %4.4f %4.4f %d %2.6e\n',fDefFull'); + fclose(fidDefFull); + end + + fprintf(fidBin,'%d %4.4f %4.4f %d\n',fFull'); + fclose(fidBin); + + % % % % % fFull(:,2:3) = fFull(:,2:3).*samplingRate; + % The model ends up seeing the pixel size as 1, so even though it loads + % properly on the full aligned stack, these coords need to be scaled by + % the pixel size since this is the input to tiltalign. + fFull(:,2:3) = fFull(:,2:3).*pixelSize; + + fprintf(fidCombine,'%d %4.4f %4.4f %d\n',fCombine'); + fclose(fidCombine); + + fprintf(fidFull,'%d %4.4f %4.4f %d\n',fFull'); + fclose(fidFull); + % convert to model + system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... + '%smapBack%d/%s.coordCombine %smapBack%d/%s_fit-comb.fid'],mbOUT{1:3},mbOUT{1:3})); + system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... + '%smapBack%d/%s.coordFull %smapBack%d/%s_fit-full.fid'],mbOUT{1:3},mbOUT{1:3})); + system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... + '%smapBack%d/%s.coordBin%d %smapBack%d/%s_fit-bin%d.fid'],mbOUT{1:3},samplingRate,mbOUT{1:3},samplingRate)); + % write the com script for running tiltalign + TN =tiltBaseName + RotDef = 5; + TltDef = 4; + aliCom = fopen(sprintf('%smapBack%d/%s.align',mbOUT{1:3}),'w'); + % % % % % fprintf(aliCom,['#!/bin/bash\n\n',... + % % % % % 'tiltalign -StandardInput << EOF\n',... + % % % % % 'ModelFile %smapBack%d/%s_fit-full.fid\n',... + % % % % % 'ImageSizeXandY %d,%d\n',... + % % % % % 'ImagePixelSizeXandY %f,%f\n',... + % % % % % 'ImagesAreBinned 1\n',... + % % % % % 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... + % % % % % 'OutputResidualFile %smapBack%d/%s%s.resid\n',... + % % % % % 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... + % % % % % 'OutputTiltFile %smapBack%d/%s%s.tlt\n',... + % % % % % 'OutputXAxisTiltFile %smapBack%d/%s%s.xtilt\n',... + % % % % % 'OutputTransformFile %smapBack%d/%s%s.tltxf\n',... + % % % % % 'RotationAngle 0.00\n',... % assumed to be rotated already + % % % % % 'TiltFile %s\n',... + % % % % % 'SurfacesToAnalyze 2\n',... + % % % % % 'RotOption 1\n',... % def solve all rotations + % % % % % 'RotDefaultGrouping 3\n',... % if rot option --> 5 use def group size + % % % % % 'TiltOption %d\n',... % Tilts are harder use automapping + % % % % % 'TiltDefaultGrouping %d\n',... + % % % % % 'MagOption 1\n',... % def solve all mags + % % % % % 'MagDefaultGrouping 3\n',... + % % % % % 'XStretchOption 0\n',... + % % % % % 'SkewOption 0\n',... + % % % % % 'BeamTiltOption 0\n',... + % % % % % 'XTiltOption 0\n',... + % % % % % 'ResidualReportCriterion 0.001\n',... + % % % % % 'ShiftZFromOriginal\n',... + % % % % % 'AxisZShift 0.0\n',... + % % % % % 'RobustFitting\n',... + % % % % % 'KFactorScaling %3.3f\n',... + % % % % % 'LocalAlignments\n',... + % % % % % 'LocalRotOption 1\n',... + % % % % % 'LocalRotDefaultGrouping 3\n',... + % % % % % 'LocalTiltOption %d\n',... + % % % % % 'LocalTiltDefaultGrouping %d\n',... + % % % % % 'LocalMagOption %d\n',... + % % % % % 'LocalMagDefaultGrouping 5\n',... + % % % % % 'OutputLocalFile %smapBack%d/%s%s.local\n',... + % % % % % 'TargetPatchSizeXandY %d,%d\n', ... + % % % % % 'MinFidsTotalAndEachSurface %d,%d\n',... + % % % % % 'MinSizeOrOverlapXandY 0.5,0.5\n',... + % % % % % 'LocalOutputOptions 1,1,1\n', ... + % % % % % 'EOF'],mbOUT{1:3},fullTiltSizeXandY,... + % % % % % fullPixelSize,fullPixelSize,... + % % % % % mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,... + % % % % % mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,mbOUT{1:3},outCTF, ... + % % % % % iRawTltName,tiltAliOption(1:2),... + % % % % % 10 / sqrt(nFidsTotal),tiltAliOption(3:4),flgLocalMag, ... + % % % % % mbOUT{1:3},outCTF,targetPatchSize,targetPatchSize,... + % % % % % nFiducialsPerPatch,floor(nFiducialsPerPatch/3)); + + if (shift_z_to_to_centroid) + final_line1 = 'ShiftZFromOriginal'; + final_line2 = 'AxisZShift 0.0'; + final_line3 = 'LocalOutputOptions 1,1,1'; + else + final_line3 = ''; + final_line2 = ''; + final_line1 = 'LocalOutputOptions 1,1,1'; + end + + mbOutAlt = mbOUT; + tilt_script_name = iRawTltName; + if (flgAltRun) + mbOutAlt{1} = 'cache/'; + [~,tn2,tn3] = fileparts(iRawTltName); + tilt_script_name = sprintf('cache/mapBack%d/%s%s',mbOUT{2},tn2,tn3); + end + + fprintf(aliCom,['#!/bin/bash\n\n',... + '#iTiltSeries %d\n',... + 'tiltalign -StandardInput << EOF\n',... + 'ModelFile %smapBack%d/%s_fit-full.fid\n',... + 'ImageSizeXandY %d,%d\n',... + 'ImagePixelSizeXandY %f,%f\n',... + 'ImagesAreBinned 1\n',... + 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... + 'OutputResidualFile %smapBack%d/%s%s.resid\n',... + 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... + 'OutputTiltFile %smapBack%d/%s%s.tlt\n',... + 'OutputXAxisTiltFile %smapBack%d/%s%s.xtilt\n',... + 'OutputTransformFile %smapBack%d/%s%s.tltxf\n',... + 'RotationAngle 0.00\n',... % assumed to be rotated already + 'TiltFile %s\n',... + 'SurfacesToAnalyze %d\n',... + 'RotOption %d\n',... % def solve all rotations + 'RotDefaultGrouping 3\n',... % if rot option --> 5 use def group size + 'TiltOption %d\n',... % Tilts are harder use automapping + 'TiltDefaultGrouping %d\n',... + 'MagOption %d\n',... % def solve all mags + 'MagDefaultGrouping %d\n',... + 'XStretchOption 0\n',... + 'SkewOption 0\n',... + 'BeamTiltOption 0\n',... + 'XTiltOption 0\n',... + 'ResidualReportCriterion 0.001\n',... + 'RobustFitting\n',... + 'KFactorScaling %3.3f\n',... + 'LocalAlignments\n',... + 'LocalRotOption %d\n',... + 'LocalRotDefaultGrouping %d\n',... + 'LocalTiltOption %d\n',... + 'LocalTiltDefaultGrouping %d\n',... + 'LocalMagOption %d\n',... + 'LocalMagDefaultGrouping %d\n',... + 'OutputLocalFile %smapBack%d/%s%s.local\n',... + 'TargetPatchSizeXandY %d,%d\n', ... + 'MinFidsTotalAndEachSurface %d,%d\n',... + 'MinSizeOrOverlapXandY %f,%f\n',... + '%s\n',... + '%s\n',... + '%s\n',... + 'EOF'],iTiltSeries,mbOutAlt{1:3},fullTiltSizeXandY,... + fullPixelSize,fullPixelSize,... + mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... + mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... + tilt_script_name,n_surfaces,rot_option_global,... + tilt_option_global,tilt_default_grouping_global,... + mag_option_global,mag_default_grouping_global,... + k_factor_scaling,... + rot_option_local,rot_default_grouping_local,... + tilt_option_local,tilt_default_grouping_local,... + mag_option_local,mag_default_grouping_local, ... + mbOutAlt{1:3},outCTF,targetPatchSize,targetPatchSize,... + nFiducialsPerPatch,floor(nFiducialsPerPatch/3),... + min_overlap,min_overlap,... + final_line1,final_line2,final_line3); + + % % % Assume that any backlash was solved well enough that there are no major + % % % discontinuities in the coarse alignment. Mag and rot are solved/ tilt in + % % % the global solution anyhow, so this shouldn't be a bit deal. + % % % 'SeparateGroup 1-%d\n',... + % % % iViewGroup, + % fprintf(aliCom,'\n\ngrep -A %d " At minimum tilt" ./mapBack%d/%s_ta.log > ./mapBack%d/tmp.log',nPrjs+2,mbOUT{1:3},mbOUT{1:3}); + % fprintf(aliCom,'\nawk ''{if(NR >3) print $5}'' ./mapBack%d/tmp.log > mapBack%d/%s.mag',mbOUT{1:3},mbOUT{1:3}); + + fclose(aliCom); + system(sprintf('chmod a=wrx %smapBack%d/%s.align',mbOUT{1:3})); + + if (is_first_run) + if ( flgAltRun ) + fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'w'); + fprintf(fOUT,['%smapBack%d/%s.align > ',... + '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); else - final_line3 = ''; - final_line2 = ''; - final_line1 = 'LocalOutputOptions 1,1,1'; + fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}),'w'); + fprintf(fOUT,['#!/bin/bash\n\n%smapBack%d/%s.align > ',... + '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); end - mbOutAlt = mbOUT; - tilt_script_name = iRawTltName; - if (flgAltRun) - mbOutAlt{1} = 'cache/'; - [~,tn2,tn3] = fileparts(iRawTltName); - tilt_script_name = sprintf('cache/mapBack%d/%s%s',mbOUT{2},tn2,tn3); + % Since we send to the background in a shell, makes sure the + % function waits on children. + %if (iTiltSeries == nTiltSeries) + % fprintf(fOUT,'\nwait\n'); + %end + fclose(fOUT); + is_first_run = false; + else + if ( flgAltRun ) + fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'a'); + else + fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}), 'a'); end - - fprintf(aliCom,['#!/bin/bash\n\n',... - '#iTiltSeries %d\n',... - 'tiltalign -StandardInput << EOF\n',... - 'ModelFile %smapBack%d/%s_fit-full.fid\n',... - 'ImageSizeXandY %d,%d\n',... - 'ImagePixelSizeXandY %f,%f\n',... - 'ImagesAreBinned 1\n',... - 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... - 'OutputResidualFile %smapBack%d/%s%s.resid\n',... - 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... - 'OutputTiltFile %smapBack%d/%s%s.tlt\n',... - 'OutputXAxisTiltFile %smapBack%d/%s%s.xtilt\n',... - 'OutputTransformFile %smapBack%d/%s%s.tltxf\n',... - 'RotationAngle 0.00\n',... % assumed to be rotated already - 'TiltFile %s\n',... - 'SurfacesToAnalyze %d\n',... - 'RotOption %d\n',... % def solve all rotations - 'RotDefaultGrouping 3\n',... % if rot option --> 5 use def group size - 'TiltOption %d\n',... % Tilts are harder use automapping - 'TiltDefaultGrouping %d\n',... - 'MagOption %d\n',... % def solve all mags - 'MagDefaultGrouping %d\n',... - 'XStretchOption 0\n',... - 'SkewOption 0\n',... - 'BeamTiltOption 0\n',... - 'XTiltOption 0\n',... - 'ResidualReportCriterion 0.001\n',... - 'RobustFitting\n',... - 'KFactorScaling %3.3f\n',... - 'LocalAlignments\n',... - 'LocalRotOption %d\n',... - 'LocalRotDefaultGrouping %d\n',... - 'LocalTiltOption %d\n',... - 'LocalTiltDefaultGrouping %d\n',... - 'LocalMagOption %d\n',... - 'LocalMagDefaultGrouping %d\n',... - 'OutputLocalFile %smapBack%d/%s%s.local\n',... - 'TargetPatchSizeXandY %d,%d\n', ... - 'MinFidsTotalAndEachSurface %d,%d\n',... - 'MinSizeOrOverlapXandY %f,%f\n',... - '%s\n',... - '%s\n',... - '%s\n',... - 'EOF'],iTiltSeries,mbOutAlt{1:3},fullTiltSizeXandY,... - fullPixelSize,fullPixelSize,... - mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... - mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... - tilt_script_name,n_surfaces,rot_option_global,... - tilt_option_global,tilt_default_grouping_global,... - mag_option_global,mag_default_grouping_global,... - k_factor_scaling,... - rot_option_local,rot_default_grouping_local,... - tilt_option_local,tilt_default_grouping_local,... - mag_option_local,mag_default_grouping_local, ... - mbOutAlt{1:3},outCTF,targetPatchSize,targetPatchSize,... - nFiducialsPerPatch,floor(nFiducialsPerPatch/3),... - min_overlap,min_overlap,... - final_line1,final_line2,final_line3); - -% % % Assume that any backlash was solved well enough that there are no major -% % % discontinuities in the coarse alignment. Mag and rot are solved/ tilt in -% % % the global solution anyhow, so this shouldn't be a bit deal. -% % % 'SeparateGroup 1-%d\n',... -% % % iViewGroup, -% fprintf(aliCom,'\n\ngrep -A %d " At minimum tilt" ./mapBack%d/%s_ta.log > ./mapBack%d/tmp.log',nPrjs+2,mbOUT{1:3},mbOUT{1:3}); -% fprintf(aliCom,'\nawk ''{if(NR >3) print $5}'' ./mapBack%d/tmp.log > mapBack%d/%s.mag',mbOUT{1:3},mbOUT{1:3}); - - fclose(aliCom); - system(sprintf('chmod a=wrx %smapBack%d/%s.align',mbOUT{1:3})); - - if (is_first_run) - if ( flgAltRun ) - fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'w'); - fprintf(fOUT,['%smapBack%d/%s.align > ',... - '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); - else - fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}),'w'); - fprintf(fOUT,['#!/bin/bash\n\n%smapBack%d/%s.align > ',... - '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); - end - - % Since we send to the background in a shell, makes sure the - % function waits on children. - %if (iTiltSeries == nTiltSeries) - % fprintf(fOUT,'\nwait\n'); - %end - fclose(fOUT); - is_first_run = false; - else - if ( flgAltRun ) - fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'a'); - else - fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}), 'a'); - end - fprintf(fOUT,['%smapBack%d/%s.align > ',... - '%smapBack%d/%s.align_ta.log &\n'], ... - mbOutAlt{1:3},mbOutAlt{1:3}); - % Since we send to the background in a shell, makes sure the - % function waits on children. - %if (iTiltSeries == nTiltSeries) - % fprintf(fOUT,'\nwait\n'); - %end - fclose(fOUT); - end - - %system(sprintf('./mapBack/%s.align > ./mapBack/%s_ta.log',TN,TN)); - - %%%%%%%%% There is still sometimes a shift in Z, fit slope of the X shifts - %%%%%%%%% and make cutoff compared to zshift to re-run the alignment with - %%%%%%%%% this value entered for AxisZShift. - - % Until I hear from DAVID get the mag from the tilt log - %%%system(sprintf('grep -A %d " At minimum tilt" ./mapBack/%s_ta.log > tmp.log',nPrjs+2,TN)); - %%%system(sprintf('awk ''{if(NR >3) print $5}'' tmp.log > mapBack/%s.mag',TN)); - %%%end %uf cibdutuib - + fprintf(fOUT,['%smapBack%d/%s.align > ',... + '%smapBack%d/%s.align_ta.log &\n'], ... + mbOutAlt{1:3},mbOutAlt{1:3}); + % Since we send to the background in a shell, makes sure the + % function waits on children. + %if (iTiltSeries == nTiltSeries) + % fprintf(fOUT,'\nwait\n'); + %end + fclose(fOUT); + end + + %system(sprintf('./mapBack/%s.align > ./mapBack/%s_ta.log',TN,TN)); + + %%%%%%%%% There is still sometimes a shift in Z, fit slope of the X shifts + %%%%%%%%% and make cutoff compared to zshift to re-run the alignment with + %%%%%%%%% this value entered for AxisZShift. + + % Until I hear from DAVID get the mag from the tilt log + %%%system(sprintf('grep -A %d " At minimum tilt" ./mapBack/%s_ta.log > tmp.log',nPrjs+2,TN)); + %%%system(sprintf('awk ''{if(NR >3) print $5}'' tmp.log > mapBack/%s.mag',TN)); + %%%end %uf cibdutuib + end % loop over tilts if ( flgRunAlignments ) - + mainFile = sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}); altFiles = sprintf('%smapBack%d/runAlignments_*.sh',mbOUT{1:2}); @@ -2209,7 +2209,7 @@ fprintf('Combining Results from alt and main\n'); system(sprintf('cat %s >> %s',altFiles,mainFile)); end - + fOUT = fopen(mainFile,'a'); fprintf(fOUT,'\nwait\n'); @@ -2218,7 +2218,7 @@ system(sprintf('chmod a=wrx %smapBack%d/runAlignments.sh', mbOUT{1:2})); system(sprintf('%smapBack%d/runAlignments.sh', mbOUT{1:2})); -end +end if ( conserveDiskSpace ) system(sprintf('rm %smapBack%d/%s_mapBack.st', mbOUT{1:3})); @@ -2249,14 +2249,14 @@ % Since we've updated (potentially) mapBackRePrjSize, save the new metaData. if (flgRunAlignments) subTomoMeta.currentTomoCPR = subTomoMeta.currentTomoCPR + 1; - + if isfield(subTomoMeta,'tomoCPR_run_in_cycle') subTomoMeta.('tomoCPR_run_in_cycle') = cat(1,subTomoMeta.('tomoCPR_run_in_cycle'),... - [subTomoMeta.currentTomoCPR,CYCLE]); + [subTomoMeta.currentTomoCPR,CYCLE]); else subTomoMeta.('tomoCPR_run_in_cycle') = [subTomoMeta.currentTomoCPR,CYCLE]; end - + save(emc.('subTomoMeta'), 'subTomoMeta'); end diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 679f3880..3802e41f 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -15,10 +15,10 @@ global bh_global_imodProjectionShifts; if isempty(bh_global_imodProjectionShifts) -% bh_global_imodProjectionShifts = [ -0.5, -0.5, 0.5 ; -0.5, -0.5, 0; 0.5,0.5,1.0 ]; + % bh_global_imodProjectionShifts = [ -0.5, -0.5, 0.5 ; -0.5, -0.5, 0; 0.5,0.5,1.0 ]; % From a more thorough sweep bh_global_imodProjectionShifts = [ 0.5, -0.5, 0.5 ; 0.0, -0.5, 0; 0.5,0.5,1.0 ]; - + end @@ -50,16 +50,16 @@ cacheAdd = ''; end else - + % in tilt the coords are transformed from the model coordinate system to % the tomogram by [0.5,0.5,1.0]. After transformation, these are then % subtracted back off. preShift = bh_global_imodProjectionShifts(1,:); postShift = bh_global_imodProjectionShifts(2,1:2); prjVectorShift = bh_global_imodProjectionShifts(3,:)'; -% prjVectorShift = [0,0,-1]; + % prjVectorShift = [0,0,-1]; + - end % baseFile = sprintf('%s_%d_%2.2f_preShift_%2.2f_%2.2f_%2.2f_postShift_%2.2f_%2.2f_prjVect_%2.2f_%2.2f_%2.2f','microShiftsFollowup',MAX_EXPOSURE, preShift, postShift, prjVectorShift); baseFile = outputName; %sprintf('%s_%d_%2.2f','withZeroedXF_IPFirst',MAX_EXPOSURE); @@ -81,8 +81,8 @@ gpuScale=3*samplingRate nWorkers = min(nGPUs*gpuScale,emc.('nCpuCores')); % 18 fprintf('Using %d workers as max of %d %d*nGPUs and %d nWorkers visible\n', ... - nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); - + nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); + tmpCache = sprintf('cache/to_cisTEM%s/',cacheAdd); CWD = ''; @@ -104,21 +104,21 @@ % Cycle 0 is named differently - I'll be deleting this in an overhaul of the way % the subTomoMeta is written. if (CYCLE) - try + try geometry = subTomoMeta.(cycleNumber).RawAlign; - fprintf('Using Alignment geometry %s\n',cycleNumber); + fprintf('Using Alignment geometry %s\n',cycleNumber); catch geometry = subTomoMeta.(cycleNumber).Avg_geometry; - fprintf('Using Average geometry %s\n',cycleNumber); + fprintf('Using Average geometry %s\n',cycleNumber); end else - try + try geometry = subTomoMeta.(cycleNumber).RawAlign; fprintf('Using Alignment geometry %s\n',cycleNumber); - + catch geometry = subTomoMeta.(cycleNumber).geometry; - fprintf('Using Average geometry %s\n',cycleNumber); + fprintf('Using Average geometry %s\n',cycleNumber); end end @@ -132,99 +132,99 @@ output_cell = {}; % TODO split this up into chunks for iTiltSeries = tiltStart:nTiltSeries - + nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos; if nTomograms == 0 % No points were saved after template matching so skip this tilt seoarries % altogether. continue - end + end - tomoList = {}; + tomoList = {}; tomoIDX = 1; for iTomo = 1:size(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords,1) % This is dumb, fix it to be explicit. if any(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords(iTomo,:)) tomoList{tomoIDX} = sprintf('%s_%d',tiltNameList{iTiltSeries},iTomo); - + tiltList{tomoIDX} = sprintf('%saliStacks/%s_ali%d.fixed',... - CWD,tiltNameList{iTiltSeries},mapBackIter+1); - % Only increment if values found. - tomoIDX = tomoIDX + 1; + CWD,tiltNameList{iTiltSeries},mapBackIter+1); + % Only increment if values found. + tomoIDX = tomoIDX + 1; end end - + if (mapBackIter) localFile = sprintf('%smapBack%d/%s_ali%d_ctf.local', ... - CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter); + CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter); else localFile = sprintf('%sfixedStacks/%s.local',CWD,tiltNameList{iTiltSeries}); end - + if exist(localFile,'file') fprintf('Found local file %s\n.', localFile); else fprintf('No local transforms found.\n'); localFile = 0; end - - + + % The model is scaled to full sampling prior to passing to tiltalign, % make sure the header in the synthetic stack is set appropriately. fullPixelSize = emc.('PIXEL_SIZE').*10^10; - + pixelSize = fullPixelSize.*samplingRate; - + PARTICLE_RADIUS = floor(max(emc.('particleRadius')./pixelSize)); - + [~,tiltBaseName,~] = fileparts(tiltList{1}); mbOUT = {[tmpCache],[mapBackIter+1],[tiltBaseName]}; fprintf('\nmBOUT name is %smapBack%d/%s\n',mbOUT{1:3}); - - + + tiltHeader = getHeader(MRCImage(tiltList{1},0)); % This is only needed in the re-projection of the model. I don't think it % should affect anything, but double check. FIXME maxZ = 100; - - + + % The tilt angles are the same for each tomo, so it is okay to just use % number 1 here. TLT = tiltGeometry.(tomoList{1}); - - + + iRawTltName = sprintf('%smapBack%d/%s_align.rawtlt',mbOUT{1:3}) iTiltFile = fopen(iRawTltName, 'w'); rawTLT = sortrows(TLT(:,[1,4]),1); fprintf(iTiltFile,'%f\n',rawTLT(:,2)'); - fclose(iTiltFile); - + fclose(iTiltFile); + coordOUT = fopen(sprintf('%smapBack%d/%s.coord',mbOUT{1:3}),'w'); coordSTART = fopen(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3}),'w'); - + defOUT = fopen(sprintf('%smapBack%d/%s.defAng',mbOUT{1:3}),'w'); % Track the number of fiducials in order to scale the K-factor to more or less % aggressivley downweight outliers in the alignment nFidsTotal = 0; for iTomo = 1:nTomograms - + TLT = tiltGeometry.(tomoList{iTomo}); - + doseList = TLT(:,[1,11]); postExposure = doseList(:,2)'; [sorted_doseList, doseIDX] = sortrows(doseList,2); preExposure = diff(sorted_doseList(:,2)); preExposure = [preExposure; preExposure(end)]; preExposure = postExposure - preExposure(doseIDX)'; - - + + % Extract a "defocus file" for tilt to calculate the defocus for each % fiducial also considering the local alignment. If this works, I can - % get rid of defAng + % get rid of defAng iDefocusFileName = sprintf('%smapBack%d/%s_align.defocus',mbOUT{1:3}); iDefocusFile = fopen(iDefocusFileName,'w'); defTLT = sortrows(TLT(:,[1,15]),1); @@ -237,14 +237,14 @@ % We also need the transform from the microscope frame in order to % get an accurate defocus value. Not sure if I should be binning? % Additionally, we do NOT want the model for alignment in the - % microscope frame, + % microscope frame, iXFName = sprintf('%smapBack%d/%s_align.XF',mbOUT{1:3}); iXF = fopen(iXFName,'w'); - + if (useFixedNotAliStack || doFullXform) - + % 20190509 - I think this is royally screwing things up FIXME % Commenting this out invalidates the defocus vals xfTLT = sortrows(TLT(:,[1,7:10,2,3],1)); @@ -254,238 +254,238 @@ % xform. if (useFixedNotAliStack) isEven = 1; - + iXFBase = sprintf('%smapBack%d/%s_align_base.XF',mbOUT{1:3}); iXFB = fopen(iXFBase,'w'); for ix = 1:size(xfTLT,1) fprintf(iXFB,'%f %f %f %f %f %f\n',[1,0,0,1,-isEven,-isEven]); end - + fclose(iXFB); system(sprintf('xfproduct %s %s %s',iXFBase, iXFName,iXFName)); - + % We need to invert this transform to map from the aligned stack to the % fixed stack iXFName_inv = sprintf('%smapBack%d/%s_align_inv.XF',mbOUT{1:3}); system(sprintf('xfinverse %s %s', iXFName, iXFName_inv)); end else - % 20190509 - I think this is royally screwing things up FIXME - % Commenting this out invalidates the defocus vals + % 20190509 - I think this is royally screwing things up FIXME + % Commenting this out invalidates the defocus vals xfTLT = zeros(size(TLT,1),6); xfTLT(:,[1,4]) = 1.0; fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT'); fclose(iXF); -% xfTLT = sortrows(TLT(:,[1,7:10,2,3],1)); -% fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT(:,2:7)'); -% fclose(iXF); + % xfTLT = sortrows(TLT(:,[1,7:10,2,3],1)); + % fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT(:,2:7)'); + % fclose(iXF); end - + positionList = geometry.(tomoList{iTomo}); positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); - + sTX = floor(tiltHeader.nX ); sTY = floor(tiltHeader.nY ); - + originPrj = floor([sTX,sTY,1]./2) + 1; - + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - + flgLoad = 0; [~,tomoReconCoords] = BH_multi_loadOrBuild(tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate, 1,reconScaling,flgLoad, 'tomoCPR'); - + reconCoords, mapBackIter, ... + samplingRate, 1,reconScaling,flgLoad, 'tomoCPR'); + originVol = floor(tomoReconCoords(1,1:3)./2) + 1; reconShift = tomoReconCoords(2,1:3); maxZ = 1000; % Does not seem to affect anything. reconstructionSize = [tiltHeader.nX,tiltHeader.nY,maxZ]; originRec = floor(reconstructionSize./2) + 1; - - + + nPrjs = size(TLT,1); nSubTomos = size(positionList,1); - - + + if (iTomo == 1) fidIDX = 0; end - + modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector'); - + for iSubTomo = 1:nSubTomos - - + + rSubTomo = reshape(positionList(iSubTomo,17:25),3,3); prjVector = (positionList(iSubTomo,11:13)./samplingRate) - originVol + reconShift; - -% % % nRefs = 1; -% % % if (nRefs > 1) -% % % iClassIDX = positionList(iSubTomo,26); -% % % else -% % % iClassIDX = 1; -% % % end -% prjVector = prjVector - [0.5,0.5,1.0]; %prjVectorShift; - prjVector = prjVector - preShift; - % Reproject using tilt, so just save the 3d coords. + % % % nRefs = 1; + % % % if (nRefs > 1) + % % % iClassIDX = positionList(iSubTomo,26); + % % % else + % % % iClassIDX = 1; + % % % end + + % prjVector = prjVector - [0.5,0.5,1.0]; %prjVectorShift; + prjVector = prjVector - preShift; + % Reproject using tilt, so just save the 3d coords. fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot*prjVector' + [originRec(1),originRec(3),originRec(2)]' - prjVectorShift([1,3,2]), fidIDX); - + nPrjsIncluded = 0; for iPrj = 1:nPrjs - + iPrj_nat = find(TLT(:,1) == iPrj); if (abs(TLT(iPrj_nat,11)) <= MAX_EXPOSURE) nPrjsIncluded = nPrjsIncluded + 1; - - + + % imod is indexing from zero - zCoord = iPrj_nat; - - - + zCoord = iPrj_nat; + + + rTilt = BH_defineMatrix([90,1.*TLT(iPrj_nat,4),-90],'Bah','forwardVector'); - - + + prjCoords = rTilt*prjVector'; - + fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)); -% d1 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; -% d2 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; + % d1 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; + % d2 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; d1 = -1.*(samplingRate.*prjVector(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15))*10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value - + fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',fidIDX, tomoNumber,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(rSubTomo,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); - + nFidsTotalDataSet = nFidsTotalDataSet + 1; - + else fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',-9999, -9999,-9999,1.0,1.0,1.0,1,1,1,1,1,1,1,1,1,0,0,1); - + nFidsTotalDataSet = nFidsTotalDataSet + 1; end - end % loop over tilt projections - - - fidIDX = fidIDX + 1; - + end % loop over tilt projections + + + fidIDX = fidIDX + 1; + end % loop over subtomos - + end - - - fclose(coordOUT); - fclose(coordSTART); - - - p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 ',... - '%smapBack%d/%s.coord %smapBack%d/%s.3dfid > /dev/null'], ... - mbOUT{1:3},mbOUT{1:3}); - system(p2m); - - - - taStr = [sprintf('%f',rawTLT(1,2))]; - for iTa = 2:length(rawTLT(:,2)) - taStr = [taStr sprintf(',%f',rawTLT(iTa,2))]; - end - - - - if (localFile) - lastLine1 = sprintf('LOCALFILE %s', localFile) - % Used if GPU fails - cpuLastLine = lastLine1; - else - lastLine1 = ''; - cpuLastLine = ''; - end - - - if (lastLine1) - lastLine2 = 'UseGPU 0'; - lastLine3 = 'ActionIfGPUFails 2,2'; - else - lastLine1 = 'UseGPU 0'; - lastLine2 = 'ActionIfGPUFails 2,2'; - lastLine3 = ''; - end - - - - % Break this up into chunks since things hang even with the - % ActionIfGPUFails option. Try 3 times 512,256,128 -% refPrj = zeros(sTX,sTY,iTLT, 'single'); - - iSave = 1; - reModFileName = sprintf('%smapBack%d/%s_%d_reMod.sh',mbOUT{1:3},iSave); - reModFile = fopen(reModFileName,'w'); - invertTiltAngles = 0; - fprintf(reModFile,['#!/bin/bash\n\n',... - 'tilt -StandardInput << EOF\n',... - 'input %s\n', ... - 'output %smapBack%d/%s.fid\n', ... - 'COSINTERP 0\n', ... - 'THICKNESS %d\n', ... - 'TILTFILE %smapBack%d/%s_align.rawtlt \n', ... - 'DefocusFile %smapBack%d/%s_align.defocus \n', ... - 'PixelForDefocus %f,%f\n', ... - 'AngleOutputFile %smapBack%d/%s.defAngTilt\n', ... - 'AlignTransformFile %smapBack%d/%s_align.XF\n', ... - 'ProjectModel %smapBack%d/%s.3dfid\n', ... - '%s\n',... - '%s\n',... - '%s\n',... - 'EOF'],tiltList{1}, mbOUT{1:3}, maxZ, ... - mbOUT{1:3},... - mbOUT{1:3},... - pixelSize/10, invertTiltAngles,... % Ang --> nm - mbOUT{1:3},... - mbOUT{1:3},... - mbOUT{1:3},... - lastLine1,lastLine2,... - lastLine3); - - fclose(reModFile); - system(sprintf('chmod a=wrx %s',reModFileName)); - - - [failedToRun,~] = system(sprintf('%s',reModFileName)); - % Sometimes the file is busy - if (failedToRun) - system(sprintf('mv %s %s.tmp',reModFileName,reModFileName)); - system(sprintf('cp %s.tmp %s',reModFileName,reModFileName)); - system(sprintf('rm %s.tmp',reModFileName)); - system(sprintf('%s',reModFileName)); - end - - if (useFixedNotAliStack) - % transform the projected model back to the fixed stack frame, and - % then convert to text. - - system(sprintf('imodtrans -2 %s %smapBack%d/%s.fid %smapBack%d/%s.invfid', iXFName_inv, mbOUT{1:3},mbOUT{1:3})); - - system(sprintf(['model2point -contour -zero ',... - '%smapBack%d/%s.invfid %smapBack%d/%s.coordPrj'],... - mbOUT{1:3}, mbOUT{1:3})) + + + fclose(coordOUT); + fclose(coordSTART); + + + p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 ',... + '%smapBack%d/%s.coord %smapBack%d/%s.3dfid > /dev/null'], ... + mbOUT{1:3},mbOUT{1:3}); + system(p2m); + + + + taStr = [sprintf('%f',rawTLT(1,2))]; + for iTa = 2:length(rawTLT(:,2)) + taStr = [taStr sprintf(',%f',rawTLT(iTa,2))]; + end + + + + if (localFile) + lastLine1 = sprintf('LOCALFILE %s', localFile) + % Used if GPU fails + cpuLastLine = lastLine1; else - system(sprintf(['model2point -contour -zero ',... - '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'],... - mbOUT{1:3}, mbOUT{1:3})) + lastLine1 = ''; + cpuLastLine = ''; end - - - + + + if (lastLine1) + lastLine2 = 'UseGPU 0'; + lastLine3 = 'ActionIfGPUFails 2,2'; + else + lastLine1 = 'UseGPU 0'; + lastLine2 = 'ActionIfGPUFails 2,2'; + lastLine3 = ''; + end + + + + % Break this up into chunks since things hang even with the + % ActionIfGPUFails option. Try 3 times 512,256,128 + % refPrj = zeros(sTX,sTY,iTLT, 'single'); + + iSave = 1; + reModFileName = sprintf('%smapBack%d/%s_%d_reMod.sh',mbOUT{1:3},iSave); + reModFile = fopen(reModFileName,'w'); + invertTiltAngles = 0; + fprintf(reModFile,['#!/bin/bash\n\n',... + 'tilt -StandardInput << EOF\n',... + 'input %s\n', ... + 'output %smapBack%d/%s.fid\n', ... + 'COSINTERP 0\n', ... + 'THICKNESS %d\n', ... + 'TILTFILE %smapBack%d/%s_align.rawtlt \n', ... + 'DefocusFile %smapBack%d/%s_align.defocus \n', ... + 'PixelForDefocus %f,%f\n', ... + 'AngleOutputFile %smapBack%d/%s.defAngTilt\n', ... + 'AlignTransformFile %smapBack%d/%s_align.XF\n', ... + 'ProjectModel %smapBack%d/%s.3dfid\n', ... + '%s\n',... + '%s\n',... + '%s\n',... + 'EOF'],tiltList{1}, mbOUT{1:3}, maxZ, ... + mbOUT{1:3},... + mbOUT{1:3},... + pixelSize/10, invertTiltAngles,... % Ang --> nm + mbOUT{1:3},... + mbOUT{1:3},... + mbOUT{1:3},... + lastLine1,lastLine2,... + lastLine3); + + fclose(reModFile); + system(sprintf('chmod a=wrx %s',reModFileName)); + + + [failedToRun,~] = system(sprintf('%s',reModFileName)); + % Sometimes the file is busy + if (failedToRun) + system(sprintf('mv %s %s.tmp',reModFileName,reModFileName)); + system(sprintf('cp %s.tmp %s',reModFileName,reModFileName)); + system(sprintf('rm %s.tmp',reModFileName)); + system(sprintf('%s',reModFileName)); + end + + if (useFixedNotAliStack) + % transform the projected model back to the fixed stack frame, and + % then convert to text. + + system(sprintf('imodtrans -2 %s %smapBack%d/%s.fid %smapBack%d/%s.invfid', iXFName_inv, mbOUT{1:3},mbOUT{1:3})); + + system(sprintf(['model2point -contour -zero ',... + '%smapBack%d/%s.invfid %smapBack%d/%s.coordPrj'],... + mbOUT{1:3}, mbOUT{1:3})) + else + system(sprintf(['model2point -contour -zero ',... + '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'],... + mbOUT{1:3}, mbOUT{1:3})) + end + + + try fidList = load(sprintf('%smapBack%d/%s.coordPrj',mbOUT{1:3})); catch @@ -495,112 +495,112 @@ parList = load(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3})); defList = load(sprintf('%smapBack%d/%s.defAngTilt',mbOUT{1:3})); -% Need to shift again from the model coordinate system - fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(prjVectorShift(1:2)', size(fidList,1),1); + % Need to shift again from the model coordinate system + fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(prjVectorShift(1:2)', size(fidList,1),1); foundNans = sum(isnan(fidList(:,3))); if (foundNans) fprintf('\n\t\tThere are %d NaNs in the projected fiducial list %3.3f\n\n',foundNans, foundNans/size(fidList,1)*100); fprintf('The only confirmed case that produced this were NaNs in the fixedStacks/tiltN.local file.\n'); error("Exiting"); end - + fidList = [1:size(fidList,1);fidList']'; - - - + + + particlePad = 2.0; tileRadius = floor(particlePad.*PARTICLE_RADIUS); tileSize = (2.*tileRadius).*[1,1]; - + tileOrigin = floor(tileSize./2) + 1; - + nFidsTotal = numel(unique(fidList(parList(:,1)~=-9999,2))); - - + + if (useFixedNotAliStack) tiltSeries = sprintf('%sfixedStacks/%s.fixed',CWD,tiltName); else tiltSeries = sprintf('%saliStacks/%s_ali%d.fixed',CWD,tiltName,mapBackIter+1); end - - % This will need to be changed to aggregate + + % This will need to be changed to aggregate output_particle_stack = zeros([tileSize,nFidsTotal*nPrjsIncluded],'single'); - + iGpuDataCounter = 1; if (firstTilt) - + iDataCounter = 1; starFile = fopen(sprintf('%s.star',baseFile),'w'); fprintf(starFile, [ ... - '# Written by emClarity Version 2.0.0-alpha on %s\n\n' ... - 'data_\n\n' ... - 'loop_\n\n' ... - '_cisTEMPositionInStack #1\n' ... - '_cisTEMAnglePsi #2\n' ... - '_cisTEMAngleTheta #3\n' ... - '_cisTEMAnglePhi #4\n' ... - '_cisTEMXShift #5\n' ... - '_cisTEMYShift #6\n' ... - '_cisTEMDefocus1 #7\n' ... - '_cisTEMDefocus2 #8\n' ... - '_cisTEMDefocusAngle #9\n' ... - '_cisTEMPhaseShift #10\n' ... - '_cisTEMOccupancy #11\n' ... - '_cisTEMLogP #12\n' ... - '_cisTEMSigma #13\n' ... - '_cisTEMScore #14\n' ... - '_cisTEMScoreChange #15\n' ... - '_cisTEMPixelSize #16\n' ... - '_cisTEMMicroscopeVoltagekV #17\n' ... - '_cisTEMMicroscopeCsMM #18\n' ... - '_cisTEMAmplitudeContrast #19\n' ... - '_cisTEMBeamTiltX #20\n' ... - '_cisTEMBeamTiltY #21\n' ... - '_cisTEMImageShiftX #22\n' ... - '_cisTEMImageShiftY #23\n' ... - '_cisTEMBest2DClass #24\n' ... - '_cisTEMBeamTiltGroup #25\n' ... - '_cisTEMParticleGroup #26\n' ... - '_cisTEMPreExposure #27\n' ... - '_cisTEMTotalExposure #28\n' ... - '# POS PSI THETA PHI SHX SHY DF1 DF2 ANGAST PSHIFT OCC LogP SIGMA SCORE CHANGE PSIZE VOLT Cs AmpC BTILTX BTILTY ISHFTX ISHFTY 2DCLS TGRP PARGRP PREEXP TOTEXP\n' ... - ], datetime); - - + '# Written by emClarity Version 2.0.0-alpha on %s\n\n' ... + 'data_\n\n' ... + 'loop_\n\n' ... + '_cisTEMPositionInStack #1\n' ... + '_cisTEMAnglePsi #2\n' ... + '_cisTEMAngleTheta #3\n' ... + '_cisTEMAnglePhi #4\n' ... + '_cisTEMXShift #5\n' ... + '_cisTEMYShift #6\n' ... + '_cisTEMDefocus1 #7\n' ... + '_cisTEMDefocus2 #8\n' ... + '_cisTEMDefocusAngle #9\n' ... + '_cisTEMPhaseShift #10\n' ... + '_cisTEMOccupancy #11\n' ... + '_cisTEMLogP #12\n' ... + '_cisTEMSigma #13\n' ... + '_cisTEMScore #14\n' ... + '_cisTEMScoreChange #15\n' ... + '_cisTEMPixelSize #16\n' ... + '_cisTEMMicroscopeVoltagekV #17\n' ... + '_cisTEMMicroscopeCsMM #18\n' ... + '_cisTEMAmplitudeContrast #19\n' ... + '_cisTEMBeamTiltX #20\n' ... + '_cisTEMBeamTiltY #21\n' ... + '_cisTEMImageShiftX #22\n' ... + '_cisTEMImageShiftY #23\n' ... + '_cisTEMBest2DClass #24\n' ... + '_cisTEMBeamTiltGroup #25\n' ... + '_cisTEMParticleGroup #26\n' ... + '_cisTEMPreExposure #27\n' ... + '_cisTEMTotalExposure #28\n' ... + '# POS PSI THETA PHI SHX SHY DF1 DF2 ANGAST PSHIFT OCC LogP SIGMA SCORE CHANGE PSIZE VOLT Cs AmpC BTILTX BTILTY ISHFTX ISHFTY 2DCLS TGRP PARGRP PREEXP TOTEXP\n' ... + ], datetime); + + firstTilt = 0; end - + if (useFixedNotAliStack) fullXform = load(iXFName_inv); end STACK = single(getVolume(MRCImage(tiltSeries))); - - for iPrj = 1:nPrjs + for iPrj = 1:nPrjs + if (abs(TLT(iPrj,11)) > MAX_EXPOSURE) - continue; + continue; end - + dataPrj = STACK(:,:,TLT(iPrj,1)); - + % Both are ordered by fiducial (imod contour number) but are not % explicitly checked to correspond. Should this be done? - + % fid list is produced by projection of the 3dmodel using tilt with wrkPrjIDX = ( fidList(:,5) == TLT(iPrj,1) - 1 ); wrkFid = fidList(wrkPrjIDX,:); wrkPar = parList(wrkPrjIDX,:); wrkDefAngTilt = defList(wrkPrjIDX,[7,6,5]); % Confirming with David but this should include the local adjustments to tilt/in-plane angle - + for iFid = 1:size(wrkFid,1) - + if (wrkPar(iFid,1) == -9999) continue; end - + pixelX = wrkFid(iFid,3) - pixelShift + postShift(1); pixelY = wrkFid(iFid,4) - pixelShift + postShift(2); @@ -608,10 +608,10 @@ ox = floor(pixelX) - tileRadius; oy = floor(pixelY) - tileRadius; - + sx = pixelX - floor(pixelX); sy = pixelY - floor(pixelY); - + particle_was_skipped = false; if ( ox > 0 && oy > 0 && ox + 2*tileRadius < sTX && oy +2*tileRadius < sTY ) output_particle_stack(:,:,iGpuDataCounter) = dataPrj(ox:ox+2.*tileRadius-1,oy:oy+2.*tileRadius-1); @@ -620,96 +620,96 @@ output_particle_stack(:,:,iGpuDataCounter) = randn(tileSize,'single').*0.1; % Why am I not just skipping these? end - if (useFixedNotAliStack) - rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); - RF = fullXform(TLT(iPrj,1),1:4); - rotFull = rTilt*[RF(1), RF(2), 0; RF(3), RF(4), 0; 0, 0, 1]*reshape(wrkPar(iFid,7:15),3,3); - else - - -% rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); - rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','forwardVector'); - - rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); - end + if (useFixedNotAliStack) + rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); + RF = fullXform(TLT(iPrj,1),1:4); + rotFull = rTilt*[RF(1), RF(2), 0; RF(3), RF(4), 0; 0, 0, 1]*reshape(wrkPar(iFid,7:15),3,3); + else - eul = rotm2eul(rotFull,'ZYZ'); - e1 = 180./pi.*eul(1); - e2 = 180./pi.*eul(2); - e3 = 180./pi.*eul(3); - - - phaseShift = 0.0; - occupancy = 100.0; % TODO test replacement with CCC score? - logp = -1000; - sigma = 10.0; - score = 10.0; % TODO test with scaled CCC score? - scoreChange = 0.0; - pixelSize = emc.('PIXEL_SIZE') * 10^10; - micVoltage = emc.('VOLTAGE') * 10^-3; - micCS = emc.('Cs') * 10^3; - ampContrast = emc.('AMPCONT') * 10^0; - beamTiltX = 0.0; - beamTiltY = 0.0; - beamTiltShiftX = 0.0; - beamTiltShiftY = 0.0; - best2dClass = 0.0; - if (particle_was_skipped) - beamTiltGroup = 0; % FSC half set, coopting this param for now. - else - beamTiltGroup = wrkPar(iFid,18); % FSC half set, coopting this param for now. - end - particleGroup = wrkPar(iFid,3); - preExposure = wrkPar(iFid,16); - totalExposure = wrkPar(iFid,17); - - xShift = pixelMultiplier*sx*pixelSize; - yShift = pixelMultiplier*sy*pixelSize; - -% df1 = wrkPar(iFid,4); -% df2 = wrkPar(iFid,5); -% dfA = wrkPar(iFid,6); - df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; - df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; - dfA = wrkPar(iFid,6); + % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); + rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','forwardVector'); - fprintf(starFile, '%8u %7.2f %7.2f %7.2f %9.2f %9.2f %8.1f %8.1f %7.2f %7.2f %5i %7.2f %9i %10.4f %7.2f %8.5f %7.2f %7.2f %7.4f %7.3f %7.3f %7.3f %7.3f %5i %5i %8u %7.2f %7.2f\n', ... - iDataCounter,-e1,-e2,-e3,xShift,yShift, ... - df1,df2,dfA, ... - phaseShift, occupancy, logp, sigma, score, scoreChange, ... - pixelSize, micVoltage, micCS, ampContrast, ... - beamTiltX, beamTiltY, beamTiltShiftX, beamTiltShiftY, ... - best2dClass, beamTiltGroup, particleGroup, preExposure, totalExposure); - + rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); + end - iDataCounter = iDataCounter + 1; - iGpuDataCounter = iGpuDataCounter + 1; - + eul = rotm2eul(rotFull,'ZYZ'); + e1 = 180./pi.*eul(1); + + e2 = 180./pi.*eul(2); + e3 = 180./pi.*eul(3); + + + phaseShift = 0.0; + occupancy = 100.0; % TODO test replacement with CCC score? + logp = -1000; + sigma = 10.0; + score = 10.0; % TODO test with scaled CCC score? + scoreChange = 0.0; + pixelSize = emc.('PIXEL_SIZE') * 10^10; + micVoltage = emc.('VOLTAGE') * 10^-3; + micCS = emc.('Cs') * 10^3; + ampContrast = emc.('AMPCONT') * 10^0; + beamTiltX = 0.0; + beamTiltY = 0.0; + beamTiltShiftX = 0.0; + beamTiltShiftY = 0.0; + best2dClass = 0.0; + if (particle_was_skipped) + beamTiltGroup = 0; % FSC half set, coopting this param for now. + else + beamTiltGroup = wrkPar(iFid,18); % FSC half set, coopting this param for now. end - - end % end of prj loop + particleGroup = wrkPar(iFid,3); + preExposure = wrkPar(iFid,16); + totalExposure = wrkPar(iFid,17); + + xShift = pixelMultiplier*sx*pixelSize; + yShift = pixelMultiplier*sy*pixelSize; + + % df1 = wrkPar(iFid,4); + % df2 = wrkPar(iFid,5); + % dfA = wrkPar(iFid,6); + df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; + df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; + dfA = wrkPar(iFid,6); + + fprintf(starFile, '%8u %7.2f %7.2f %7.2f %9.2f %9.2f %8.1f %8.1f %7.2f %7.2f %5i %7.2f %9i %10.4f %7.2f %8.5f %7.2f %7.2f %7.4f %7.3f %7.3f %7.3f %7.3f %5i %5i %8u %7.2f %7.2f\n', ... + iDataCounter,-e1,-e2,-e3,xShift,yShift, ... + df1,df2,dfA, ... + phaseShift, occupancy, logp, sigma, score, scoreChange, ... + pixelSize, micVoltage, micCS, ampContrast, ... + beamTiltX, beamTiltY, beamTiltShiftX, beamTiltShiftY, ... + best2dClass, beamTiltGroup, particleGroup, preExposure, totalExposure); + + + iDataCounter = iDataCounter + 1; + iGpuDataCounter = iGpuDataCounter + 1; + + end + end % end of prj loop + output_cell{iCell}= gather(output_particle_stack); iCell = iCell + 1; - + end fclose(starFile); - SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',baseFile),pixelSize); - - maxThreads = emc.('nCpuCores'); +SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',baseFile),pixelSize); - %%%%%%%%%%%%%%%%%%%%%%%%%% - % Initial reconstruction - %%%%%%%%%%%%%%%%%%%%%%%%%%% - system(sprintf('rm -f %s_rec.sh',baseFile)); - recScript = fopen(sprintf('%s_rec.sh',baseFile), 'w'); - fprintf(recScript,[ ... - '#!/bin/bash\n\n', ... +maxThreads = emc.('nCpuCores'); + +%%%%%%%%%%%%%%%%%%%%%%%%%% +% Initial reconstruction +%%%%%%%%%%%%%%%%%%%%%%%%%%% +system(sprintf('rm -f %s_rec.sh',baseFile)); +recScript = fopen(sprintf('%s_rec.sh',baseFile), 'w'); +fprintf(recScript,[ ... + '#!/bin/bash\n\n', ... '%s << eof\n', ... '%s.mrc\n', ... sprintf('%s.mrc',baseFile) '%s.star\n', ... sprintf('%s.star',baseFile) @@ -727,111 +727,111 @@ '%3.3f\n', ... outermas ang '0.0\n', ... rec res limit '0.0\n', ... ref res limit - '5.0\n', ... Particle weighting factor (A^2) [5.0] - '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] - '1.0\n', ...Tuning parameter: smoothing factor [1.0] : - '1.0\n', ...Tuning parameters: padding factor [1.0] : - 'Yes\n', ...Normalize particles [Yes] : - 'No\n', ...Adjust scores for defocus dependence [no] : - 'No\n', ...Invert particle contrast [No] : - 'Yes\n', ...Exclude images with blank edges [yes] : - 'No\n', ...Crop particle images [no] : - 'Yes\n', ...FSC calculation with even/odd particles [Yes] : - 'No\n', ...Center mass [No] : - 'No\n', ...Apply likelihood blurring [No] : - 'No\n', ...Threshold input reconstruction [No] : - 'No\n', ...Dump intermediate arrays (merge later) [No] : - 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : - 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : - '%2.2d\n', ...Max. threads to use for calculation [36] : + '5.0\n', ... Particle weighting factor (A^2) [5.0] + '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] + '1.0\n', ...Tuning parameter: smoothing factor [1.0] : + '1.0\n', ...Tuning parameters: padding factor [1.0] : + 'Yes\n', ...Normalize particles [Yes] : + 'No\n', ...Adjust scores for defocus dependence [no] : + 'No\n', ...Invert particle contrast [No] : + 'Yes\n', ...Exclude images with blank edges [yes] : + 'No\n', ...Crop particle images [no] : + 'Yes\n', ...FSC calculation with even/odd particles [Yes] : + 'No\n', ...Center mass [No] : + 'No\n', ...Apply likelihood blurring [No] : + 'No\n', ...Threshold input reconstruction [No] : + 'No\n', ...Dump intermediate arrays (merge later) [No] : + 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : + 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%2.2d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,emc.('PIXEL_SIZE')*10^10, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); - - fprintf(recScript, '\neof\n'); - - fclose(recScript); - system(sprintf('chmod a=wrx %s_rec.sh',baseFile)); - pause(3) - system(sprintf('./%s_rec.sh',baseFile)); - - - %%%%%%%%%%%%%%%%%%%%%%%%% - % Refine - %%%%%%%%%%%%%%%%%%%%%%%%%%%% - system(sprintf('rm -f %s_ref.sh',baseFile)); - refineScript = fopen(sprintf('%s_ref.sh',baseFile), 'w'); - fprintf(refineScript,[ ... + symmetry,emc.('PIXEL_SIZE')*10^10, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); + +fprintf(recScript, '\neof\n'); + +fclose(recScript); +system(sprintf('chmod a=wrx %s_rec.sh',baseFile)); +pause(3) +system(sprintf('./%s_rec.sh',baseFile)); + + +%%%%%%%%%%%%%%%%%%%%%%%%% +% Refine +%%%%%%%%%%%%%%%%%%%%%%%%%%%% +system(sprintf('rm -f %s_ref.sh',baseFile)); +refineScript = fopen(sprintf('%s_ref.sh',baseFile), 'w'); +fprintf(refineScript,[ ... '#!/bin/bash\n\n', ... - '%s << eof\n', ... + '%s << eof\n', ... '%s.mrc\n', ... sprintf('%s.mrc',baseFile) '%s.star\n', ... sprintf('%s.star',baseFile) '%s_recFilt.mrc\n',... '%s_stats.txt\n',... - 'yes\n',... Use statistics [Yes] : - 'my_projection_stack.mrc\n',... not going to be used : + 'yes\n',... Use statistics [Yes] : + 'my_projection_stack.mrc\n',... not going to be used : '%s_refined.star\n', ... '%s_changes.star\n', ...Output parameter changes - '%s\n',... Particle symmetry [C1] : - '1\n', ...First particle to refine (0 = first in stack) [1] : - '0\n', ...Last particle to refine (0 = last in stack) [0] : - '1.0\n',...Percent of particles to use (1 = all) [1.0] : + '%s\n',... Particle symmetry [C1] : + '1\n', ...First particle to refine (0 = first in stack) [1] : + '0\n', ...Last particle to refine (0 = last in stack) [0] : + '1.0\n',...Percent of particles to use (1 = all) [1.0] : '%3.3f\n', ... pixel size '%4.4f\n', ... molecularMass' '%3.3f\n', ... inermask ang '%3.3f\n', ... outermas ang - '300.0\n',...Low resolution limit (A) [300.0] : - '%3.3f\n',...High resolution limit (A) [8.0] : - '0.0\n',...Resolution limit for signed CC (A) (0.0 = max [0.0] : - '0.0\n',...Res limit for classification (A) (0.0 = max) [0.0] : - '0.0\n',...Mask radius for global search (A) (0.0 = max)[100.0] : - '%3.3f\n',...Approx. resolution limit for search (A) [8] : - '0.0\n',...Angular step (0.0 = set automatically) [0.0] : - '20\n',...Number of top hits to refine [20] : - '10\n',...Search range in X (A) (0.0 = 0.5 * mask radius)[12] : - '10\n',...[12] : - '100.0\n',...2D mask X coordinate (A) [100.0] : - '100.0\n',...2D mask Y coordinate (A) [100.0] : - '100.0\n',...2D mask Z coordinate (A) [100.0] : - '100.0\n',...2D mask radius (A) [100.0] : - '500.0\n',...Defocus search range (A) [500.0] : - '50.0\n',...Defocus step (A) [50.0] : - '1.0\n',...Tuning parameters: padding factor [1.0] : + '300.0\n',...Low resolution limit (A) [300.0] : + '%3.3f\n',...High resolution limit (A) [8.0] : + '0.0\n',...Resolution limit for signed CC (A) (0.0 = max [0.0] : + '0.0\n',...Res limit for classification (A) (0.0 = max) [0.0] : + '0.0\n',...Mask radius for global search (A) (0.0 = max)[100.0] : + '%3.3f\n',...Approx. resolution limit for search (A) [8] : + '0.0\n',...Angular step (0.0 = set automatically) [0.0] : + '20\n',...Number of top hits to refine [20] : + '10\n',...Search range in X (A) (0.0 = 0.5 * mask radius)[12] : + '10\n',...[12] : + '100.0\n',...2D mask X coordinate (A) [100.0] : + '100.0\n',...2D mask Y coordinate (A) [100.0] : + '100.0\n',...2D mask Z coordinate (A) [100.0] : + '100.0\n',...2D mask radius (A) [100.0] : + '500.0\n',...Defocus search range (A) [500.0] : + '50.0\n',...Defocus step (A) [50.0] : + '1.0\n',...Tuning parameters: padding factor [1.0] : 'no\n',...Global search [No] : - 'yes\n',... Local refinement [Yes] : - 'no\n',...Refine Psi [no] : - 'no\n',...Refine Theta [no] : - 'no\n',...Refine Phi [no] : - 'yes\n',...Refine ShiftX [Yes] : - 'yes\n',...Refine ShiftY [Yes] : - 'no\n',...Calculate matching projections [No] : - 'no\n',...Apply 2D masking [No] : - 'no\n',...Refine defocus [No] : - 'yes\n',...Normalize particles [Yes] : - 'no\n',...Invert particle contrast [No] : - 'yes\n',...Exclude images with blank edges [Yes] : - 'yes\n',...Normalize input reconstruction [Yes] : - 'no\n',...Threshold input reconstruction [No] : - '%2.2d\n', ...Max. threads to use for calculation [36] : - ], getenv('EMC_REFINE3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,emc.('PIXEL_SIZE')*10^10, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... - resForFitting,resForFitting,maxThreads); - - fprintf(refineScript, '\neof\n'); - fclose(refineScript); - pause(3); - system(sprintf('chmod a=wrx %s_ref.sh',baseFile)); - system(sprintf('./%s_ref.sh',baseFile)); - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % Reconstruct refined - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - system(sprintf('rm -f %s_rec2.sh',baseFile)); - recScript = fopen(sprintf('%s_rec2.sh',baseFile), 'w'); - fprintf(recScript,[ ... - '#!/bin/bash\n\n', ... + 'yes\n',... Local refinement [Yes] : + 'no\n',...Refine Psi [no] : + 'no\n',...Refine Theta [no] : + 'no\n',...Refine Phi [no] : + 'yes\n',...Refine ShiftX [Yes] : + 'yes\n',...Refine ShiftY [Yes] : + 'no\n',...Calculate matching projections [No] : + 'no\n',...Apply 2D masking [No] : + 'no\n',...Refine defocus [No] : + 'yes\n',...Normalize particles [Yes] : + 'no\n',...Invert particle contrast [No] : + 'yes\n',...Exclude images with blank edges [Yes] : + 'yes\n',...Normalize input reconstruction [Yes] : + 'no\n',...Threshold input reconstruction [No] : + '%2.2d\n', ...Max. threads to use for calculation [36] : + ], getenv('EMC_REFINE3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... + symmetry,emc.('PIXEL_SIZE')*10^10, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... + resForFitting,resForFitting,maxThreads); + +fprintf(refineScript, '\neof\n'); +fclose(refineScript); +pause(3); +system(sprintf('chmod a=wrx %s_ref.sh',baseFile)); +system(sprintf('./%s_ref.sh',baseFile)); + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Reconstruct refined +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +system(sprintf('rm -f %s_rec2.sh',baseFile)); +recScript = fopen(sprintf('%s_rec2.sh',baseFile), 'w'); +fprintf(recScript,[ ... + '#!/bin/bash\n\n', ... '%s << eof\n', ... '%s.mrc\n', ... sprintf('%s.mrc',baseFile) '%s_refined.star\n', ... sprintf('%s.star',baseFile) @@ -849,34 +849,34 @@ '%3.3f\n', ... outermas ang '0.0\n', ... rec res limit '0.0\n', ... ref res limit - '5.0\n', ... Particle weighting factor (A^2) [5.0] - '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] - '1.0\n', ...Tuning parameter: smoothing factor [1.0] : - '1.0\n', ...Tuning parameters: padding factor [1.0] : - 'Yes\n', ...Normalize particles [Yes] : - 'No\n', ...Adjust scores for defocus dependence [no] : - 'No\n', ...Invert particle contrast [No] : - 'Yes\n', ...Exclude images with blank edges [yes] : - 'No\n', ...Crop particle images [no] : - 'Yes\n', ...FSC calculation with even/odd particles [Yes] : - 'No\n', ...Center mass [No] : - 'No\n', ...Apply likelihood blurring [No] : - 'No\n', ...Threshold input reconstruction [No] : - 'No\n', ...Dump intermediate arrays (merge later) [No] : - 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : - 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : - '%2.2d\n', ...Max. threads to use for calculation [36] : + '5.0\n', ... Particle weighting factor (A^2) [5.0] + '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] + '1.0\n', ...Tuning parameter: smoothing factor [1.0] : + '1.0\n', ...Tuning parameters: padding factor [1.0] : + 'Yes\n', ...Normalize particles [Yes] : + 'No\n', ...Adjust scores for defocus dependence [no] : + 'No\n', ...Invert particle contrast [No] : + 'Yes\n', ...Exclude images with blank edges [yes] : + 'No\n', ...Crop particle images [no] : + 'Yes\n', ...FSC calculation with even/odd particles [Yes] : + 'No\n', ...Center mass [No] : + 'No\n', ...Apply likelihood blurring [No] : + 'No\n', ...Threshold input reconstruction [No] : + 'No\n', ...Dump intermediate arrays (merge later) [No] : + 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : + 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%2.2d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'), baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,emc.('PIXEL_SIZE')*10^10, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); - - fprintf(recScript, '\neof\n'); - - fclose(recScript); - pause(2) - system(sprintf('chmod a=wrx %s_rec2.sh',baseFile)); - pause(2) - system(sprintf('./%s_rec2.sh',baseFile)); + symmetry,emc.('PIXEL_SIZE')*10^10, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); + +fprintf(recScript, '\neof\n'); + +fclose(recScript); +pause(2) +system(sprintf('chmod a=wrx %s_rec2.sh',baseFile)); +pause(2) +system(sprintf('./%s_rec2.sh',baseFile)); end diff --git a/testScripts/BH_benchmark.m b/testScripts/BH_benchmark.m index bb2abeaa..f49f99d9 100644 --- a/testScripts/BH_benchmark.m +++ b/testScripts/BH_benchmark.m @@ -3,7 +3,7 @@ % following variable conditions % % Size - <=12 x 128 -% <=6 x 256^3 +% <=6 x 256^3 % <=3 x 384^3 (0.2265 Gb/each) x 3 times mem % % Num processes - MPS has been changing, particularly on volta @@ -12,7 +12,7 @@ % --> I'm not sure the disks available are equivalent, may need to % scale this based on the models % -% Operations - +% Operations - % % Simulate average by: a multiply (mask) % an Interoplation @@ -23,7 +23,7 @@ % FFT --> bandpass, norm % conj(multiply) % IFFT -% A maxVal op +% A maxVal op % This 2x % % @@ -41,14 +41,14 @@ finalContainer = cell(3,1); for memLevel = [0,1,2] - + try parpool(nWorkers); catch delete(gcp('nocreate')) parpool(nWorkers); end - + % For monitoring get existing pids on system, start get new (to kill at the % end [~,existingPIDS] = system('pgrep nvidia-smi'); @@ -57,7 +57,7 @@ [~,updatedPIDS] = system('pgrep nvidia-smi'); updatedPIDS = EMC_str2double(updatedPIDS) - + % Get the newPID if isempty(existingPIDS) dmonPID = updatedPIDS @@ -66,14 +66,14 @@ end clear updatedPIDS existingPIDS - + nTrials = 10; parContainer = cell(nWorkers,1); parfor iProc = 1:nWorkers g = gpuDevice(1); - + % Create the volume to operate on if memLevel == 2 % Create and write to disk @@ -86,16 +86,16 @@ volMem = randn(SIZE.*[1,1,1],'single'); elseif memLevel == 0 volumeData = 0; - volOp = randn(SIZE.*[1,1,1],'single','gpuArray'); + volOp = randn(SIZE.*[1,1,1],'single','gpuArray'); else error('memLevel 0,1,2 gpu,main,disk, not %d',memLevel); end - + % Run the "averaging" benchmark aliTiming = 0; readTiming = 0; - for iTrial = 1:nTrials - + for iTrial = 1:nTrials + tic volOp = []; if ( volumeData ) @@ -106,7 +106,7 @@ volOp = randn(SIZE.*[1,1,1],'single','gpuArray'); end readTiming = readTiming + toc; - + tic bp = BH_bandpass3d(SIZE.*[1,1,1]+2,0,60,4,'GPU',1); volOp = volOp .* (volOp > 1); @@ -121,11 +121,11 @@ elseif memLevel == 1 volOp = gpuArray(volMem); else - volOp = randn(SIZE.*[1,1,1],'single','gpuArray'); - end + volOp = randn(SIZE.*[1,1,1],'single','gpuArray'); + end end - volTmp = BH_resample3d(volOp,randn(1,3).*100,randn(1,3).*5,'Bah','GPU','inv'); + volTmp = BH_resample3d(volOp,randn(1,3).*100,randn(1,3).*5,'Bah','GPU','inv'); volTmp = real(ifftn(fftn(volOp).*conj(fftn(volTmp)))); max(volTmp(:)); volTmp = []; @@ -136,8 +136,8 @@ - - parContainer{iProc} = [readTiming,aliTiming]./nTrials; + + parContainer{iProc} = [readTiming,aliTiming]./nTrials; end % end of parfor loop avgTiming = [0,0]; @@ -149,11 +149,11 @@ [failToKill]=system(sprintf('kill %d',dmonPID)); -% if ( failToKill ) -% finalContainer{memLevel+1}{2} = 'did not kill nvidia-smi dmon'; -% else -% finalContainer{memLevel+1}{2} = 'successfully killed nvidia-smi dmon'; -% end + % if ( failToKill ) + % finalContainer{memLevel+1}{2} = 'did not kill nvidia-smi dmon'; + % else + % finalContainer{memLevel+1}{2} = 'successfully killed nvidia-smi dmon'; + % end end save(sprintf('%s_%d.mat',fileNameOut,nWorkers),'finalContainer'); diff --git a/testScripts/BH_whitenNoiseSpectrum.m b/testScripts/BH_whitenNoiseSpectrum.m index a5e6c300..cd6a35ae 100644 --- a/testScripts/BH_whitenNoiseSpectrum.m +++ b/testScripts/BH_whitenNoiseSpectrum.m @@ -20,9 +20,9 @@ end maskIN = gpuArray(maskIN); elseif numel(pixelSize) > 1 - bandpass = pixelSize; - pixelSize = pixelSize(3); - [~,~,~,maskIN] = BH_localWiener2d( imgIN,bandpass,1,1); + bandpass = pixelSize; + pixelSize = pixelSize(3); + [~,~,~,maskIN] = BH_localWiener2d( imgIN,bandpass,1,1); else maskIN = 0; end @@ -38,11 +38,11 @@ [radialGrid] = BH_multi_gridCoordinates([d1,d2],'Cartesian',... - 'GPU',{'none'},1,0,1); + 'GPU',{'none'},1,0,1); -radialGrid = radialGrid ./ pixelSize; -[gKernel] = BH_multi_gaussian2d(-1.*[d1,d2],3,0); -nMax = 4; +radialGrid = radialGrid ./ pixelSize; +[gKernel] = BH_multi_gaussian2d(-1.*[d1,d2],3,0); +nMax = 4; % Divide bins into roughly equal areas, with min ~ 2 at nyquist oX = ceil((d1+1)/2); radialMax = oX; @@ -56,9 +56,9 @@ n = floor(0.005*oX); idxVect = n; while n < oX -% % % % % idxVect = [idxVect (n+floor((idxVect(i-1).^.40)))]; + % % % % % idxVect = [idxVect (n+floor((idxVect(i-1).^.40)))]; idxVect = [idxVect (n+floor((idxVect(i-1).^.20)))]; - + n = idxVect(i); i = i + 1; end @@ -86,7 +86,7 @@ nShrink = nShrink + 1; end radialSampledAt(nRing) = radialVect(idxVect(iRing+1)); - nRing = nRing +1; + nRing = nRing +1; end @@ -96,20 +96,20 @@ % rFit = fit(gather(double(radialSampledAt)),gather(double(radialAvg)),'spline'); - rFit = csape(gather(double(radialSampledAt)),gather(double(radialAvg)),'clamped'); +rFit = csape(gather(double(radialSampledAt)),gather(double(radialAvg)),'clamped'); % figure, plot(radialVect,fnval(rFit,radialVect)) - r = fnval(rFit,radialGrid).^-1; - r = r - min(r(:)) + 0.001; - rInv = fnval(rFit,radialGrid); - rInv = rInv(1,1:length(radialVect)); - - r = r.^0.5; - - % r = r ./ max(r(:)); - - - +r = fnval(rFit,radialGrid).^-1; +r = r - min(r(:)) + 0.001; +rInv = fnval(rFit,radialGrid); +rInv = rInv(1,1:length(radialVect)); + +r = r.^0.5; + +% r = r ./ max(r(:)); + + + %figure, plot(radialVect,r(1,1:length(radialVect))); %figure, plot(radialVect,rInv(1,1:length(radialVect))); diff --git a/testScripts/CTF.m b/testScripts/CTF.m index 699da72a..30afe803 100644 --- a/testScripts/CTF.m +++ b/testScripts/CTF.m @@ -13,7 +13,7 @@ ctf_img = ''; ctf_abs = ''; ctf_sq = ''; - + end methods @@ -22,26 +22,26 @@ % Detailed explanation goes here obj.pixelSize = pixelSize; obj.useGPU = useGPU; - obj.img_nX = SIZE(1); obj.img_nY = SIZE(2); - if numel(SIZE) == 3 + obj.img_nX = SIZE(1); obj.img_nY = SIZE(2); + if numel(SIZE) == 3 obj.img_nZ = SIZE(3); else obj.img_nZ = 1; end obj.fou_nX = floor(obj.img_nX/2) + 1; - if strcmp(useGPU,'GPU') + if strcmp(useGPU,'GPU') % Maybe add flag for shifted coordinates - [ obj.coord_grids{1},obj.coord_grids{3},~,~,~,~ ] = ... - BH_multi_gridCoordinates( ... - [obj.img_nX,obj.img_nY,obj.img_nZ],... - 'Cylindrical','GPU', {'none'},1,0,0); + [ obj.coord_grids{1},obj.coord_grids{3},~,~,~,~ ] = ... + BH_multi_gridCoordinates( ... + [obj.img_nX,obj.img_nY,obj.img_nZ],... + 'Cylindrical','GPU', {'none'},1,0,0); elseif strcmp(useGPU,'cpu') - % Maybe add flag for shifted coordinates - [ obj.coord_grids{1},obj.coord_grids{3},~,~,~,~ ] = ... - BH_multi_gridCoordinates( ... - [obj.img_nX,obj.img_nY,obj.img_nZ],... - 'Cylindrical','cpu', {'none'},1,0,0); + % Maybe add flag for shifted coordinates + [ obj.coord_grids{1},obj.coord_grids{3},~,~,~,~ ] = ... + BH_multi_gridCoordinates( ... + [obj.img_nX,obj.img_nY,obj.img_nZ],... + 'Cylindrical','cpu', {'none'},1,0,0); else error('useGPU must be GPU or cpu string'); end @@ -58,15 +58,15 @@ % Detailed explanation goes here if nargin > 6 obj.ctf_img = BH_ctfCalc(obj.coord_grids,CS,WL,defocusVector,... - [obj.img_nX,obj.img_nY,obj.img_nZ],... - AMP,DC_SCALE,varargin{1}); + [obj.img_nX,obj.img_nY,obj.img_nZ],... + AMP,DC_SCALE,varargin{1}); else obj.ctf_img = BH_ctfCalc(obj.coord_grids,CS,WL,defocusVector,... - [obj.img_nX,obj.img_nY,obj.img_nZ],... - AMP,DC_SCALE); + [obj.img_nX,obj.img_nY,obj.img_nZ],... + AMP,DC_SCALE); end - - + + end function ctf_product = multiply(obj,otherImg,varargin) @@ -88,14 +88,14 @@ case 'abs' if isempty(obj.ctf_abs) obj.ctf_abs = abs(obj.ctf_img); - end + end ctf_product = obj.ctf_abs .* otherImg; case 'sq' if isempty(obj.ctf_sq) obj.ctf_sq = abs(obj.ctf_img).^2; - end + end ctf_product = obj.ctf_sq .* otherImg; - + otherwise error('form must be none, abs, or sq, not %s\n',form) diff --git a/testScripts/EMC_convn.m b/testScripts/EMC_convn.m index 70471490..866ed6c9 100644 --- a/testScripts/EMC_convn.m +++ b/testScripts/EMC_convn.m @@ -41,7 +41,7 @@ % let convn|conv2 do the checks for IMAGE. if ~isnumeric(IMAGE) || isscalar(IMAGE) error('EMC:IMAGE', 'IMAGE should be a 2d or 3d numeric, got %s of size: %s', ... - class(IMAGE), mat2str(size(IMAGE))) + class(IMAGE), mat2str(size(IMAGE))) elseif ~EMC_sharePrecision(IMAGE, KERNEL) || ~EMC_shareMethod(IMAGE, KERNEL) error('EMC:IMAGE', 'IMAGE should have same precision and method than KERNEL') elseif isrow(KERNEL) diff --git a/testScripts/EMC_is3d.m b/testScripts/EMC_is3d.m index 6f4facbc..0355f94f 100644 --- a/testScripts/EMC_is3d.m +++ b/testScripts/EMC_is3d.m @@ -52,7 +52,7 @@ is3d = false; case 1 error('EMC:SIZE', ['SIZE should have at least 2 elements. To describe vectors, the SIZE should ', ... - 'be [1, N] for row vectors or [N, 1] for column vector.']); + 'be [1, N] for row vectors or [N, 1] for column vector.']); otherwise error('EMC:SIZE', 'SIZE has more element than maximum supported (3), got %d', ndim); end diff --git a/testScripts/EMC_maximizeSNR.m b/testScripts/EMC_maximizeSNR.m index a718aa4c..7474d7a0 100644 --- a/testScripts/EMC_maximizeSNR.m +++ b/testScripts/EMC_maximizeSNR.m @@ -9,10 +9,10 @@ pixelSize = 0.5/fscParams{4}(end); wienerThreshold = zeros(2,1,'single','gpuArray'); -for iGold = 1:2 +for iGold = 1:2 [wgt{iGold}, wienerThreshold(iGold)] = BH_multi_cRef_wgtCritical(gpuArray(wgt{iGold}{iRef})); - % TODO This is always even so this should be safe. - wgt{iGold} = wgt{iGold}([bhF.halfDimSize:size(wgt{iGold},1),1],:,:); + % TODO This is always even so this should be safe. + wgt{iGold} = wgt{iGold}([bhF.halfDimSize:size(wgt{iGold},1),1],:,:); end LSQ = real(bhF.invFFT(ref{1} ./ ( bhF.swapIndexINV(wgt{1}) + wienerThreshold(1)),2)); @@ -22,212 +22,212 @@ MASKCORE = find(MASKCORE > 0.98); cccLSQ = (LSQ(MASKCORE) - mean(LSQ(MASKCORE))) ./ std(LSQ(MASKCORE)); - - - radialGrid = BH_multi_gridCoordinates(bhF.inputSize,'Cartesian','GPU', ... - {'none'},1,0,1,{'halfGrid'}); - radialGrid = radialGrid ./ pixelSize; - - bFactor = 0; - - [ fsc3D, avgCTF ] = calc_anisoFSC(fscParams, radialGrid, wgt, bFactor, pixelSize, bhF); - - if any(bFactor) - [ bFactor, bandFilter ] = calc_bfact(fscParams, radialGrid,flgReference, bFactor, noForceMask,highPassFilter,pixelSize); - else - bFactor = {1}; - end - - for iGold = 1:2 - wgt{iGold} = bhF.swapIndexINV(wgt{iGold}); - end - - iBfact = 1; - snrWeight = 0.5; - search1 = 10 .^ [-4:0]; - ccc = 0.*search1; - n=1; - for fPfM = search1 - [ weightedImg ] = gather(apply_weights((fsc3D), ... - (avgCTF{2}), ... - (radialGrid < 0.5./pixelSize),... - (ref{2}), ... - wgt{2},... - fPfM, bFactor{iBfact}, ... - snrWeight,bhF)); - - cccVal = cccLSQ .* (weightedImg(MASKCORE) - mean(weightedImg(MASKCORE))) ./ std(weightedImg(MASKCORE)); - ccc(n) = gather(sum(cccVal(:)) ./ numel(MASKCORE)); - n = n + 1; - end - - [m,c] = max(ccc) - ccc - search2 = search1(c-1):search1(c-1):search1(c+1); - ccc = 0.*search2; - n=1; - for fPfM = search2 - [ weightedImg ] = gather(apply_weights((fsc3D), ... - (avgCTF{2}), ... - (radialGrid < 0.5./pixelSize),... - (ref{2}), ... - wgt{2},... - fPfM, bFactor{iBfact}, ... - snrWeight,bhF)); - - cccVal = cccLSQ .* (weightedImg(MASKCORE) - mean(weightedImg(MASKCORE))) ./ std(weightedImg(MASKCORE)); - ccc(n) = gather(sum(cccVal(:)) ./ numel(MASKCORE)); - n = n + 1; - end - - ccc - [m,c] = max(ccc) - fParticle = search2(c) - error('asdf') -end +radialGrid = BH_multi_gridCoordinates(bhF.inputSize,'Cartesian','GPU', ... + {'none'},1,0,1,{'halfGrid'}); +radialGrid = radialGrid ./ pixelSize; -function [ fsc3D, weights ] = calc_anisoFSC(fscParams, radialGrid, weights, bFactor, pixelSize, bhF) +bFactor = 0; - nCones = fscParams{7}; - if (nCones) - firstCone = 1; - else - firstCone = 0; - end +[ fsc3D, avgCTF ] = calc_anisoFSC(fscParams, radialGrid, wgt, bFactor, pixelSize, bhF); - coneList = fscParams{8}; - halfAngle = fscParams{9}; - samplingRate = fscParams{10}; +if any(bFactor) + [ bFactor, bandFilter ] = calc_bfact(fscParams, radialGrid,flgReference, bFactor, noForceMask,highPassFilter,pixelSize); +else + bFactor = {1}; +end - osX = fscParams{4}; +for iGold = 1:2 + wgt{iGold} = bhF.swapIndexINV(wgt{iGold}); +end - radialBinary = (radialGrid < 0.5./pixelSize); - - mtfX = 0:0.5/(length(osX)-1):0.5; - - - coneMask = zeros(size(radialBinary) ,'single','gpuArray'); - fsc3D = zeros(size(radialBinary), 'single','gpuArray'); - - bin = floor(size(weights{1},1)/1); - inc = 0.5 / (bin*pixelSize); - for iCone = firstCone:nCones - fprintf('assembling the 3D FSC from cone %d/%d\n',iCone,nCones); +iBfact = 1; +snrWeight = 0.5; +search1 = 10 .^ [-4:0]; +ccc = 0.*search1; +n=1; +for fPfM = search1 + [ weightedImg ] = gather(apply_weights((fsc3D), ... + (avgCTF{2}), ... + (radialGrid < 0.5./pixelSize),... + (ref{2}), ... + wgt{2},... + fPfM, bFactor{iBfact}, ... + snrWeight,bhF)); + + cccVal = cccLSQ .* (weightedImg(MASKCORE) - mean(weightedImg(MASKCORE))) ./ std(weightedImg(MASKCORE)); + ccc(n) = gather(sum(cccVal(:)) ./ numel(MASKCORE)); + n = n + 1; +end - iFSCfit = csape(fscParams{1}(:,1),fscParams{2}(:,iCone+1),'variational'); +[m,c] = max(ccc) +ccc +search2 = search1(c-1):search1(c-1):search1(c+1); +ccc = 0.*search2; +n=1; +for fPfM = search2 + [ weightedImg ] = gather(apply_weights((fsc3D), ... + (avgCTF{2}), ... + (radialGrid < 0.5./pixelSize),... + (ref{2}), ... + wgt{2},... + fPfM, bFactor{iBfact}, ... + snrWeight,bhF)); + + cccVal = cccLSQ .* (weightedImg(MASKCORE) - mean(weightedImg(MASKCORE))) ./ std(weightedImg(MASKCORE)); + ccc(n) = gather(sum(cccVal(:)) ./ numel(MASKCORE)); + n = n + 1; +end - - % Don't lowpass in-case of FSC calculation - if any(bFactor) - iForceMask = fscParams{6}{iCone+1}; - else - iForceMask = 1; - end - - iFSCclean = csape(osX,fnval(iFSCfit,osX).*iForceMask,'variational'); - - - - if ( nCones ) - coneOrientation = BH_defineMatrix(coneList{iCone},'Bah','invVector'); - [ radius,~,height,~,~,~ ] = ... - BH_multi_gridCoordinates( bhF.inputSize, ... - 'Cylindrical', ... - 'GPU', ... - {'single',... - coneOrientation,... - [0,0,0]', ... - 'invVector',... - 1,1},... - 0, 0, 0, {'halfGrid'}); - iConeMask = (rad2deg(atan2(radius,abs(height))) < halfAngle{iCone}); - - - coneMask = coneMask + iConeMask; - else - iConeMask = radialBinary; - coneMask = ones(size(radialBinary), 'single', 'gpuArray'); - end - - - - iConeMask = iConeMask & radialBinary; - iFSC = iConeMask; - iFSC(iConeMask) = fnval(iFSCclean,radialGrid(iConeMask)); - - fsc3D = fsc3D + iFSC; - - - - clear iFSC iConeMask radius height +ccc +[m,c] = max(ccc) +fParticle = search2(c) +error('asdf') +end +function [ fsc3D, weights ] = calc_anisoFSC(fscParams, radialGrid, weights, bFactor, pixelSize, bhF) - end +nCones = fscParams{7}; +if (nCones) + firstCone = 1; +else + firstCone = 0; +end +coneList = fscParams{8}; +halfAngle = fscParams{9}; +samplingRate = fscParams{10}; +osX = fscParams{4}; +radialBinary = (radialGrid < 0.5./pixelSize); - fsc3D = fsc3D./coneMask; - fsc3D(fsc3D < 10^-10) = 10^-10; +mtfX = 0:0.5/(length(osX)-1):0.5; - fsc3D = bhF.swapIndexFWD(fsc3D); - kernelDim = 5; - padDim = ceil(kernelDim/2)+1; +coneMask = zeros(size(radialBinary) ,'single','gpuArray'); +fsc3D = zeros(size(radialBinary), 'single','gpuArray'); + +bin = floor(size(weights{1},1)/1); +inc = 0.5 / (bin*pixelSize); + +for iCone = firstCone:nCones + fprintf('assembling the 3D FSC from cone %d/%d\n',iCone,nCones); - [ LIMITS ] = EMC_limits(size(fsc3D), size(fsc3D)+padDim, {}); - [ fsc3D ] = EMC_resize(fsc3D, LIMITS, {'taper',false;'value',nan}); + iFSCfit = csape(fscParams{1}(:,1),fscParams{2}(:,iCone+1),'variational'); - for iGold = 1:2 - % The weights start out centered so no need to fwdSwap - weights{iGold} = EMC_resize(weights{iGold}, LIMITS, {'taper',false}); + + % Don't lowpass in-case of FSC calculation + if any(bFactor) + iForceMask = fscParams{6}{iCone+1}; + else + iForceMask = 1; end - % Set the padded region to something close by. It may be better to think - % more about this - m = isnan(fsc3D(:)); - fsc3D(m) = 0; - dilationKernel = zeros([1,kernelDim],'single','gpuArray'); - dilationKernel(1) = 1.0; - tmp = EMC_convn(fsc3D, dilationKernel); - fsc3D(m) = fsc3D(m) + tmp(m); + iFSCclean = csape(osX,fnval(iFSCfit,osX).*iForceMask,'variational'); - for iGold = 1:2 - tmp = EMC_convn(weights{iGold}, dilationKernel); - weights{iGold}(m) = weights{iGold}(m) + tmp(m); + + + if ( nCones ) + coneOrientation = BH_defineMatrix(coneList{iCone},'Bah','invVector'); + [ radius,~,height,~,~,~ ] = ... + BH_multi_gridCoordinates( bhF.inputSize, ... + 'Cylindrical', ... + 'GPU', ... + {'single',... + coneOrientation,... + [0,0,0]', ... + 'invVector',... + 1,1},... + 0, 0, 0, {'halfGrid'}); + iConeMask = (rad2deg(atan2(radius,abs(height))) < halfAngle{iCone}); + + + coneMask = coneMask + iConeMask; + else + iConeMask = radialBinary; + coneMask = ones(size(radialBinary), 'single', 'gpuArray'); end - clear tmp m dilationKernel + + + + iConeMask = iConeMask & radialBinary; + iFSC = iConeMask; + iFSC(iConeMask) = fnval(iFSCclean,radialGrid(iConeMask)); + + fsc3D = fsc3D + iFSC; + + + + clear iFSC iConeMask radius height + + + +end - for i = 1.5:-0.5:0.5 - KERNEL = EMC_gaussianKernel([1,kernelDim], i, 'gpu', {}); - fsc3D = EMC_convn(single(fsc3D), KERNEL); - for iGold = 1:2 - weights{iGold} = EMC_convn(weights{iGold},KERNEL); - end - end - [ LIMITS ] = EMC_limits(size(fsc3D), size(fsc3D)-padDim, {}); - [ fsc3D ] = EMC_resize(fsc3D, LIMITS, {'taper',false}); - fsc3D = bhF.swapIndexINV(fsc3D); +fsc3D = fsc3D./coneMask; +fsc3D(fsc3D < 10^-10) = 10^-10; + + +fsc3D = bhF.swapIndexFWD(fsc3D); +kernelDim = 5; +padDim = ceil(kernelDim/2)+1; + +[ LIMITS ] = EMC_limits(size(fsc3D), size(fsc3D)+padDim, {}); +[ fsc3D ] = EMC_resize(fsc3D, LIMITS, {'taper',false;'value',nan}); + +for iGold = 1:2 + % The weights start out centered so no need to fwdSwap + weights{iGold} = EMC_resize(weights{iGold}, LIMITS, {'taper',false}); +end + +% Set the padded region to something close by. It may be better to think +% more about this +m = isnan(fsc3D(:)); +fsc3D(m) = 0; +dilationKernel = zeros([1,kernelDim],'single','gpuArray'); +dilationKernel(1) = 1.0; +tmp = EMC_convn(fsc3D, dilationKernel); +fsc3D(m) = fsc3D(m) + tmp(m); + +for iGold = 1:2 + tmp = EMC_convn(weights{iGold}, dilationKernel); + weights{iGold}(m) = weights{iGold}(m) + tmp(m); +end +clear tmp m dilationKernel + + +for i = 1.5:-0.5:0.5 + KERNEL = EMC_gaussianKernel([1,kernelDim], i, 'gpu', {}); + fsc3D = EMC_convn(single(fsc3D), KERNEL); for iGold = 1:2 - [ weights{iGold} ] = EMC_resize(weights{iGold}, LIMITS, {'taper',false}); - weights{iGold} = bhF.swapIndexINV(weights{iGold}); - end + weights{iGold} = EMC_convn(weights{iGold},KERNEL); + end +end +[ LIMITS ] = EMC_limits(size(fsc3D), size(fsc3D)-padDim, {}); +[ fsc3D ] = EMC_resize(fsc3D, LIMITS, {'taper',false}); +fsc3D = bhF.swapIndexINV(fsc3D); -% -% clear radialBinary coneMask + +for iGold = 1:2 + [ weights{iGold} ] = EMC_resize(weights{iGold}, LIMITS, {'taper',false}); + weights{iGold} = bhF.swapIndexINV(weights{iGold}); +end + +% +% clear radialBinary coneMask end function [ weightedImg ] = apply_weights(anisoFSC, avgCTF, nyquistLimit, img, ... - wgt, fPfM, bFactor, snrWeight,bhF) - + wgt, fPfM, bFactor, snrWeight,bhF) + % bFactor contains any sharpening, and/or forced cutoffs @@ -237,8 +237,8 @@ weightedImg = real(bhF.invFFT(img .* weight,2)); - - + + end diff --git a/testScripts/EMC_setPrecision.m b/testScripts/EMC_setPrecision.m index 025d6058..60d7b6ea 100644 --- a/testScripts/EMC_setPrecision.m +++ b/testScripts/EMC_setPrecision.m @@ -25,7 +25,7 @@ NUM = uint64(NUM); otherwise error('EMC:precision', ["PRECISION should be 'single', 'double', 'logical', ", ... - "'int16', 'int32', 'int64', 'uint16', 'uint32' or 'uint64'"]) + "'int16', 'int32', 'int64', 'uint16', 'uint32' or 'uint64'"]) end end diff --git a/testScripts/ellipsoid_fit.m b/testScripts/ellipsoid_fit.m index 9cc59046..990b36fa 100644 --- a/testScripts/ellipsoid_fit.m +++ b/testScripts/ellipsoid_fit.m @@ -22,9 +22,9 @@ % * center - ellispoid or other conic center coordinates [xc; yc; zc] % * radii - ellipsoid or other conic radii [a; b; c] % * evecs - the radii directions as columns of the 3x3 matrix -% * v - the 10 parameters describing the ellipsoid / conic algebraically: +% * v - the 10 parameters describing the ellipsoid / conic algebraically: % Ax^2 + By^2 + Cz^2 + 2Dxy + 2Exz + 2Fyz + 2Gx + 2Hy + 2Iz + J = 0 -% * chi2 - residual sum of squared errors (chi^2), this chi2 is in the +% * chi2 - residual sum of squared errors (chi^2), this chi2 is in the % coordinate frame in which the ellipsoid is a unit sphere. % % Author: @@ -37,7 +37,7 @@ if nargin == 1 equals = ''; % no constraints by default end - + if size( X, 2 ) ~= 3 error( 'Input data must have three columns!' ); else @@ -47,30 +47,30 @@ end % need nine or more data points -if length( x ) < 9 - switch length(x) - case 8 - fprintf('WARNING: Only 8 subTomos, reducing fit to a unique ellipsoid with two equal radii\n'); - equals = 'xy'; - case 7 - fprintf('WARNING: Only 7 subTomos, reducing fit to a unique oriented ellipsoid\n'); - equals = '0'; - case 6 - fprintf('WARNING: Only 6 subTomos, reducing fit to a unique oriented ellipsoid\n'); - equals = '0'; - case 5 - fprintf('WARNING: Only 5 subTomos, reducing fit to a unique oriented ellipsoid with two equal radii\n'); - equals = '0xy'; - case 4 - fprintf('WARNING: Only 4 subTomos, reducing fit to a unique unique sphere\n'); - equals = 'xyz'; - otherwise - error('There are less than 4 points, no fitting is possible!'); - end +if length( x ) < 9 + switch length(x) + case 8 + fprintf('WARNING: Only 8 subTomos, reducing fit to a unique ellipsoid with two equal radii\n'); + equals = 'xy'; + case 7 + fprintf('WARNING: Only 7 subTomos, reducing fit to a unique oriented ellipsoid\n'); + equals = '0'; + case 6 + fprintf('WARNING: Only 6 subTomos, reducing fit to a unique oriented ellipsoid\n'); + equals = '0'; + case 5 + fprintf('WARNING: Only 5 subTomos, reducing fit to a unique oriented ellipsoid with two equal radii\n'); + equals = '0xy'; + case 4 + fprintf('WARNING: Only 4 subTomos, reducing fit to a unique unique sphere\n'); + equals = 'xyz'; + otherwise + error('There are less than 4 points, no fitting is possible!'); + end end % % need nine or more data points -% if length( x ) < 9 && strcmp( equals, '' ) +% if length( x ) < 9 && strcmp( equals, '' ) % error( 'Must have at least 9 points to fit a unique ellipsoid' ); % end % if length( x ) < 8 && ( strcmp( equals, 'xy' ) || strcmp( equals, 'xz' ) ) @@ -120,31 +120,31 @@ % fit ellipsoid in the form Ax^2 + By^2 + Cz^2 + 2Gx + 2Hy + 2Iz = 1 elseif strcmp( equals, '0' ) D = [ x .* x + y .* y - 2 * z .* z, ... - x .* x + z .* z - 2 * y .* y, ... - 2 * x, ... - 2 * y, ... - 2 * z, ... - 1 + 0 * x ]; % ndatapoints x 6 ellipsoid parameters + x .* x + z .* z - 2 * y .* y, ... + 2 * x, ... + 2 * y, ... + 2 * z, ... + 1 + 0 * x ]; % ndatapoints x 6 ellipsoid parameters % fit ellipsoid in the form Ax^2 + By^2 + Cz^2 + 2Gx + 2Hy + 2Iz = 1, % where A = B or B = C or A = C elseif strcmp( equals, '0xy' ) D = [ x .* x + y .* y - 2 * z .* z, ... - 2 * x, ... - 2 * y, ... - 2 * z, ... - 1 + 0 * x ]; % ndatapoints x 5 ellipsoid parameters + 2 * x, ... + 2 * y, ... + 2 * z, ... + 1 + 0 * x ]; % ndatapoints x 5 ellipsoid parameters elseif strcmp( equals, '0xz' ) D = [ x .* x + z .* z - 2 * y .* y, ... - 2 * x, ... - 2 * y, ... - 2 * z, ... - 1 + 0 * x ]; % ndatapoints x 5 ellipsoid parameters - % fit sphere in the form A(x^2 + y^2 + z^2) + 2Gx + 2Hy + 2Iz = 1 + 2 * x, ... + 2 * y, ... + 2 * z, ... + 1 + 0 * x ]; % ndatapoints x 5 ellipsoid parameters + % fit sphere in the form A(x^2 + y^2 + z^2) + 2Gx + 2Hy + 2Iz = 1 elseif strcmp( equals, 'xyz' ) D = [ 2 * x, ... - 2 * y, ... - 2 * z, ... - 1 + 0 * x ]; % ndatapoints x 4 ellipsoid parameters + 2 * y, ... + 2 * z, ... + 1 + 0 * x ]; % ndatapoints x 4 ellipsoid parameters else error( [ 'Unknown parameter value ' equals '!' ] ); end @@ -195,9 +195,9 @@ % form the algebraic form of the ellipsoid A = [ v(1) v(4) v(5) v(7); ... - v(4) v(2) v(6) v(8); ... - v(5) v(6) v(3) v(9); ... - v(7) v(8) v(9) v(10) ]; + v(4) v(2) v(6) v(8); ... + v(5) v(6) v(3) v(9); ... + v(7) v(8) v(9) v(10) ]; % find the center of the ellipsoid center = -A( 1:3, 1:3 ) \ v( 7:9 ); % form the corresponding translation matrix @@ -212,7 +212,7 @@ radii = radii .* sgns; % calculate difference of the fitted points from the actual data normalized by the conic radii -d = [ x - center(1), y - center(2), z - center(3) ]; +d = [ x - center(1), y - center(2), z - center(3) ]; d = d * evecs; % rotate to cardinal axes of the conic; d = [ d(:,1) / radii(1), d(:,2) / radii(2), d(:,3) / radii(3) ]; % normalize to the conic radii chi2 = sum( abs( 1 - sum( d.^2 .* repmat( sgns', size( d, 1 ), 1 ), 2 ) ) ); diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index d93eb3cf..d468f6c7 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -99,8 +99,8 @@ fprintf('\t\t***************************************\n\n'); % Get rid of the shorthead passed in by the emClarity script. if isdeployed - varargin = varargin(2:end); - nArgs = nArgs - 1; + varargin = varargin(2:end); + nArgs = nArgs - 1; end emcHelp = false; @@ -132,7 +132,7 @@ else myErr = sprintf('\n\n\tRun with help for a list of functions\n\n'); error(myErr); -% checkHelp = 0; + % checkHelp = 0; end @@ -160,7 +160,7 @@ multiGPUs = 0; case 'cleanTemplateSearch' multiGPUs = 0; - + otherwise emc = emC_testParse(varargin{2}); end @@ -176,7 +176,7 @@ fprintf('\nThere are more gpus visible than requested, selecting the largest memory devices\n'); select_gpus(nGPUs_wanted,nGPUs_visible,cmdIN); else - fprintf('\nThe number of gpus requested matches those visible to the system\n'); + fprintf('\nThe number of gpus requested matches those visible to the system\n'); end end end @@ -184,37 +184,37 @@ switch varargin{1} case 'help' fprintf(['\nAvailable commands (case sensitive):\n\n',... - '\nhelp - this message\n',... - '\n\t\t for more details, emClarity help\n',... - '\ncheck - system check for dependencies\n',... - '\ninit - create a new project from template matching results.\n',... - '\nautoAlign - align tilt-serie\n',... - '\navg - average subtomograms\n',... - '\nfsc - calculate the fsc\n',... - '\nmask - create a mask\n',... - '\nbenchmark - run a benchmark\n',... - '\ncalcWeights - calculate the weights for a given cycle\n',... - '\nalignRaw - align one or more references against individual subtomograms.\n',... - '\npca - reduce dimensionality prior to clustering, possibly on smaller subset of data.\n',... - '\ncluster - use one of a number of approaches to sort populations.\n',... - '\nskip - after avering classes & possible removing some, skip to next cycle.\n',... - '\ngeometry - edit or analyze the experimental metadata.\n',... - '\ncombineProjects - combine two or more projects together', ... - '\nctf - estimate, correct, or refine the CTF.\n',... - '\ntomoCPR - tomogram constrained projection refinement\n',... - '\ntemplateSearch - template matching/ global search\n',... - '\ncleanTemplateSearch - clean search results based on neighbor constraints\n',... - '\nrescale - change the mag on a volume\n',... - '\nreconstruct - reconstruct a volume from a set of subtomograms\n',... - '\nremoveDuplicates - remove subtomos that have migrated to the same position\n',... - '\nexperimental - experimental options\n',... - '\nremoveNeighbors - clean templateSearch results based on lattice constraints\n']); - -% Currently disabled options. Multi-reference alignment -% % % '\nalignRef - align one or more references against a primary ref\n',... -% % % ' optionally add multiple instances aligned to each\n',... -% % % ' to its respective reference.\n',... -% % % '\nalignCls - align refs from alignRef to a usually much larger number of class averages.\n',... + '\nhelp - this message\n',... + '\n\t\t for more details, emClarity help\n',... + '\ncheck - system check for dependencies\n',... + '\ninit - create a new project from template matching results.\n',... + '\nautoAlign - align tilt-serie\n',... + '\navg - average subtomograms\n',... + '\nfsc - calculate the fsc\n',... + '\nmask - create a mask\n',... + '\nbenchmark - run a benchmark\n',... + '\ncalcWeights - calculate the weights for a given cycle\n',... + '\nalignRaw - align one or more references against individual subtomograms.\n',... + '\npca - reduce dimensionality prior to clustering, possibly on smaller subset of data.\n',... + '\ncluster - use one of a number of approaches to sort populations.\n',... + '\nskip - after avering classes & possible removing some, skip to next cycle.\n',... + '\ngeometry - edit or analyze the experimental metadata.\n',... + '\ncombineProjects - combine two or more projects together', ... + '\nctf - estimate, correct, or refine the CTF.\n',... + '\ntomoCPR - tomogram constrained projection refinement\n',... + '\ntemplateSearch - template matching/ global search\n',... + '\ncleanTemplateSearch - clean search results based on neighbor constraints\n',... + '\nrescale - change the mag on a volume\n',... + '\nreconstruct - reconstruct a volume from a set of subtomograms\n',... + '\nremoveDuplicates - remove subtomos that have migrated to the same position\n',... + '\nexperimental - experimental options\n',... + '\nremoveNeighbors - clean templateSearch results based on lattice constraints\n']); + + % Currently disabled options. Multi-reference alignment + % % % '\nalignRef - align one or more references against a primary ref\n',... + % % % ' optionally add multiple instances aligned to each\n',... + % % % ' to its respective reference.\n',... + % % % '\nalignCls - align refs from alignRef to a usually much larger number of class averages.\n',... case 'experimental' print_experimental_options(); @@ -226,7 +226,7 @@ end case 'init' if emcProgramHelp || ... - length(varargin) < 2 && length(varargin)> 5 + length(varargin) < 2 && length(varargin)> 5 fprintf(['\nUsage: emClarity init param.m [tomoCpr iter, for continuing after second globalsearch]\n']); elseif length(varargin) == 5 emC_testParse(varargin{2}) @@ -240,10 +240,10 @@ else emC_testParse(varargin{2}) BH_geometryInitialize(varargin{2}); - end + end case 'removeNeighbors' if emcProgramHelp || ... - length(varargin) ~= 6 + length(varargin) ~= 6 fprintf(['\nUsage: emCLarity removeNeighbors pixelSize CYCLE distanceCutoff (Ang) angleCutoff (Deg) N-neighbors\n']); else %emC_testParse(varargin{2}) @@ -252,14 +252,14 @@ else BH_geometry_Constraints(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6},varargin{7}); end - end + end - case 'autoAlign' - if emcProgramHelp || ... - (length(varargin) ~= 5 && length(varargin) ~= 6) + case 'autoAlign' + if emcProgramHelp || ... + (length(varargin) ~= 5 && length(varargin) ~= 6) fprintf(['\nUsage: emClarity autoAlign param.m stackName tiltFile tilt-axis Rotation\n']); - else - emC_testParse(varargin{2}) + else + emC_testParse(varargin{2}) if ~exist(varargin{4}, 'file') fprintf('Did not find your .rawtlt file %s\n',varargin{3}); error('Expecting tiltName.st tiltName.rawtlt pixelSize (Ang) imageRotation (degrees)'); @@ -268,26 +268,26 @@ fprintf('Did not find your .st file %s\n',varargin{2}); error('Expecting tiltName.st tiltName.rawtlt pixelSize (Ang) imageRotation (degrees)'); end - + if length(varargin) == 5 - BH_runAutoAlign(varargin{2},getenv('EMC_AUTOALIGN'),getenv('EMC_FINDBEADS'),varargin{3},varargin{4},varargin{5}); + BH_runAutoAlign(varargin{2},getenv('EMC_AUTOALIGN'),getenv('EMC_FINDBEADS'),varargin{3},varargin{4},varargin{5}); else - BH_runAutoAlign(varargin{2},getenv('EMC_AUTOALIGN'),getenv('EMC_FINDBEADS'),varargin{3},varargin{4},varargin{5},varargin{6}); + BH_runAutoAlign(varargin{2},getenv('EMC_AUTOALIGN'),getenv('EMC_FINDBEADS'),varargin{3},varargin{4},varargin{5},varargin{6}); end - + return - end + end case 'skip' if emcProgramHelp || ... - length(varargin) ~= 3 + length(varargin) ~= 3 fprintf(['\nUsage: emClarity skip param.m iter\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}) BH_skipClassAlignment(varargin{2},varargin{3},'RawAlignment','1'); end case 'rescale' if emcProgramHelp || ... - length(varargin) ~= 6 + length(varargin) ~= 6 fprintf(['\nUsage: emClarity rescale fileNameIN fileNameOut angPixIN angPixOut cpu/GPU\n']); else mag = EMC_str2double(varargin{4})/EMC_str2double(varargin{5}); @@ -295,46 +295,46 @@ end case 'mask' if emcProgramHelp || ... - (~ismember(length(varargin),[5,8,9])) - length(varargin) + (~ismember(length(varargin),[5,8,9])) + length(varargin) fprintf(['\nFor geometric mask:\n', ... - 'fileNameOUT.mrc, pixelSize (Ang), Shape (sphere,cylinder,rectangle), Size/radius/center in pixels: [nX,nY,nZ], [rX,rY,rZ], [cX,cY,cZ], optional: "2d"',... - '\n\nFor a shape based mask\n', ... - 'fileNameIN.mrc,fileNameOUT.mrc, pixelSize (Ang)\n']); - + 'fileNameOUT.mrc, pixelSize (Ang), Shape (sphere,cylinder,rectangle), Size/radius/center in pixels: [nX,nY,nZ], [rX,rY,rZ], [cX,cY,cZ], optional: "2d"',... + '\n\nFor a shape based mask\n', ... + 'fileNameIN.mrc,fileNameOUT.mrc, pixelSize (Ang)\n']); + else - switch length(varargin) + switch length(varargin) case 5 maskVol = getVolume(MRCImage(varargin{3})); pixelSize = EMC_str2double(varargin{5}); maskVol = BH_mask3d(maskVol,EMC_str2double(varargin{5}),'',''); SAVE_IMG(MRCImage(gather(maskVol)),varargin{4},pixelSize); case 8 - pixelSize = EMC_str2double(varargin{4}); + pixelSize = EMC_str2double(varargin{4}); maskVol = BH_mask3d(varargin{5},EMC_str2double(varargin{6}), ... - EMC_str2double(varargin{7}), ... - EMC_str2double(varargin{8})); + EMC_str2double(varargin{7}), ... + EMC_str2double(varargin{8})); SAVE_IMG(MRCImage(gather(maskVol)),varargin{3},pixelSize); case 9 - pixelSize = EMC_str2double(varargin{4}); + pixelSize = EMC_str2double(varargin{4}); maskVol = BH_mask3d(varargin{5},EMC_str2double(varargin{6}), ... - EMC_str2double(varargin{7}), ... - EMC_str2double(varargin{8}), ... - EMC_str2double(varargin{9})); + EMC_str2double(varargin{7}), ... + EMC_str2double(varargin{8}), ... + EMC_str2double(varargin{9})); SAVE_IMG(MRCImage(gather(maskVol)),varargin{3},pixelSize); end - end + end case 'benchmark' if emcProgramHelp || ... - length(varargin) ~= 4 + length(varargin) ~= 4 fprintf(['\nUsage: emClarity benchmark fileNameOut fastScratchDisk nWorkers']); else BH_benchmark(varargin{2},varargin{3},varargin{4}); - end + end case 'calcWeights' if emcProgramHelp || ... - length(varargin) ~= 6 + length(varargin) ~= 6 fprintf('%f\n',length(varargin)) fprintf(['\nUsage: emClarity calcWeights param.m cycle prefixOUT symmetry [gpuIDX, tiltStart, tiltStop]\n']); else @@ -343,26 +343,26 @@ end case 'avg' if emcProgramHelp || ... - length(varargin) ~= 4 + length(varargin) ~= 4 fprintf(['\nUsage: emClarity avg\n',... - 'param.m\n',... - 'cycle number\n',... - 'stage of alignment\n',... - ' raw (post raw alignment)\n',... - ' cluster_cls (post classification)\n']); + 'param.m\n',... + 'cycle number\n',... + 'stage of alignment\n',... + ' raw (post raw alignment)\n',... + ' cluster_cls (post classification)\n']); else - emC_testParse(varargin{2}) - BH_average3d(varargin{2}, varargin{3}, varargin{4}); - end + emC_testParse(varargin{2}) + BH_average3d(varargin{2}, varargin{3}, varargin{4}); + end case 'fsc' if emcProgramHelp || ... - (length(varargin) ~= 4 && length(varargin) ~= 6) + (length(varargin) ~= 4 && length(varargin) ~= 6) fprintf(['\nUsage: emClarity fsc\n',... - 'param.m\n',... - 'cycle number\n',... - 'stage of alignment\n',... - ' raw (post raw alignment)\n',... - ' cluster_cls (post classification)\n']); + 'param.m\n',... + 'cycle number\n',... + 'stage of alignment\n',... + ' raw (post raw alignment)\n',... + ' cluster_cls (post classification)\n']); elseif length(varargin) == 4 emC_testParse(varargin{2}) BH_fscGold_class(varargin{2}, varargin{3}, varargin{4}); @@ -371,142 +371,142 @@ end case 'alignRaw' if emcProgramHelp || ... - (length(varargin) ~= 3 && length(varargin) ~= 4) - fprintf(['\nUsage: emClarity alignRaw\n',... - 'param.m\n',... - 'cycle number\n',... - '[experimental option 1/2/3, 1 - abs(ccc),2 - weighted,3 -abs(weighted)]']); + (length(varargin) ~= 3 && length(varargin) ~= 4) + fprintf(['\nUsage: emClarity alignRaw\n',... + 'param.m\n',... + 'cycle number\n',... + '[experimental option 1/2/3, 1 - abs(ccc),2 - weighted,3 -abs(weighted)]']); else - emC_testParse(varargin{2}) - if length(varargin) == 3 - BH_alignRaw3d_v2(varargin{2}, varargin{3}); - else - % Switching to v2 always, 1.5.0.9 20200520 - BH_alignRaw3d_v2(varargin{2},varargin{3}, varargin{4}); - end + emC_testParse(varargin{2}) + if length(varargin) == 3 + BH_alignRaw3d_v2(varargin{2}, varargin{3}); + else + % Switching to v2 always, 1.5.0.9 20200520 + BH_alignRaw3d_v2(varargin{2},varargin{3}, varargin{4}); + end end case 'alignRef' if emcProgramHelp || ... - length(varargin) ~= 3 - fprintf(['\nparam.m\n',... - 'cycle number\n',... - 'stage of alignment\n']); + length(varargin) ~= 3 + fprintf(['\nparam.m\n',... + 'cycle number\n',... + 'stage of alignment\n']); else - emC_testParse(varargin{2}) - BH_alignReferences3d(varargin{2}, varargin{3}); - end + emC_testParse(varargin{2}) + BH_alignReferences3d(varargin{2}, varargin{3}); + end case 'alignCls' if emcProgramHelp || ... - length(varargin) ~= 3 - fprintf(['\nparam.m\n',... - 'cycle number\n',... - 'stage of alignment\n']); + length(varargin) ~= 3 + fprintf(['\nparam.m\n',... + 'cycle number\n',... + 'stage of alignment\n']); else - emC_testParse(varargin{2}) - BH_alignClassRotAvg3d(varargin{2}, varargin{3}); - end -% case 'alignFrames' -% if emcProgramHelp || ... -% (length(varargin) < 9 && length(varargin) > 11) -% fprintf(['\nnameIN\n',... -% 'nameOUT\n',... -% 'gpuIDX\n',... -% 'FPN\n',... -% 'pixelSizeIN\n',... -% 'pixelSizeOUT\n',... -% 'overSampleBy\n',... -% 'doLocal [particleRadiusAng, maxRes]\n']); -% else -% switch length(varargin) -% case 9 -% BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... -% EMC_str2double(varargin{5}),... -% EMC_str2double(varargin{6}),... -% EMC_str2double(varargin{7}),... -% EMC_str2double(varargin{8}),... -% EMC_str2double(varargin{9})); -% case 10 -% BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... -% EMC_str2double(varargin{5}),... -% EMC_str2double(varargin{6}),... -% EMC_str2double(varargin{7}),... -% EMC_str2double(varargin{8}),... -% EMC_str2double(varargin{9}),... -% EMC_str2double(varargin{10})); -% case 11 -% BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... -% EMC_str2double(varargin{5}),... -% EMC_str2double(varargin{6}),... -% EMC_str2double(varargin{7}),... -% EMC_str2double(varargin{8}),... -% EMC_str2double(varargin{9}),... -% EMC_str2double(varargin{10}),... -% EMC_str2double(varargin{11})); -% end -% end + emC_testParse(varargin{2}) + BH_alignClassRotAvg3d(varargin{2}, varargin{3}); + end + % case 'alignFrames' + % if emcProgramHelp || ... + % (length(varargin) < 9 && length(varargin) > 11) + % fprintf(['\nnameIN\n',... + % 'nameOUT\n',... + % 'gpuIDX\n',... + % 'FPN\n',... + % 'pixelSizeIN\n',... + % 'pixelSizeOUT\n',... + % 'overSampleBy\n',... + % 'doLocal [particleRadiusAng, maxRes]\n']); + % else + % switch length(varargin) + % case 9 + % BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... + % EMC_str2double(varargin{5}),... + % EMC_str2double(varargin{6}),... + % EMC_str2double(varargin{7}),... + % EMC_str2double(varargin{8}),... + % EMC_str2double(varargin{9})); + % case 10 + % BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... + % EMC_str2double(varargin{5}),... + % EMC_str2double(varargin{6}),... + % EMC_str2double(varargin{7}),... + % EMC_str2double(varargin{8}),... + % EMC_str2double(varargin{9}),... + % EMC_str2double(varargin{10})); + % case 11 + % BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... + % EMC_str2double(varargin{5}),... + % EMC_str2double(varargin{6}),... + % EMC_str2double(varargin{7}),... + % EMC_str2double(varargin{8}),... + % EMC_str2double(varargin{9}),... + % EMC_str2double(varargin{10}),... + % EMC_str2double(varargin{11})); + % end + % end case 'pca' if emcProgramHelp || ... - length(varargin) ~= 4 && length(varargin) ~= 5 - fprintf(['\nparam.m\n',... - 'cycle number\n',... - 'randomSubset\n']); - % 'use focused mask\n',... - % ' 1 from standard devation\n',... - % ' 2 from variance\n',... - % ' 3 user supplied (not recommended)\n']); + length(varargin) ~= 4 && length(varargin) ~= 5 + fprintf(['\nparam.m\n',... + 'cycle number\n',... + 'randomSubset\n']); + % 'use focused mask\n',... + % ' 1 from standard devation\n',... + % ' 2 from variance\n',... + % ' 3 user supplied (not recommended)\n']); else - emC_testParse(varargin{2}) - - if (EMC_str2double(varargin{4})) - % project onto full set - BH_pcaPub(varargin{2}, varargin{3}, '1'); + emC_testParse(varargin{2}) + + if (EMC_str2double(varargin{4})) + % project onto full set + BH_pcaPub(varargin{2}, varargin{3}, '1'); + else + + if length(varargin) == 5 + maskVal = EMC_str2double(varargin{5}); else - - if length(varargin) == 5 - maskVal = EMC_str2double(varargin{5}); - else - maskVal = 0; - end - - if (maskVal) - % re-run on full or randomsubset now using variance or stddev mask - BH_pcaPub(varargin{2}, varargin{3}, sprintf('%d',-1.*maskVal)) - else - BH_pcaPub(varargin{2}, varargin{3}, '0') - end - + maskVal = 0; + end - - end - - end + if (maskVal) + % re-run on full or randomsubset now using variance or stddev mask + BH_pcaPub(varargin{2}, varargin{3}, sprintf('%d',-1.*maskVal)) + else + BH_pcaPub(varargin{2}, varargin{3}, '0') + end + + + + end + + end case 'cluster' if emcProgramHelp || ... - length(varargin) ~= 3 - fprintf(['\nparam.m\n',... - 'cycle number\n']); + length(varargin) ~= 3 + fprintf(['\nparam.m\n',... + 'cycle number\n']); else - emC_testParse(varargin{2}) - BH_clusterPub(varargin{2}, varargin{3}); - end + emC_testParse(varargin{2}) + BH_clusterPub(varargin{2}, varargin{3}); + end case 'ctf' - if emcProgramHelp + if emcProgramHelp fprintf(['\nestimate\n',... - ' param.m tiltBaseName\n',... - '\nrefine\n',... - ' param.m tiltBaseName gpuIDX\n',... - '\nupdate\n',... - ' param.m tiltBaseName (full,refine,update)\n',... - '\ncorrect\n',... - ' param.m precision usuable-area nWorkers\n',... - ' (single,double) [nx,ny,nz] \n',... - '\n3d\n',... - ' param.m [/local/Scratch]\n']); + ' param.m tiltBaseName\n',... + '\nrefine\n',... + ' param.m tiltBaseName gpuIDX\n',... + '\nupdate\n',... + ' param.m tiltBaseName (full,refine,update)\n',... + '\ncorrect\n',... + ' param.m precision usuable-area nWorkers\n',... + ' (single,double) [nx,ny,nz] \n',... + '\n3d\n',... + ' param.m [/local/Scratch]\n']); else - emC_testParse(varargin{3}) - + emC_testParse(varargin{3}) + switch varargin{2} case 'estimate' if (useV2) @@ -515,7 +515,7 @@ BH_ctf_Estimate_2(varargin{3},varargin{4}); else BH_ctf_Estimate_2(varargin{3},varargin{4},varargin{5}); - end + end else if nArgs == 4 BH_ctf_Estimate(varargin{3},varargin{4}); @@ -526,144 +526,144 @@ case 'refine' if length(varargin) ~= 4 error('You need to specify a parameter file and tilt name') - end + end BH_ctf_Refine2(varargin{3},varargin{4}); case 'update' if length(varargin) > 3 error('\n\nYou now only need to specify %s parameter file.\n\n','the') - end + end BH_ctf_Updatefft(varargin{3},'-1','full'); case 'correct' BH_ctf_Correct(varargin{3},varargin{4},varargin{5},varargin{6},varargin{7}); case '3d' if nArgs == 6 % last is a dummy, used for tomoCPR background - BH_ctf_Correct3d(varargin{3},varargin{4},varargin{5},varargin{6}); + BH_ctf_Correct3d(varargin{3},varargin{4},varargin{5},varargin{6}); elseif nArgs == 5 - % Not a public option, start from tilt # (of nTilts) - BH_ctf_Correct3d(varargin{3},varargin{4},varargin{5}); + % Not a public option, start from tilt # (of nTilts) + BH_ctf_Correct3d(varargin{3},varargin{4},varargin{5}); elseif nArgs == 4 BH_ctf_Correct3d(varargin{3},varargin{4}); else BH_ctf_Correct3d(varargin{3}); end - + otherwise error('ctf operations are estimate,refine,update,correct, or 3d.'); end - end + end case 'tomoCPR' fprintf('In tomoCPR the MCR is %s\n',getenv('MCR_CACHE_ROOT')); - + if emcProgramHelp || ... - ( length(varargin) < 3 || length(varargin) > 4 ) + ( length(varargin) < 3 || length(varargin) > 4 ) fprintf(['\nparam.m\n',... - 'cycle number\n',... - 'nTiltStart\n']); + 'cycle number\n',... + 'nTiltStart\n']); else - emC_testParse(varargin{2}) - if length(varargin) == 4 - tiltStart = EMC_str2double(varargin{4}); - else - tiltStart = 1; - end - BH_synthetic_mapBack(varargin{2}, varargin{3},tiltStart); - end + emC_testParse(varargin{2}) + if length(varargin) == 4 + tiltStart = EMC_str2double(varargin{4}); + else + tiltStart = 1; + end + BH_synthetic_mapBack(varargin{2}, varargin{3},tiltStart); + end case 'removeDuplicates' if emcProgramHelp || ... - length(varargin) ~= 3 + length(varargin) ~= 3 fprintf(['\nparam.m\n',... - 'cycle number\n',... - ]); + 'cycle number\n',... + ]); else - emC_testParse(varargin{2}) - BH_removeDuplicates(varargin{2}, varargin{3} ); - end + emC_testParse(varargin{2}) + BH_removeDuplicates(varargin{2}, varargin{3} ); + end case 'geometry' if emcProgramHelp || ... - length(varargin) ~= 7 + length(varargin) ~= 7 fprintf(['\nparam.m\n',... - 'cycle number\n',... - 'stage of alignment\n',... - 'operation []\n',... - ' SwitchCurrentCycle, UpdateTilts, WriteCsv, RemoveClasses,\n'... - ' ShiftAll, ShiftBin, ListTomos, RemoveTomos,\n',... - ' ListPercentiles, RemoveFraction, RandomizeEulers\n',... - 'vectOP [0,0,0]\n',... - 'STD, EVE, ODD\n']); + 'cycle number\n',... + 'stage of alignment\n',... + 'operation []\n',... + ' SwitchCurrentCycle, UpdateTilts, WriteCsv, RemoveClasses,\n'... + ' ShiftAll, ShiftBin, ListTomos, RemoveTomos,\n',... + ' ListPercentiles, RemoveFraction, RandomizeEulers\n',... + 'vectOP [0,0,0]\n',... + 'STD, EVE, ODD\n']); else - emC_testParse(varargin{2}) - BH_geometryAnalysis(varargin{2}, varargin{3},varargin{4}, ... - varargin{5}, varargin{6},varargin{7}); - end + emC_testParse(varargin{2}) + BH_geometryAnalysis(varargin{2}, varargin{3},varargin{4}, ... + varargin{5}, varargin{6},varargin{7}); + end case 'combineProjects' BH_combineProjects(varargin{1},varargin(2:end)); - + case 'templateSearch' if emcProgramHelp || ... - ~ismember(length(varargin),[7,8]) - fprintf(['\nparam.m\n',... - 'tomoName\n',... - 'tomoNumber\n', ... - 'template name\n',... - 'symmetry\n',... - '[threshold override]\n',... - 'gpuIDX.\n']); + ~ismember(length(varargin),[7,8]) + fprintf(['\nparam.m\n',... + 'tomoName\n',... + 'tomoNumber\n', ... + 'template name\n',... + 'symmetry\n',... + '[threshold override]\n',... + 'gpuIDX.\n']); else wedgeType = 2; - switch length(varargin) + switch length(varargin) case 7 if (useV1) BH_templateSearch3d( varargin{2}, varargin{3},varargin{4}, ... - varargin{5}, varargin{6},wedgeType,varargin{7}); + varargin{5}, varargin{6},wedgeType,varargin{7}); else BH_templateSearch3d_2( varargin{2}, varargin{3},varargin{4}, ... - varargin{5}, varargin{6},wedgeType,varargin{7}); + varargin{5}, varargin{6},wedgeType,varargin{7}); end - + case 8 - if (useV1) - + if (useV1) + BH_templateSearch3d( varargin{2}, varargin{3},varargin{4}, ... - varargin{5}, varargin{6},wedgeType, ... - varargin{7},varargin{8}); + varargin{5}, varargin{6},wedgeType, ... + varargin{7},varargin{8}); else BH_templateSearch3d_2( varargin{2}, varargin{3},varargin{4}, ... - varargin{5}, varargin{6},wedgeType, ... - varargin{7},varargin{8}); + varargin{5}, varargin{6},wedgeType, ... + varargin{7},varargin{8}); end - + end - + end case 'cleanTemplateSearch' - if emcProgramHelp || ... - length(varargin) ~= 5 - fprintf(['\npixelSize (Ang)\n',... - 'distance to neightbor (Ang)\n',... - 'angular deviation to neighbor (degrees)\n', ... - 'min number neighbors (one less than expected is usually good)\n']); - else - + if emcProgramHelp || ... + length(varargin) ~= 5 + fprintf(['\npixelSize (Ang)\n',... + 'distance to neightbor (Ang)\n',... + 'angular deviation to neighbor (degrees)\n', ... + 'min number neighbors (one less than expected is usually good)\n']); + else + BH_geometry_Constraints(EMC_str2double(varargin{2}), '0', varargin{3}, varargin{4}, varargin{5}); - - end - + + end + case 'reconstruct' - if emcProgramHelp || ... - length(varargin) ~= 6 - fprintf(['paramterfile\n',... - 'cycle #\n',... - 'output prefix\n', ... - 'symmetry (C1)\n',... - 'max exposure (e/A^2)\n']); - else - BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}); - end + if emcProgramHelp || ... + length(varargin) ~= 6 + fprintf(['paramterfile\n',... + 'cycle #\n',... + 'output prefix\n', ... + 'symmetry (C1)\n',... + 'max exposure (e/A^2)\n']); + else + BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}); + end otherwise error('command --%s-- not recognized. Try "help" for a list.', varargin{1}) end @@ -691,17 +691,17 @@ emc = BH_parseParameterFile( paramTest ); - - - + + + %%%%%%%% GLOBALS %%%%%%%%%%%%%%%%%%%% - + % These variables are to maintain some flexibility for parameters that have % an ill-defined dependence on experimental factors. Preferably only until % they can be resolved. - + global bh_global_window_cutoff; - try + try bh_global_window_cutoff = emc.('windowCutoff'); catch bh_global_window_cutoff = -2; @@ -709,12 +709,12 @@ % These are for making shape based masks. I think the problem is likely % dependent on the current resolution of the sub-tomogram, and that a - % single set of values will not work for everything. - + % single set of values will not work for everything. + % Note that these must also be declared in the relevant functions %%%%%%% BH_mask3d.m %%%%%%% - global bh_global_binary_mask_low_pass; + global bh_global_binary_mask_low_pass; global bh_global_binary_mask_threshold; %%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -725,22 +725,22 @@ %%%%%%% Anything that reads geometry. This way if size changes, its okay. %%%%%%% Needed only for tracking multiple copies of a single particle. %%%%%%% (Currently not used) - global bh_global_nCol + global bh_global_nCol bh_global_nCol = 26; %%%%%%% - + %%%%%% BH_mask3d - affects then the FSC calc global bh_global_vol_est_scaling; global bh_global_MTF; - %%%%% + %%%%% global bh_global_fast_scratch_disk; global bh_global_ram_disk; %%%%%%% BH_ctfCorrect_3d %%%%%%% Wiener filter and cut off past this point global bh_global_turn_on_phase_plate; - + try bh_global_turn_on_phase_plate = emc.('phakePhasePlate'); catch @@ -767,33 +767,33 @@ global bh_global_imodProjectionShifts; bh_global_imodProjectionShifts = [ -0.5, -0.5, 0.5 ; -0.5, -0.5, 0; 0.5,0.5,1.0 ]; -%%%%%%%%%%%%%% - + %%%%%%%%%%%%%% + %%%%% For profiling global bh_global_do_profile; - try + try bh_global_do_profile = emc.('doProfile'); catch bh_global_do_profile = false; end - - + + try bh_global_fast_scratch_disk = emc.('fastScratchDisk'); catch bh_global_fast_scratch_disk=''; end - - + + try bh_global_ram_disk = emc.('ramDisk'); catch bh_global_ram_disk = '/dev/shm'; end - + testFileName = sprintf('%s/thisEmCDiskCheck123456.txt',bh_global_ram_disk); - [writeError] = system(sprintf('echo a > %s',testFileName)); + [writeError] = system(sprintf('echo a > %s',testFileName)); if (writeError) fprintf('\nRan into an error trying to write to the fastScatchDisk %s\n',bh_global_ram_disk); bh_global_ram_disk = ''; @@ -801,9 +801,9 @@ fprintf('Found and using your ramDisk\n'); system(sprintf('rm %s',testFileName)); end - - try + + try bh_global_binary_mask_low_pass = emc.('setMaskLowPass'); catch % These seem to be okay for higher-resolution data (EMPIAR ribo sets) @@ -821,7 +821,7 @@ catch bh_global_binary_pcaMask_threshold = 0.5; end - + global bh_global_kFactorScaling; try bh_global_kFactorScaling = emc.('kFactorScaling'); @@ -829,7 +829,7 @@ bh_global_kFactorScaling = 1.0; end - + try bh_global_vol_est_scaling = emc.('setParticleVolumeScaling'); @@ -846,22 +846,22 @@ try % 0 - off, 2 original (matches closely measured MTF), 1 stronger - % Anthing else, float, iX = scalar, dX = cap val e.g. + % Anthing else, float, iX = scalar, dX = cap val e.g. % opiton 1 100.04 and 2 (default) is 25.06 bh_global_MTF = emc.('mtfVal'); catch bh_global_MTF = 2; end - + global bh_global_print_shifts_in_particle_basis; - try + try bh_global_print_shifts_in_particle_basis = emc.('printShiftsInParticleBasis'); catch bh_global_print_shifts_in_particle_basis = true; end global bh_global_zero_lag_score; - try + try bh_global_zero_lag_score = emc.('useZeroLagScore'); catch bh_global_zero_lag_score = false; @@ -881,9 +881,9 @@ end fprintf('nExpGlobals %2.2f maskLP, %2.2f maskThr, %2.2f pcaMaskThr\n', ... - bh_global_binary_mask_low_pass, ... - bh_global_binary_mask_threshold, ... - bh_global_binary_pcaMask_threshold); + bh_global_binary_mask_low_pass, ... + bh_global_binary_mask_threshold, ... + bh_global_binary_pcaMask_threshold); catch error('error parsing parameter file %s\n', paramTest) end @@ -911,7 +911,7 @@ devList = ''; for iGPU = 1:nGPUs_total if iGPU <= nGPUs_wanted - devList = strcat(devList,uuidlist{memList(iGPU,1)}); + devList = strcat(devList,uuidlist{memList(iGPU,1)}); if iGPU < nGPUs_wanted devList = strcat(devList,','); elseif iGPU == nGPUs_wanted && nGPUs_total > nGPUs_wanted @@ -936,28 +936,28 @@ function print_experimental_options - fprintf('\n\n\tExperimental Options: use at your own RISK\n'); - fprintf('(\t\tOr better yet, check with ben!\t\t)\n'); - fprintf('\nIf you do use/change any of these, please mention in your methods and EMDB entry!\n'); - fprintf('\n\n----------------------------------\n\n'); - fprintf('\nscaleCalcSize\toversampling of vol for xcorr. Def:\t1.5\n'); - fprintf('\npaddedSize\tpadded size of tiles in ctf estimateion\n'); - - fprintf('\nflgFscShapeMask\t default 1\n'); - fprintf('\nflgPcaShapeMask\t default 1\n'); - - fprintf('\nflgQualityWeight\t Downweight high-freq of low scoring sub-tomos. Def:\t4\n'); - fprintf('\ninterpOrder\t Linear interpolation (1) Spline/Fourier (4 - not working, do not use)\n'); - fprintf('\nflgLimitToOneProcess\t For OOM issues in averaging. Boolean Def:\t0\n'); - fprintf('\nflgCenterRefCOM\tShift reference to center of mass. Boolean Def:\t1\n'); - fprintf('\nconserveDiskSpace\n'); - fprintf('\nPca_distMeasure\tMeasure for difference. euclidean, cityblock, correlation, cosine Def:\t sqeuclidean\n'); - fprintf('\nPca_nReplicates\tThe number of times Kmeans is intialized. Def:\t 128\n'); - fprintf('\nflgSymmetrizeSubTomos\tApply symmetry to subtomos in alignment.\nCurrently not worse, not better, but much slower.\n'); - - fprintf('\ndeltaZTolerance\tallowed defocus variance in ctf estimation:Def:\t100e-9\n'); - fprintf('\nzShift\tselect tiles with a defocus offset. Determine tilt gradient.\n\n'); - +fprintf('\n\n\tExperimental Options: use at your own RISK\n'); +fprintf('(\t\tOr better yet, check with ben!\t\t)\n'); +fprintf('\nIf you do use/change any of these, please mention in your methods and EMDB entry!\n'); +fprintf('\n\n----------------------------------\n\n'); +fprintf('\nscaleCalcSize\toversampling of vol for xcorr. Def:\t1.5\n'); +fprintf('\npaddedSize\tpadded size of tiles in ctf estimateion\n'); + +fprintf('\nflgFscShapeMask\t default 1\n'); +fprintf('\nflgPcaShapeMask\t default 1\n'); + +fprintf('\nflgQualityWeight\t Downweight high-freq of low scoring sub-tomos. Def:\t4\n'); +fprintf('\ninterpOrder\t Linear interpolation (1) Spline/Fourier (4 - not working, do not use)\n'); +fprintf('\nflgLimitToOneProcess\t For OOM issues in averaging. Boolean Def:\t0\n'); +fprintf('\nflgCenterRefCOM\tShift reference to center of mass. Boolean Def:\t1\n'); +fprintf('\nconserveDiskSpace\n'); +fprintf('\nPca_distMeasure\tMeasure for difference. euclidean, cityblock, correlation, cosine Def:\t sqeuclidean\n'); +fprintf('\nPca_nReplicates\tThe number of times Kmeans is intialized. Def:\t 128\n'); +fprintf('\nflgSymmetrizeSubTomos\tApply symmetry to subtomos in alignment.\nCurrently not worse, not better, but much slower.\n'); + +fprintf('\ndeltaZTolerance\tallowed defocus variance in ctf estimation:Def:\t100e-9\n'); +fprintf('\nzShift\tselect tiles with a defocus offset. Determine tilt gradient.\n\n'); + end % end of print experimental options diff --git a/testScripts/fourierCtfRec.m b/testScripts/fourierCtfRec.m index 6e7aa523..4d4415f7 100644 --- a/testScripts/fourierCtfRec.m +++ b/testScripts/fourierCtfRec.m @@ -1,133 +1,133 @@ function [ reconstruction ] = fourierCtfRec(wantedSize, positionList, TLT, reconGeometry, originPrj, varargin) - % % % % - % For testing override inputs - % % % % - flgTesting = false; - - +% % % % +% For testing override inputs +% % % % +flgTesting = false; + + % Insert a check forces a cubic dimension during reconstruction, followed - % by rescaling - d1 = wantedSize(1); - d2 = wantedSize(2); - d3 = wantedSize(3); - - samplingRate = 1; - - exposureWeight = 0; - if nargin == 6 - iThickness = 75; - samplingRate = varargin{1}; - fractionOfDose = TLT(:,14)/mean(TLT(:,14)); - fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(:,4))*400 )); - doHalfGrid = 1; - centerGrid = 1; - [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); - for iPrj = 1:size(exposureWeight,3) - exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)) .* ... - (fractionOfDose(iPrj).*(fractionOfElastics(iPrj))); - end +% by rescaling +d1 = wantedSize(1); +d2 = wantedSize(2); +d3 = wantedSize(3); + +samplingRate = 1; + +exposureWeight = 0; +if nargin == 6 + iThickness = 75; + samplingRate = varargin{1}; + fractionOfDose = TLT(:,14)/mean(TLT(:,14)); + fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(:,4))*400 )); + doHalfGrid = 1; + centerGrid = 1; + [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); + for iPrj = 1:size(exposureWeight,3) + exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)) .* ... + (fractionOfDose(iPrj).*(fractionOfElastics(iPrj))); end +end - if (d1 ~= d2 || d1 ~= d3) - error('fix the cubic check ya twit'); - end +if (d1 ~= d2 || d1 ~= d3) + error('fix the cubic check ya twit'); +end - if ( flgTesting ) - % These will just go in the object properties - rawtlt = -60:3:60; - nTilts = length(rawtlt); - TLT = zeros(nTilts,23,'single','gpuArray'); - TLT(:,4) = rawtlt; - TLT(:,1) = 1:length(rawtlt); - % 12,13 = astigmatism = 0 - % def,pix,cs,wl,ampcont - TLT(:,[15,16,17,18,19]) = repmat([-1.2e-6,2.0e-10,2.7e-3,1.969e-12,1e-1],nTilts,1); - reconGeometry = wantedSize; - reconShift = [0,0,0]; - originPrj = wantedSize; - originPrj = ceil((originPrj+1)./2); - originPrj(3) = 1; - % Position list is just for the wanted subTomo - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - - % This matrix would rotate a vector into the particles reference frame, - % we want to rotate back, so take the transpose to invert. - particleRotMat = BH_defineMatrix([0,0,0],'Bah','forward'); - prjVector = [0,0,0] - originVol + reconShift + [0.0,0.0,1.0]; - else - % These will just go in the object properties - reconShift = reconGeometry(2,:); - originPrj = ceil((originPrj+1)./2); - originPrj(3) = 1; - % Position list is just for the wanted subTomo - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - - - % This matrix would rotate a vector into the particles reference frame, - % we want to rotate back, so take the transpose to invert. - particleRotMat = reshape(positionList(17:25),3,3)'; - - % Taken from synthetic mapback, not sure why I'm adding the 1 here. - prjVector = positionList(11:13) - originVol + reconShift + [0.0,0.0,1.0]; - % We want the origin of the subTomogram in the tilted image to calculate - % a defocus offset. - end +if ( flgTesting ) + % These will just go in the object properties + rawtlt = -60:3:60; + nTilts = length(rawtlt); + TLT = zeros(nTilts,23,'single','gpuArray'); + TLT(:,4) = rawtlt; + TLT(:,1) = 1:length(rawtlt); + % 12,13 = astigmatism = 0 + % def,pix,cs,wl,ampcont + TLT(:,[15,16,17,18,19]) = repmat([-1.2e-6,2.0e-10,2.7e-3,1.969e-12,1e-1],nTilts,1); + reconGeometry = wantedSize; + reconShift = [0,0,0]; + originPrj = wantedSize; + originPrj = ceil((originPrj+1)./2); + originPrj(3) = 1; + % Position list is just for the wanted subTomo + originVol = ceil((reconGeometry(1,1:3)+1)./2); + lowerLeftVol = originPrj+reconShift-originVol; + + % This matrix would rotate a vector into the particles reference frame, + % we want to rotate back, so take the transpose to invert. + particleRotMat = BH_defineMatrix([0,0,0],'Bah','forward'); + prjVector = [0,0,0] - originVol + reconShift + [0.0,0.0,1.0]; +else + % These will just go in the object properties + reconShift = reconGeometry(2,:); + originPrj = ceil((originPrj+1)./2); + originPrj(3) = 1; + % Position list is just for the wanted subTomo + originVol = ceil((reconGeometry(1,1:3)+1)./2); + lowerLeftVol = originPrj+reconShift-originVol; + + + % This matrix would rotate a vector into the particles reference frame, + % we want to rotate back, so take the transpose to invert. + particleRotMat = reshape(positionList(17:25),3,3)'; + % Taken from synthetic mapback, not sure why I'm adding the 1 here. + prjVector = positionList(11:13) - originVol + reconShift + [0.0,0.0,1.0]; + % We want the origin of the subTomogram in the tilted image to calculate + % a defocus offset. +end - - oX = floor(d1/2)+1; - oY = floor(d2/2)+1; - oZ = floor(d3/2)+1; - spreadVal = [1]; - %spreadVal = [0.4904 0.3733 0.1363] - nOffCenter = length(spreadVal) - 1; - % Replicate the taper - spreadVal = [flip(spreadVal(2:end)),spreadVal]; +oX = floor(d1/2)+1; +oY = floor(d2/2)+1; +oZ = floor(d3/2)+1; - [X,Y] = BH_multi_gridCoordinates([d1,d2],'Cartesian',... - 'GPU',{'none'},0,1,0,{'halfgrid'}); - - [rad,phi] = BH_multi_gridCoordinates([d1,d2],'Cylindrical',... - 'GPU',{'none'},1,1,0,{'halfgrid'}); +spreadVal = [1]; +%spreadVal = [0.4904 0.3733 0.1363] +nOffCenter = length(spreadVal) - 1; - % Assuming the pixel size is constant. - rad = {rad(:) ./ (samplingRate.*TLT(1,16)),1,phi(:)}; - clear phi +% Replicate the taper +spreadVal = [flip(spreadVal(2:end)),spreadVal]; - spatialToPixel = 0.5; - - X = X(:) + spatialToPixel; - Y = Y(:) + spatialToPixel; +[X,Y] = BH_multi_gridCoordinates([d1,d2],'Cartesian',... + 'GPU',{'none'},0,1,0,{'halfgrid'}); + +[rad,phi] = BH_multi_gridCoordinates([d1,d2],'Cylindrical',... + 'GPU',{'none'},1,1,0,{'halfgrid'}); + +% Assuming the pixel size is constant. +rad = {rad(:) ./ (samplingRate.*TLT(1,16)),1,phi(:)}; +clear phi - reconstruction = zeros([oX,d2,d3],'single','gpuArray'); +spatialToPixel = 0.5; - weights = reconstruction; +X = X(:) + spatialToPixel; +Y = Y(:) + spatialToPixel; + +reconstruction = zeros([oX,d2,d3],'single','gpuArray'); + +weights = reconstruction; for iPrj = 1:size(TLT,1) - + % Need a defocus offset based on XYZ position in the tomogram rTilt = BH_defineMatrix([0,TLT(iPrj,4),0],'SPIDER','forwardVector'); prjCoords = rTilt * prjVector'; - + iDefocusOffset = prjCoords(3).*(TLT(iPrj,16)); iDefocus = [iDefocusOffset + TLT(iPrj,15) - TLT(iPrj,12), ... - iDefocusOffset + TLT(iPrj,15) + TLT(iPrj,12), ... - TLT(iPrj,13)]; - - - - + iDefocusOffset + TLT(iPrj,15) + TLT(iPrj,12), ... + TLT(iPrj,13)]; + + + + iCTF = (BH_ctfCalc(rad,TLT(iPrj,17),TLT(iPrj,18),iDefocus,[oX,d2],TLT(iPrj,19),-1)).^2; if (length(exposureWeight) > 1) iCTF = iCTF(:).*reshape(exposureWeight(:,:,TLT(iPrj,1)),oX.*d2,1); @@ -137,10 +137,10 @@ % The sample is rotated by tiltA, which is like rotating the beam by % -tiltA. We want to insert in the plane perpendicular to the beam. - xform = particleRotMat *rTilt; - + xform = particleRotMat *rTilt; + for iOffset = 1:nOffCenter*2+1 - + iZ = iOffset - nOffCenter - 1 + spatialToPixel; Xnew = X.*xform(1) + Y.*xform(4) + (iZ*xform(7)) ; Ynew = X.*xform(2) + Y.*xform(5) + (iZ*xform(8)) ; @@ -159,68 +159,68 @@ Xnew = Xnew + 1 ; Ynew = Ynew + oY ; Znew = Znew + oZ ; - + for dZ = [0:1] zCoord = (floor(Znew)+dZ); - z_inBounds = (zCoord <= d3 & zCoord > 0); + z_inBounds = (zCoord <= d3 & zCoord > 0); weight_z = abs(1.0 - abs(Znew-zCoord)); - + for dY = [0:1] yCoord = (floor(Ynew)+dY); weight_yz = abs(1.0 - abs(Ynew-yCoord)) .* weight_z; - + y_inBounds = (z_inBounds & yCoord <= d2 & yCoord > 0); - + for dX = [0:1] - + xCoord = (floor(Xnew)+dX); weight_xyz = weight_yz .* (spreadVal(iOffset) * abs(1.0 - abs(Xnew-xCoord))) ; - x_inBounds = ( y_inBounds & xCoord <= oX & xCoord > 0 ); - + x_inBounds = ( y_inBounds & xCoord <= oX & xCoord > 0 ); + linearIDX = sub2ind([oX,d2,d3],xCoord(x_inBounds), ... - yCoord(x_inBounds), ... - zCoord(x_inBounds)); - + yCoord(x_inBounds), ... + zCoord(x_inBounds)); + weight_xyz = weight_xyz(x_inBounds); - + xCTF = weight_xyz.*iCTF(x_inBounds); - + toKeep = true(length(linearIDX),1,'gpuArray'); - - - % Currently ~ 2.7s for a 257^3 - half that time is in this block. - % Unique is the biggest killer, then assignments out. - % If all parts of loop could be assigned to a single output grid which - % is already unique, this could save most of this time. + + + % Currently ~ 2.7s for a 257^3 - half that time is in this block. + % Unique is the biggest killer, then assignments out. + % If all parts of loop could be assigned to a single output grid which + % is already unique, this could save most of this time. while ~isempty(linearIDX) [~,ai] = unique(linearIDX); ci = linearIDX(ai); - + reconstruction(ci) = reconstruction(ci) + xCTF(ai); weights(ci) = weights(ci) + weight_xyz(ai); - + toKeep(ai) = false; linearIDX = linearIDX(toKeep); - weight_xyz = weight_xyz(toKeep); + weight_xyz = weight_xyz(toKeep); xCTF = xCTF(toKeep); toKeep = toKeep(toKeep); end - - - + + + end end - end + end end % end of offset loop - - + + end % TODO these should be set up top, and possible outside. @@ -232,10 +232,10 @@ % 'GPU','single'); % weights = BH_padZeros3d(weights,gaussDiam.*[1,1,1], ... % gaussDiam.*[1,1,1], ... -% 'GPU','single'); -% % Mirror along the origin -% reconstruction(1:gaussDiam,:,:) = reconstruction(2*gaussDiam:-1:gaussDiam+1,:,size(reconstruction,3):-1:1); -% weights(1:gaussDiam,:,:) = weights(2*gaussDiam:-1:gaussDiam+1,:,size(weights,3):-1:1); +% 'GPU','single'); +% % Mirror along the origin +% reconstruction(1:gaussDiam,:,:) = reconstruction(2*gaussDiam:-1:gaussDiam+1,:,size(reconstruction,3):-1:1); +% weights(1:gaussDiam,:,:) = weights(2*gaussDiam:-1:gaussDiam+1,:,size(weights,3):-1:1); reconstruction = convn(reconstruction,g,'same')./convn((weights+.01),g,'same'); % % Trim the fat diff --git a/testScripts/fourierCtfRecTex.m b/testScripts/fourierCtfRecTex.m index 2d238b3d..b113f003 100644 --- a/testScripts/fourierCtfRecTex.m +++ b/testScripts/fourierCtfRecTex.m @@ -1,209 +1,209 @@ function [ reconstruction ] = fourierCtfRecTex(wantedSize, positionList, TLT, reconGeometry, originPrj, varargin) - % % % % - % For testing override inputs - % % % % +% % % % +% For testing override inputs +% % % % + +if length(varargin) > 1 + flgTesting = varargin{1,2}; +else + flgTesting = false; +end - if length(varargin) > 1 - flgTesting = varargin{1,2}; - else - flgTesting = false; - end - - -% Insert a check forces a cubic dimension during reconstruction, followed - % by rescaling - d1 = wantedSize(1); - d2 = wantedSize(2); - d3 = wantedSize(3); - - samplingRate = 1; - exposureWeight = 0; - if nargin == 6 && ~flgTesting - iThickness = 75; - samplingRate = varargin{1}; - fractionOfDose = TLT(:,14)./mean(TLT(:,14)); - fractionOfElastics = exp(-1.*iThickness./( cosd(TLT(:,4)).*400 )); - fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics); - doHalfGrid = 1; - centerGrid = 1; - [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); - for iPrj = 1:size(exposureWeight,3) - exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)).^2 .* ... - (fractionOfDose(iPrj).*(fractionOfElastics(iPrj))); - end +% Insert a check forces a cubic dimension during reconstruction, followed +% by rescaling +d1 = wantedSize(1); +d2 = wantedSize(2); +d3 = wantedSize(3); + +samplingRate = 1; + +exposureWeight = 0; +if nargin == 6 && ~flgTesting + iThickness = 75; + samplingRate = varargin{1}; + fractionOfDose = TLT(:,14)./mean(TLT(:,14)); + fractionOfElastics = exp(-1.*iThickness./( cosd(TLT(:,4)).*400 )); + fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics); + doHalfGrid = 1; + centerGrid = 1; + [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); + for iPrj = 1:size(exposureWeight,3) + exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)).^2 .* ... + (fractionOfDose(iPrj).*(fractionOfElastics(iPrj))); end +end - - if (d1 ~= d2 || d1 ~= d3) - error('fix the cubic check ya twit'); - end +if (d1 ~= d2 || d1 ~= d3) + error('fix the cubic check ya twit'); +end - if ( flgTesting ) - % These will just go in the object properties - rawtlt = single(-60:3:60); - nTilts = uint32(length(rawtlt)); - TLT = zeros(nTilts,23,'single','gpuArray'); - TLT(:,4) = rawtlt; - TLT(:,1) = 1:length(rawtlt); - % 12,13 = astigmatism = 0 - % def,pix,cs,wl,ampcont - pixelSize = single(2.0); - defocus = single(12000); - CS = single(2.7); - WL = single(0.01969); - AC = single(0.1); - TLT(:,[12,13,14,15,16,17,18,19]) = repmat([1e-7,pi/4,1,-defocus*10^-10,pixelSize*10^-10,CS*10^-3,WL*10^-10,AC],single(nTilts),1); - TLT(:,11) = TLT(:,1).*2;%%[[41:-1:20,1:19].*3]'; - iThickness = 75; - samplingRate = 1; - fractionOfDose =1; - fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(:,4)).*400 )); - fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics(:)); - doHalfGrid = true; - centerGrid = 1; - reconGeometry = (wantedSize); - reconShift = [0,0,0]; - originPrj = wantedSize; - originPrj = ceil((originPrj+1)./2); - originPrj(3) = 1; - % Position list is just for the wanted subTomo - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); - for iPrj = 1:size(exposureWeight,3) - exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)).^2 .* ... - (fractionOfDose.*(fractionOfElastics(iPrj))); - end % This matrix would rotate a vector into the particles reference frame, - % we want to rotate back, so take the transpose to invert. - particleRotMat = BH_defineMatrix([0,0,0],'Bah','forward'); - prjVector = [0,0,0] - originVol + reconShift + [0.0,0.0,1.0]; - - defocusAst = single(0); - exposure = gather(single(TLT(:,11))); - [ SF3D] = BH_weightMaskMex(gather(uint32(wantedSize)), ... - gather(single(samplingRate)),... - gather(single(TLT)), ... - gather(single([0,0,0])),gather(single([reconGeometry;0,0,0]))); - - tic + +if ( flgTesting ) + % These will just go in the object properties + rawtlt = single(-60:3:60); + nTilts = uint32(length(rawtlt)); + TLT = zeros(nTilts,23,'single','gpuArray'); + TLT(:,4) = rawtlt; + TLT(:,1) = 1:length(rawtlt); + % 12,13 = astigmatism = 0 + % def,pix,cs,wl,ampcont + pixelSize = single(2.0); + defocus = single(12000); + CS = single(2.7); + WL = single(0.01969); + AC = single(0.1); + TLT(:,[12,13,14,15,16,17,18,19]) = repmat([1e-7,pi/4,1,-defocus*10^-10,pixelSize*10^-10,CS*10^-3,WL*10^-10,AC],single(nTilts),1); + TLT(:,11) = TLT(:,1).*2;%%[[41:-1:20,1:19].*3]'; + iThickness = 75; + samplingRate = 1; + fractionOfDose =1; + fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(:,4)).*400 )); + fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics(:)); + doHalfGrid = true; + centerGrid = 1; + reconGeometry = (wantedSize); + reconShift = [0,0,0]; + originPrj = wantedSize; + originPrj = ceil((originPrj+1)./2); + originPrj(3) = 1; + % Position list is just for the wanted subTomo + originVol = ceil((reconGeometry(1,1:3)+1)./2); + lowerLeftVol = originPrj+reconShift-originVol; + [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); + for iPrj = 1:size(exposureWeight,3) + exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)).^2 .* ... + (fractionOfDose.*(fractionOfElastics(iPrj))); + end % This matrix would rotate a vector into the particles reference frame, + % we want to rotate back, so take the transpose to invert. + particleRotMat = BH_defineMatrix([0,0,0],'Bah','forward'); + prjVector = [0,0,0] - originVol + reconShift + [0.0,0.0,1.0]; + + defocusAst = single(0); + exposure = gather(single(TLT(:,11))); + [ SF3D] = BH_weightMaskMex(gather(uint32(wantedSize)), ... + gather(single(samplingRate)),... + gather(single(TLT)), ... + gather(single([0,0,0])),gather(single([reconGeometry;0,0,0]))); + + tic for i = 1:30 - [ SF3D] = BH_weightMaskMex(gather(uint32(wantedSize)), ... - gather(single(samplingRate)),... - gather(single(TLT)), ... - gather(single([0,0,0])),gather(single([reconGeometry;0,0,0]))); + [ SF3D] = BH_weightMaskMex(gather(uint32(wantedSize)), ... + gather(single(samplingRate)),... + gather(single(TLT)), ... + gather(single([0,0,0])),gather(single([reconGeometry;0,0,0]))); end toc/30 - + + + SAVE_IMG(SF3D,'test3d.mrc'); +else + % These will just go in the object properties + reconShift = reconGeometry(2,:); + originPrj = ceil((originPrj+1)./2); + originPrj(3) = 1; + % Position list is just for the wanted subTomo + originVol = ceil((reconGeometry(1,1:3)+1)./2); + lowerLeftVol = originPrj+reconShift-originVol; + + + % This matrix would rotate a vector into the particles reference frame, + % we want to rotate back, so take the transpose to invert. + particleRotMat = reshape(positionList(17:25),3,3)'; + + % Taken from synthetic mapback, not sure why I'm adding the 1 here. + prjVector = positionList(11:13) - originVol + reconShift + [0.0,0.0,1.0]; + % We want the origin of the subTomogram in the tilted image to calculate + % a defocus offset. + + +end - SAVE_IMG(SF3D,'test3d.mrc'); - else - % These will just go in the object properties - reconShift = reconGeometry(2,:); - originPrj = ceil((originPrj+1)./2); - originPrj(3) = 1; - % Position list is just for the wanted subTomo - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - - - % This matrix would rotate a vector into the particles reference frame, - % we want to rotate back, so take the transpose to invert. - particleRotMat = reshape(positionList(17:25),3,3)'; - - % Taken from synthetic mapback, not sure why I'm adding the 1 here. - prjVector = positionList(11:13) - originVol + reconShift + [0.0,0.0,1.0]; - % We want the origin of the subTomogram in the tilted image to calculate - % a defocus offset. - +nZpad = 9; +if mod(nZpad+1,2) + error('nZpad is assmued odd'); +end + +[ taper ] = BH_multi_calcTaper(floor(nZpad./2)); +taper = [ flip(taper), 1, taper ]; +taper = taper./sum(taper(:)); - end - nZpad = 9; - if mod(nZpad+1,2) - error('nZpad is assmued odd'); - end - - [ taper ] = BH_multi_calcTaper(floor(nZpad./2)); - taper = [ flip(taper), 1, taper ]; - taper = taper./sum(taper(:)); +oX = floor(d1/2)+1; +oY = floor(d2/2)+1; +oZ = floor(d3/2)+1; - - oX = floor(d1/2)+1; - oY = floor(d2/2)+1; - oZ = floor(d3/2)+1; +[X,Y,Z] = BH_multi_gridCoordinates([d1,d2,nZpad],'Cartesian',... + 'GPU',{'none'},0,1,0,{'halfgrid'}); - [X,Y,Z] = BH_multi_gridCoordinates([d1,d2,nZpad],'Cartesian',... - 'GPU',{'none'},0,1,0,{'halfgrid'}); - - [rad,phi] = BH_multi_gridCoordinates([d1,d2],'Cylindrical',... - 'GPU',{'none'},1,1,0,{'halfgrid'}); +[rad,phi] = BH_multi_gridCoordinates([d1,d2],'Cylindrical',... + 'GPU',{'none'},1,1,0,{'halfgrid'}); % [X,Y,Z] = BH_multi_gridCoordinates([d1,d2,nZpad],'Cartesian',... % 'GPU',{'none'},0,1,0); -% +% % [rad,phi] = BH_multi_gridCoordinates([d1,d2],'Cylindrical',... -% 'GPU',{'none'},1,1,0); +% 'GPU',{'none'},1,1,0); - % Assuming the pixel size is constant. - rad = {(rad./ (samplingRate.*TLT(1,16).*10^10)),[1,1],1.*phi}; - clear phi +% Assuming the pixel size is constant. +rad = {(rad./ (samplingRate.*TLT(1,16).*10^10)),[1,1],1.*phi}; +clear phi - spatialToPixel = 0.5; - - xIn = X + 1; - yIn = Y + oY; - zIn = Z + floor(nZpad/2); - - wIn = ones(size(X)); - for iTaper = 1:length(taper) - wIn(:,:,iTaper) = taper(iTaper); - end +spatialToPixel = 0.5; + +xIn = X + 1; +yIn = Y + oY; +zIn = Z + floor(nZpad/2); + +wIn = ones(size(X)); +for iTaper = 1:length(taper) + wIn(:,:,iTaper) = taper(iTaper); +end - - X = X + spatialToPixel; - Y = Y + spatialToPixel; - Z = Z + spatialToPixel; - reconstruction = zeros([oX,d2,d3],'single','gpuArray'); +X = X + spatialToPixel; +Y = Y + spatialToPixel; +Z = Z + spatialToPixel; - weights = reconstruction; +reconstruction = zeros([oX,d2,d3],'single','gpuArray'); + +weights = reconstruction; for iPrj = 1:size(TLT,1) - - + + % Need a defocus offset based on XYZ position in the tomogram rTilt = BH_defineMatrix(TLT(iPrj,4),'TILT','forwardVector') ; prjCoords = rTilt * prjVector'; iDefocusOffset = (prjCoords(3).*(TLT(iPrj,16))+TLT(iPrj,15)); iDefocus = [iDefocusOffset - TLT(iPrj,12), ... - iDefocusOffset + TLT(iPrj,12), ... - TLT(iPrj,13)]; - - + iDefocusOffset + TLT(iPrj,12), ... + TLT(iPrj,13)]; + + % TODO add an option to check and oversample the CTF, then crop in real space, to fix aliasing probs. - + iCTF = (BH_ctfCalc(rad,TLT(iPrj,17),TLT(iPrj,18),iDefocus,[oX,d2],TLT(iPrj,19),-1)).^2; -% SAVE_IMG(iCTF.*exposureWeight(:,:,TLT(iPrj,1)),'BHctf.mrc'); -% iCTF = mexCTF(false,true,gather(int16(d1)),gather(int16(d2)),... -% gather(single(samplingRate.*TLT(iPrj,16)*10^10)),gather(single(TLT(iPrj,18)*10^10)),... -% gather(single(TLT(iPrj,17)*10^3)),gather(single(iDefocus(1)*-1*10^10)),... -% gather(single(iDefocus(2)*-1*10^10)),gather(single(iDefocus(3)*180/pi)),gather(single(TLT(iPrj,19))),true,... -% single(1.0), gather(single(TLT(iPrj,11)))); -% SAVE_IMG(iCTF,'MExctf.mrc'); -% error('iCTF') + % SAVE_IMG(iCTF.*exposureWeight(:,:,TLT(iPrj,1)),'BHctf.mrc'); + % iCTF = mexCTF(false,true,gather(int16(d1)),gather(int16(d2)),... + % gather(single(samplingRate.*TLT(iPrj,16)*10^10)),gather(single(TLT(iPrj,18)*10^10)),... + % gather(single(TLT(iPrj,17)*10^3)),gather(single(iDefocus(1)*-1*10^10)),... + % gather(single(iDefocus(2)*-1*10^10)),gather(single(iDefocus(3)*180/pi)),gather(single(TLT(iPrj,19))),true,... + % single(1.0), gather(single(TLT(iPrj,11)))); + % SAVE_IMG(iCTF,'MExctf.mrc'); + % error('iCTF') % In some cases we may be before the first zero. TODO fixme if (mean(iCTF(:)) < 0.25 ) iCTF = iCTF.*0 + 1; end -% iCTF = mexCTF(true,true,int16(d1),int16(d2),single(TLT(iPrj,16)*10^10),single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),single(iDefocus(1)*-1*10^10),single(iDefocus(2)*-1*10^10),single(30),single(0.1)); + % iCTF = mexCTF(true,true,int16(d1),int16(d2),single(TLT(iPrj,16)*10^10),single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),single(iDefocus(1)*-1*10^10),single(iDefocus(2)*-1*10^10),single(30),single(0.1)); if (length(exposureWeight) > 1) iWeight = wIn.*repmat(exposureWeight(:,:,TLT(iPrj,1)),1,1,nZpad); iCTF = iWeight .* repmat(iCTF,1,1,nZpad); @@ -211,22 +211,22 @@ iCTF = repmat(iCTF,1,1,nZpad).*wIn; end - - + + % The sample is rotated by tiltA, which is like rotating the beam by % -tiltA. We want to insert in the plane perpendicular to the beam. - xform = particleRotMat *rTilt; - - Xnew = X.*xform(1) + (Z.*xform(7)) ; - Ynew = Y; - Znew = X.*xform(3) + (Z.*xform(9)) ; + xform = particleRotMat *rTilt; + + Xnew = X.*xform(1) + (Z.*xform(7)) ; + Ynew = Y; + Znew = X.*xform(3) + (Z.*xform(9)) ; + + % iZ = iOffset - nOffCenter - 1 + spatialToPixel; + % Xnew = X.*xform(1) + Y.*xform(4) + (iZ*xform(7)) ; + % Ynew = X.*xform(2) + Y.*xform(5) + (iZ*xform(8)) ; + % Znew = X.*xform(3) + Y.*xform(6) + (iZ*xform(9)) ; -% iZ = iOffset - nOffCenter - 1 + spatialToPixel; -% Xnew = X.*xform(1) + Y.*xform(4) + (iZ*xform(7)) ; -% Ynew = X.*xform(2) + Y.*xform(5) + (iZ*xform(8)) ; -% Znew = X.*xform(3) + Y.*xform(6) + (iZ*xform(9)) ; - % For any points rotated into the unsampled half, generate Hermitian % mate. The CTF is real, so no need to also negate that, however, for % reconstruc @@ -234,11 +234,11 @@ Xnew(hermitianSym) = -1.*Xnew(hermitianSym); Ynew(hermitianSym) = -1.*Ynew(hermitianSym); Znew(hermitianSym) = -1.*Znew(hermitianSym); - + clear hermitianSym - + % Need to use the conjugate!!! - + Xnew = floor(Xnew + 1) ; Ynew = floor(Ynew + oY); Znew = floor(Znew + oZ) ; @@ -247,12 +247,12 @@ xOutOfBounds = Xnew(:,1,:) <= 0 & Xnew(:,1,:) > size(reconstruction,1); yOutOfBounds = Ynew(:,1,:) <= 0 & Ynew(:,1,:) > size(reconstruction,2); zOutOfBounds = Znew(:,1,:) <= 0 & Znew(:,1,:) > size(reconstruction,3); - - + + % Not sure what is the most efficient way to do this. All at once, better mem harder sort, or incrementally. - + % For rotation about a single axis, we only need to worry about one value - % of that axis to deterimine unique positions + % of that axis to deterimine unique positions [~, ia] = unique([reshape(Xnew(:,1,:),oX*nZpad,1),reshape(Znew(:,1,:),oX*nZpad,1)],'rows'); k = false([oX*nZpad,1],'gpuArray'); k(ia) = true; @@ -260,72 +260,72 @@ k = repmat(reshape(k,oX,1,nZpad),1,d2,1); k = find(k); -% A = [Xnew(k),Ynew(k),Znew(k)]; - -% A = unique([Xnew,Ynew,Znew],'rows'); - + % A = [Xnew(k),Ynew(k),Znew(k)]; + + % A = unique([Xnew,Ynew,Znew],'rows'); + Xnew = Xnew(k); Ynew = Ynew(k); Znew = Znew(k); - - -% keepVal = Xnew(k) > 0 & Ynew(k) > 0 & Znew(k) > 0 &... -% Xnew(k) <= size(reconstruction,1) & ... -% Ynew(k) <= size(reconstruction,2) & ... -% Znew(k) <= size(reconstruction,3); -% A = A(keepVal,:); - linearIDX = sub2ind([oX,d2,d3],Xnew,Ynew,Znew); -% A = A(keepVal,:); -% linearIDX = sub2ind([oX,d2,d3],A(:,1),A(:,2),A(:,3)); - - - - % A now has a list of uniqe 3D coords in the reconstruction + + + % keepVal = Xnew(k) > 0 & Ynew(k) > 0 & Znew(k) > 0 &... + % Xnew(k) <= size(reconstruction,1) & ... + % Ynew(k) <= size(reconstruction,2) & ... + % Znew(k) <= size(reconstruction,3); + % A = A(keepVal,:); + linearIDX = sub2ind([oX,d2,d3],Xnew,Ynew,Znew); + % A = A(keepVal,:); + % linearIDX = sub2ind([oX,d2,d3],A(:,1),A(:,2),A(:,3)); + + + + % A now has a list of uniqe 3D coords in the reconstruction % Those coords now need to be xformed back into the projection (transpose, should it also be negative?) % Optimized for single-axis Xnew = Xnew - 1; Znew = Znew - oZ; xInterp = Xnew.*xform(1) + Znew .* xform(3) + 1; zInterp = Xnew.*xform(7) + Znew .* xform(9) + floor(nZpad/2); -% xInterp = (A(:,1)-1).*xform(1) + (A(:,2)-oY) .*xform(2) + (A(:,3)-oZ) .* xform(3) + 1; -% yInterp = (A(:,1)-1).*xform(4) + (A(:,2)-oY) .*xform(5) + (A(:,3)-oZ) .* xform(6) + oY; -% zInterp = (A(:,1)-1).*xform(7) + (A(:,2)-oY) .*xform(8) + (A(:,3)-oZ) .* xform(9) + floor(nZpad/2); - - - - + % xInterp = (A(:,1)-1).*xform(1) + (A(:,2)-oY) .*xform(2) + (A(:,3)-oZ) .* xform(3) + 1; + % yInterp = (A(:,1)-1).*xform(4) + (A(:,2)-oY) .*xform(5) + (A(:,3)-oZ) .* xform(6) + oY; + % zInterp = (A(:,1)-1).*xform(7) + (A(:,2)-oY) .*xform(8) + (A(:,3)-oZ) .* xform(9) + floor(nZpad/2); + + + + reconstruction(linearIDX) = reconstruction(linearIDX) + ... - interpn(xIn,yIn,zIn,iCTF,xInterp,Ynew,zInterp,'linear',0); - + interpn(xIn,yIn,zIn,iCTF,xInterp,Ynew,zInterp,'linear',0); + weights(linearIDX) = weights(linearIDX) + ... - interpn(xIn,yIn,zIn,iWeight,xInterp,Ynew,zInterp,'linear',0); - - -% weights(linearIDX) = weights(linearIDX) + 1; - - - + interpn(xIn,yIn,zIn,iWeight,xInterp,Ynew,zInterp,'linear',0); + + + % weights(linearIDX) = weights(linearIDX) + 1; + + + end -clear iCTF xIn yIn - clear xInterp yInterp iWeight +clear iCTF xIn yIn +clear xInterp yInterp iWeight doConv = false; if doConv -% TODO these should be set up top, and possible outside. -gaussDiam = 5; -g = gpuArray(BH_multi_gaussian3d(gaussDiam.*[1,1,1],0.75)); - -reconstruction = BH_multi_makeHermitian(reconstruction,gaussDiam,1); -weights = BH_multi_makeHermitian(weights + 0.01,gaussDiam,1); -reconstruction = convn(reconstruction,g,'same')./convn(weights,g,'same'); -recosntruction = reconstruction(1+gaussDiam:end,:,:); - -else -reconstruction = reconstruction ./ (weights + 0.01); + % TODO these should be set up top, and possible outside. + gaussDiam = 5; + g = gpuArray(BH_multi_gaussian3d(gaussDiam.*[1,1,1],0.75)); + + reconstruction = BH_multi_makeHermitian(reconstruction,gaussDiam,1); + weights = BH_multi_makeHermitian(weights + 0.01,gaussDiam,1); + reconstruction = convn(reconstruction,g,'same')./convn(weights,g,'same'); + recosntruction = reconstruction(1+gaussDiam:end,:,:); + +else + reconstruction = reconstruction ./ (weights + 0.01); end % % Trim the fat % reconstruction = reconstruction(gaussDiam+1:end-gaussDiam, ... diff --git a/testScripts/fourierTransformer.m b/testScripts/fourierTransformer.m index 23ec9cfc..a5a5d455 100644 --- a/testScripts/fourierTransformer.m +++ b/testScripts/fourierTransformer.m @@ -1,9 +1,9 @@ classdef fourierTransformer < handle %Create and store cufft plans, do R2C and C2R with optional normalization. - + properties (Access = 'public') - + bandpass; % for now create one internally and just chop off half. inputSize = ''; phaseSwapOffset = ''; @@ -13,19 +13,19 @@ padValIn = ''; padValOut = ''; -% end -% -% properties (Access = 'protected') - + % end + % + % properties (Access = 'protected') + normalization_factor; - + plan_FWD = ''; - plan_INV = ''; + plan_INV = ''; bpVals = [0,0,0,0]; bpDoesNotExist = true; invTrim; % boolean for the input size reduced dimensino phaseCenter = ''; - indexCenterFWD = ''; + indexCenterFWD = ''; indexCenterINV = '' paddedVolumeNeedsToBeInitialized = true; paddedVolumeIsNonZero = true; @@ -50,16 +50,16 @@ error('Did not recognize the extra argument when intializing the fourierTransformer'); end else - if (isnumeric(varargin{1})) - if (numel(varargin{1}) == 6) - padValIn = varargin{1}; - shouldPad = true; - else - error('Did not recognize the extra argument as a padding value when intializing the fourierTransformer'); - end + if (isnumeric(varargin{1})) + if (numel(varargin{1}) == 6) + padValIn = varargin{1}; + shouldPad = true; else - error('Did not recognize the extra argument as str or number when intializing the fourierTransformer'); - end + error('Did not recognize the extra argument as a padding value when intializing the fourierTransformer'); + end + else + error('Did not recognize the extra argument as str or number when intializing the fourierTransformer'); + end end end % Must be single and on gpu @@ -73,10 +73,10 @@ obj.inputSize = size(inputVol); obj.halfDim = 1; obj.halfDimSize = floor(obj.inputSize(1)/2)+1; - + obj.phaseSwapOffset = mod(obj.inputSize,2); - + if (mod(obj.inputSize(obj.halfDim),2)) obj.invTrim = int16(1); @@ -93,7 +93,7 @@ % All dimensions even, only use a fwd mask for fwd/inv swap obj.useFwdSwapForInverse = true; end - + if (numel(obj.inputSize) > 2) obj.is2d = false; else @@ -101,47 +101,47 @@ end [ft] = fwdFFT(obj,inputVol); - + obj.normalization_factor = 1./sqrt(numel(inputVol)); - + end - + function [ft] = fwdFFT(obj, inputVol, varargin) - % Vararginr = + % Vararginr = % 1 - normalize scaling, 0 for none, 1 if only fourier comp, 2 for % complete. I.e. unlike FFTW or MATLAB, cufft scales by 1/root(n) on the fwd - % and 1/root(n) on the inverse (the other two are 1/n on the forward) + % and 1/root(n) on the inverse (the other two are 1/n on the forward) % 2 - bool center and standardize to 1 % 3 - Bandpass filter ( or a 1 ) doBandpass = false; doCenter = false; doCTF = false; - + if nargin > 2 doNorm = varargin{1}; - doCenter = varargin{2}; + doCenter = varargin{2}; if length(varargin) > 2 && ~isempty(varargin{3}) - doBandpass = true; + doBandpass = true; obj.makeBandPass(size(inputVol),varargin{3}) - end + end if length(varargin) > 3 doCTF = true; end - + else doNorm = 0; end - + if isempty([obj.plan_FWD,obj.plan_INV]) [ ft, obj.plan_FWD, obj.plan_INV ] = mexFFT(inputVol,obj.invTrim); else - [ ft ] = mexFFT(inputVol,obj.invTrim,obj.plan_FWD, obj.plan_INV); + [ ft ] = mexFFT(inputVol,obj.invTrim,obj.plan_FWD, obj.plan_INV); end @@ -153,7 +153,7 @@ ft = ft .* (obj.normalization_factor^doNorm); end if (doBandpass) - + ft = ft .* obj.bandpass; end if (doCenter) @@ -162,14 +162,14 @@ end end - + function [ft] = invFFT(obj, inputVol, varargin) if nargin > 2 doNorm = varargin{1}; else doNorm = 0; end - + [ ft ] = mexFFT(inputVol,obj.invTrim,obj.plan_FWD, obj.plan_INV); % For some reason calling this is OUTRAGEOUSLY slow, add option to jsut pass it in to the mexFFT @@ -178,7 +178,7 @@ end end - + function to_cpu(obj) % There are a number of places where I reset the gpuDevice. Use this % to protect the properties. @@ -198,29 +198,29 @@ function delete(obj) % first argument has one element; if ~(isempty(obj.plan_FWD) && isempty(obj.plan_INV)) - mexFFT(gpuArray(1),obj.invTrim,obj.plan_FWD,obj.plan_INV); - + mexFFT(gpuArray(1),obj.invTrim,obj.plan_FWD,obj.plan_INV); + end - end + end function [inputVol] = shiftStretch(obj, inputVol, shiftXY, Mag, isCentered) - [ dU, dV ] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... - {'none'},1,isCentered,0,{'halfgrid'}); - inputVol = inputVol .* (Mag.^-2.*exp(-2i.*pi.*(dU.*shiftXY(1)+dV.*shiftXY(2)))); - - clear dU dV - + [ dU, dV ] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... + {'none'},1,isCentered,0,{'halfgrid'}); + inputVol = inputVol .* (Mag.^-2.*exp(-2i.*pi.*(dU.*shiftXY(1)+dV.*shiftXY(2)))); + + clear dU dV + end function [inputVol] = swapPhase(obj, inputVol, direction) - + % Create the phase swap indices if needed - if isempty(obj.phaseCenter) + if isempty(obj.phaseCenter) if obj.is2d [ obj.phaseCenter, dV ] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... - {'none'},1,0,0,{'halfgrid'}); + {'none'},1,0,0,{'halfgrid'}); if (obj.inputSize(1) == obj.inputSize(2)) sx = obj.halfDimSize-1+obj.phaseSwapOffset(1); obj.phaseCenter = exp(-2i.*pi.*sx.*(obj.phaseCenter+dV)); @@ -232,11 +232,11 @@ function delete(obj) end else [ obj.phaseCenter, dV, dW] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... - {'none'},1,0,0,{'halfgrid'}); + {'none'},1,0,0,{'halfgrid'}); if ((obj.inputSize(1) == obj.inputSize(2)) && (obj.inputSize(2) == obj.inputSize(3))) sx = obj.halfDimSize-1+obj.OddSizeOversampled; obj.phaseCenter = exp(-2i.*pi.*sx.*(obj.phaseCenter+dV+dW)); - clear dU dV dW + clear dU dV dW else hX = floor(obj.inputSize(1)/2) + obj.phaseSwapOffset(1); hY = floor(obj.inputSize(2)/2) + obj.phaseSwapOffset(2); @@ -247,8 +247,8 @@ function delete(obj) end end - - + + if strcmp(direction,'fwd') inputVol = inputVol .* obj.phaseCenter; elseif strcmp(direction,'inv') @@ -256,7 +256,7 @@ function delete(obj) else error('Direction not fwd or inv: %s',direction); end - + end @@ -267,14 +267,14 @@ function delete(obj) window = 0; % Create the phase swap indices if needed - if isempty(obj.indexCenterFWD) -% obj.indexCenterFWD = BH_fftShift(window,obj.inputSize,1,'halfgrid'); + if isempty(obj.indexCenterFWD) + % obj.indexCenterFWD = BH_fftShift(window,obj.inputSize,1,'halfgrid'); obj.indexCenterFWD = EMC_maskIndex('fftshift', obj.inputSize, 'GPU', {'half',true}); - + end inputVol = inputVol(obj.indexCenterFWD); - + end function [inputVol] = swapIndexINV(obj, inputVol) @@ -283,22 +283,22 @@ function delete(obj) % the peak is expected near the center. window = 0; -% if (obj.useFwdSwapForInverse) -% inputVol = obj.swapIndexFWD(inputVol); -% else - % Create the phase swap indices if needed - if isempty(obj.indexCenterINV) -% obj.indexCenterINV = BH_fftShift(window,-1.*obj.inputSize,1,'halfgrid'); - obj.indexCenterINV = EMC_maskIndex('ifftshift', obj.inputSize, 'GPU', {'half',true}); - end - inputVol = inputVol(obj.indexCenterINV); + % if (obj.useFwdSwapForInverse) + % inputVol = obj.swapIndexFWD(inputVol); + % else + % Create the phase swap indices if needed + if isempty(obj.indexCenterINV) + % obj.indexCenterINV = BH_fftShift(window,-1.*obj.inputSize,1,'halfgrid'); + obj.indexCenterINV = EMC_maskIndex('ifftshift', obj.inputSize, 'GPU', {'half',true}); end - -% end + inputVol = inputVol(obj.indexCenterINV); + end + + % end function [inputVol] = fwdSwap(obj,inputVol) - % For transformations in Fourier space. + % For transformations in Fourier space. % Assumed that the input vol is not index swapped. TODO add a check inputVol = obj.swapPhase(obj.swapIndexFWD(inputVol),'fwd'); @@ -307,12 +307,12 @@ function delete(obj) function [inputVol] = invSwap(obj,inputVol) % For transformations in Fourier space - + inputVol = obj.swapIndexINV(obj.swapPhase(inputVol,'inv')); end - + function makeBandPass(obj, sizeInput, bpValsNew) @@ -321,18 +321,18 @@ function makeBandPass(obj, sizeInput, bpValsNew) if (obj.bpDoesNotExist) doCalc = true; else - if abs(sum( obj.bpVals - bpValsNew )) > 1e-4 + if abs(sum( obj.bpVals - bpValsNew )) > 1e-4 doCalc = true; - end + end end - - if (doCalc) + + if (doCalc) obj.bandpass = BH_bandpass3d(sizeInput,bpValsNew(1),... - bpValsNew(2),... - bpValsNew(3),... - 'GPU', ... - bpValsNew(4)); + bpValsNew(2),... + bpValsNew(3),... + 'GPU', ... + bpValsNew(4)); switch ndims(obj.bandpass) case 3 obj.bandpass = obj.bandpass(1:obj.halfDimSize,:,:); diff --git a/testScripts/imshow3D.m b/testScripts/imshow3D.m index 15f49c4f..3d820dc1 100644 --- a/testScripts/imshow3D.m +++ b/testScripts/imshow3D.m @@ -6,17 +6,17 @@ function imshow3D( Img, disprange ) % imshow3D ( Image ) % imshow3D ( Image , [] ) % imshow3D ( Image , [LOW HIGH] ) -% -% Image: 3D image MxNxK (K slices of MxN images) +% +% Image: 3D image MxNxK (K slices of MxN images) % [LOW HIGH]: display range that controls the display intensity range of % a grayscale image (default: the widest available range) % % Use the scroll bar or mouse scroll wheel to switch between slices. To % adjust window and level values keep the mouse right button pressed and % drag the mouse up and down (for level adjustment) or right and left (for -% window adjustment). -% -% "Auto W/L" button adjust the window and level automatically +% window adjustment). +% +% "Auto W/L" button adjust the window and level automatically % % While "Fine Tune" check box is checked the window/level adjustment gets % 16 times less sensitive to mouse movement, to make it easier to control @@ -25,15 +25,15 @@ function imshow3D( Img, disprange ) % Note: The sensitivity of mouse based window and level adjustment is set % based on the user defined display intensity range; the wider the range % the more sensitivity to mouse drag. -% -% +% +% % Example % -------- % % Display an image (MRI example) -% load mri -% Image = squeeze(D); -% figure, -% imshow3D(Image) +% load mri +% Image = squeeze(D); +% figure, +% imshow3D(Image) % % % Display the image, adjust the display range % figure, @@ -45,7 +45,7 @@ function imshow3D( Img, disprange ) % - Maysam Shahedi (mshahedi@gmail.com) % - Released: 1.0.0 Date: 2013/04/15 % - Revision: 1.1.0 Date: 2013/04/19 -% +% sno = size(Img,3); % number of slices S = round(sno/2); @@ -113,7 +113,7 @@ function imshow3D( Img, disprange ) LevV =0.5; Win = 1; WLAdjCoe = 0.1; -end +end SFntSz = 9; LFntSz = 10; @@ -155,7 +155,7 @@ function imshow3D( Img, disprange ) stxthand = uicontrol('Style', 'text','Position', Stxt_Pos,'String',sprintf('Slice# %d / %d',S, sno), 'BackgroundColor', [0.8 0.8 0.8], 'FontSize', SFntSz); else stxthand = uicontrol('Style', 'text','Position', Stxt_Pos,'String','2D image', 'BackgroundColor', [0.8 0.8 0.8], 'FontSize', SFntSz); -end +end ltxthand = uicontrol('Style', 'text','Position', Ltxt_Pos,'String','Level: ', 'BackgroundColor', [0.8 0.8 0.8], 'FontSize', LFntSz); wtxthand = uicontrol('Style', 'text','Position', Wtxt_Pos,'String','Window: ', 'BackgroundColor', [0.8 0.8 0.8], 'FontSize', WFntSz); lvalhand = uicontrol('Style', 'edit','Position', Lval_Pos,'String',sprintf('%6.0f',LevV), 'BackgroundColor', [1 1 1], 'FontSize', LVFntSz,'Callback', @WinLevChanged); @@ -240,13 +240,13 @@ function mouseClick (object, eventdata) % -=< Window and level mouse adjustment >=- function WinLevAdj(varargin) PosDiff = get(0,'PointerLocation') - InitialCoord; - + Win = Win + PosDiff(1) * WLAdjCoe * FineTuneC(get(ChBxhand,'Value')+1); LevV = LevV - PosDiff(2) * WLAdjCoe * FineTuneC(get(ChBxhand,'Value')+1); if (Win < 1) Win = 1; end - + [Rmin, Rmax] = WL2R(Win,LevV); caxis([Rmin, Rmax]) set(lvalhand, 'String', sprintf('%6.0f',LevV)); @@ -256,13 +256,13 @@ function WinLevAdj(varargin) % -=< Window and level text adjustment >=- function WinLevChanged(varargin) - + LevV = str2double(get(lvalhand, 'string')); Win = str2double(get(wvalhand, 'string')); if (Win < 1) Win = 1; end - + [Rmin, Rmax] = WL2R(Win,LevV); caxis([Rmin, Rmax]) end diff --git a/testScripts/interpolator.m b/testScripts/interpolator.m index 3b4a0c70..c7307b3e 100644 --- a/testScripts/interpolator.m +++ b/testScripts/interpolator.m @@ -11,8 +11,8 @@ symmetry_matrices; symmetry_type = ''; % The resampled volume -% resampledVol; - % + % resampledVol; + % input_size = ''; dummy_vol = ''; @@ -46,19 +46,19 @@ obj.dummy_vol = inputVol(1:2,1:2,1:2); [resampledVol, obj.texObject, obj.cuArray] = mexXform3d(obj.input_size,... - inputVol, ... - angles, ... - shifts, ... - boolDirection); + inputVol, ... + angles, ... + shifts, ... + boolDirection); for iSym = 2:obj.nSymMats - + symAngles = gather(angles*obj.symmetry_matrices{iSym}); resampledVol = resampledVol + mexXform3d(obj.input_size, ... - obj.dummy_vol,... - symAngles, ... - shifts, ... - boolDirection, ... - obj.texObject); + obj.dummy_vol,... + symAngles, ... + shifts, ... + boolDirection, ... + obj.texObject); end @@ -78,8 +78,8 @@ obj.input_volume = inputVol; obj.free_cuda_objects(); end - - + + end function boolDirection = check_inputs(obj, direction) @@ -94,7 +94,7 @@ end end - + function [ resampledVol ] = interp3d(obj, angles, shifts, convention, direction, symmetry) %METHOD1 Summary of this method goes here @@ -103,41 +103,41 @@ check_symmetry(obj, symmetry, convention); [angles, shifts] = check_anglesAndShifts(obj, angles, shifts, convention, direction); boolDirection = check_inputs(obj, direction); - -% resampledVol = zeros(size(inputVol), 'single', 'gpuArray'); + + % resampledVol = zeros(size(inputVol), 'single', 'gpuArray'); for iSym = 1:obj.nSymMats - + symAngles = gather(angles*obj.symmetry_matrices{iSym}); if (iSym == 1) - + if (obj.make_tex_persistent) - resampledVol = mexXform3d(obj.input_size, ... - obj.dummy_vol,... - symAngles, ... - shifts, ... - boolDirection, ... - obj.texObject); + resampledVol = mexXform3d(obj.input_size, ... + obj.dummy_vol,... + symAngles, ... + shifts, ... + boolDirection, ... + obj.texObject); else % If we didn't make the tex obj persistent then we need to % return it here, for use at least with the symmetric vols [resampledVol, obj.texObject, obj.cuArray] = ... - mexXform3d(obj.input_size, ... - obj.input_volume,... - symAngles, ... - shifts, ... - boolDirection); - + mexXform3d(obj.input_size, ... + obj.input_volume,... + symAngles, ... + shifts, ... + boolDirection); + end - + else - % w or w/o persistent tex, this call is the same. - resampledVol = resampledVol + mexXform3d(obj.input_size, ... - obj.dummy_vol,... - symAngles, ... - shifts, ... - boolDirection, ... - obj.texObject); + % w or w/o persistent tex, this call is the same. + resampledVol = resampledVol + mexXform3d(obj.input_size, ... + obj.dummy_vol,... + symAngles, ... + shifts, ... + boolDirection, ... + obj.texObject); end end @@ -149,11 +149,11 @@ if ~(obj.make_tex_persistent) obj.free_cuda_objects(); end - + end - function [angles, shifts] = check_anglesAndShifts(obj,angles, shifts, convention, direction) + function [angles, shifts] = check_anglesAndShifts(obj,angles, shifts, convention, direction) %METHOD1 Summary of this method goes here % Detailed explanation goes here if (numel(angles) == 3) @@ -168,52 +168,52 @@ shifts = gather(single(shifts)); end - end + end - function [ ] = check_symmetry(obj, symmetry, convention) - - if isempty(obj.symmetry_type | ~strcmpi(symmetry, obj.symmetry_type)) - - if (strcmpi(symmetry, obj.symmetry_type)) - return; - else - obj.symmetry_type = symmetry; -% warning('The requested symmetry (%s) is different from that initialized (%s)\n',symmetry,obj.symmetry_type); - end - - if (symmetry(1) ~= 'C' && ~strcmp(convention, 'Bah')) - error('Alternate conventions like Helical (%s encountered) only support CX symmetry\n', convention) - end - - switch symmetry(1) - case 'C' - if (length(symmetry) < 2) - error('Cyclic symmetry requires an int specifying CX'); - end - obj.symmetry_matrices = cell(EMC_str2double(symmetry(2:end)),1); + function [ ] = check_symmetry(obj, symmetry, convention) + + if isempty(obj.symmetry_type | ~strcmpi(symmetry, obj.symmetry_type)) + + if (strcmpi(symmetry, obj.symmetry_type)) + return; + else + obj.symmetry_type = symmetry; + % warning('The requested symmetry (%s) is different from that initialized (%s)\n',symmetry,obj.symmetry_type); + end + + if (symmetry(1) ~= 'C' && ~strcmp(convention, 'Bah')) + error('Alternate conventions like Helical (%s encountered) only support CX symmetry\n', convention) + end + + switch symmetry(1) + case 'C' + if (length(symmetry) < 2) + error('Cyclic symmetry requires an int specifying CX'); + end + obj.symmetry_matrices = cell(EMC_str2double(symmetry(2:end)),1); obj.nSymMats = length(obj.symmetry_matrices); symInc = 360 / obj.nSymMats; for iSym = 0: obj.nSymMats - 1 obj.symmetry_matrices{iSym + 1} = BH_defineMatrix([0,0,iSym.*symInc], convention, 'forward'); - end - case 'D' - if (length(symmetry) < 2) - error('D symmetry requires an int specifying DX'); - end + end + case 'D' + if (length(symmetry) < 2) + error('D symmetry requires an int specifying DX'); + end n_inplane = EMC_str2double(symmetry(2:end)); - obj.symmetry_matrices = cell(n_inplane,1); + obj.symmetry_matrices = cell(n_inplane,1); obj.nSymMats = length(obj.symmetry_matrices); symInc = 360 / obj.nSymMats; for iSym = 0: obj.nSymMats - 1 obj.symmetry_matrices{iSym + 1} = BH_defineMatrix([0,0,iSym.*symInc], 'Bah', 'forward'); obj.symmetry_matrices{iSym + 1 + n_inplane} = obj.symmetry_matrices{iSym + 1} .* [1,-1,1;1,-1,1;1,1,-1]; - end - case 'O' - if (length(symmetry) > 1) - error('Octahedral symmetry requires no int'); - end - obj.nSymMats = 24; - obj.symmetry_matrices = cell(obj.nSymMats,1); + end + case 'O' + if (length(symmetry) > 1) + error('Octahedral symmetry requires no int'); + end + obj.nSymMats = 24; + obj.symmetry_matrices = cell(obj.nSymMats,1); % Pulling this from symmetry matrix.cpp from cisTEM. If it % works, get rid of the addition and just change it to 1 based % indexing manually. @@ -241,172 +241,172 @@ obj.symmetry_matrices{21 + 1} = [ 0.000000, 1.000000, 0.000000;-1.000000, 0.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; obj.symmetry_matrices{22 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; obj.symmetry_matrices{23 + 1} = [ 0.000000,-1.000000, 0.000000; 1.000000, 0.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; - - case 'I' - - obj.nSymMats = 60; - obj.symmetry_matrices = cell(obj.nSymMats,1); - if (length(symmetry) < 2) + + case 'I' + + obj.nSymMats = 60; + obj.symmetry_matrices = cell(obj.nSymMats,1); + if (length(symmetry) < 2) + + obj.symmetry_matrices{ 0 + 1} = [ 1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; + obj.symmetry_matrices{ 1 + 1} = [-0.500000,-0.309017,-0.809017;-0.309017,-0.809017, 0.500000;-0.809017, 0.500000, 0.309017]'; + obj.symmetry_matrices{ 2 + 1} = [-0.309017,-0.809017,-0.500000; 0.809017,-0.500000, 0.309017;-0.500000,-0.309017, 0.809017]'; + obj.symmetry_matrices{ 3 + 1} = [ 0.309017,-0.809017,-0.500000; 0.809017, 0.500000,-0.309017; 0.500000,-0.309017, 0.809017]'; + obj.symmetry_matrices{ 4 + 1} = [ 0.500000,-0.309017,-0.809017;-0.309017, 0.809017,-0.500000; 0.809017, 0.500000, 0.309017]'; + obj.symmetry_matrices{ 5 + 1} = [ 0.000000, 0.000000,-1.000000;-1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000]'; + obj.symmetry_matrices{ 6 + 1} = [ 0.809017,-0.500000,-0.309017;-0.500000,-0.309017,-0.809017; 0.309017, 0.809017,-0.500000]'; + obj.symmetry_matrices{ 7 + 1} = [ 0.500000, 0.309017,-0.809017;-0.309017,-0.809017,-0.500000;-0.809017, 0.500000,-0.309017]'; + obj.symmetry_matrices{ 8 + 1} = [-0.500000, 0.309017,-0.809017; 0.309017,-0.809017,-0.500000;-0.809017,-0.500000, 0.309017]'; + obj.symmetry_matrices{ 9 + 1} = [-0.809017,-0.500000,-0.309017; 0.500000,-0.309017,-0.809017; 0.309017,-0.809017, 0.500000]'; + obj.symmetry_matrices{10 + 1} = [ 0.000000,-1.000000, 0.000000; 0.000000, 0.000000,-1.000000; 1.000000, 0.000000, 0.000000]'; + obj.symmetry_matrices{11 + 1} = [-0.309017,-0.809017, 0.500000; 0.809017,-0.500000,-0.309017; 0.500000, 0.309017, 0.809017]'; + obj.symmetry_matrices{12 + 1} = [ 0.809017,-0.500000, 0.309017; 0.500000, 0.309017,-0.809017; 0.309017, 0.809017, 0.500000]'; + obj.symmetry_matrices{13 + 1} = [ 0.809017, 0.500000,-0.309017;-0.500000, 0.309017,-0.809017;-0.309017, 0.809017, 0.500000]'; + obj.symmetry_matrices{14 + 1} = [-0.309017, 0.809017,-0.500000;-0.809017,-0.500000,-0.309017;-0.500000, 0.309017, 0.809017]'; + obj.symmetry_matrices{15 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; + obj.symmetry_matrices{16 + 1} = [-0.500000,-0.309017,-0.809017; 0.309017, 0.809017,-0.500000; 0.809017,-0.500000,-0.309017]'; + obj.symmetry_matrices{17 + 1} = [-0.309017,-0.809017,-0.500000;-0.809017, 0.500000,-0.309017; 0.500000, 0.309017,-0.809017]'; + obj.symmetry_matrices{18 + 1} = [ 0.309017,-0.809017,-0.500000;-0.809017,-0.500000, 0.309017;-0.500000, 0.309017,-0.809017]'; + obj.symmetry_matrices{19 + 1} = [ 0.500000,-0.309017,-0.809017; 0.309017,-0.809017, 0.500000;-0.809017,-0.500000,-0.309017]'; + obj.symmetry_matrices{20 + 1} = [ 0.000000, 0.000000,-1.000000; 1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000]'; + obj.symmetry_matrices{21 + 1} = [ 0.809017,-0.500000,-0.309017; 0.500000, 0.309017, 0.809017;-0.309017,-0.809017, 0.500000]'; + obj.symmetry_matrices{22 + 1} = [ 0.500000, 0.309017,-0.809017; 0.309017, 0.809017, 0.500000; 0.809017,-0.500000, 0.309017]'; + obj.symmetry_matrices{23 + 1} = [-0.500000, 0.309017,-0.809017;-0.309017, 0.809017, 0.500000; 0.809017, 0.500000,-0.309017]'; + obj.symmetry_matrices{24 + 1} = [-0.809017,-0.500000,-0.309017;-0.500000, 0.309017, 0.809017;-0.309017, 0.809017,-0.500000]'; + obj.symmetry_matrices{25 + 1} = [ 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000;-1.000000, 0.000000, 0.000000]'; + obj.symmetry_matrices{26 + 1} = [-0.309017,-0.809017, 0.500000;-0.809017, 0.500000, 0.309017;-0.500000,-0.309017,-0.809017]'; + obj.symmetry_matrices{27 + 1} = [ 0.809017,-0.500000, 0.309017;-0.500000,-0.309017, 0.809017;-0.309017,-0.809017,-0.500000]'; + obj.symmetry_matrices{28 + 1} = [ 0.809017, 0.500000,-0.309017; 0.500000,-0.309017, 0.809017; 0.309017,-0.809017,-0.500000]'; + obj.symmetry_matrices{29 + 1} = [-0.309017, 0.809017,-0.500000; 0.809017, 0.500000, 0.309017; 0.500000,-0.309017,-0.809017]'; + obj.symmetry_matrices{30 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000; 0.000000, 0.000000,-1.000000]'; + obj.symmetry_matrices{31 + 1} = [ 0.500000, 0.309017, 0.809017;-0.309017,-0.809017, 0.500000; 0.809017,-0.500000,-0.309017]'; + obj.symmetry_matrices{32 + 1} = [ 0.309017, 0.809017, 0.500000; 0.809017,-0.500000, 0.309017; 0.500000, 0.309017,-0.809017]'; + obj.symmetry_matrices{33 + 1} = [-0.309017, 0.809017, 0.500000; 0.809017, 0.500000,-0.309017;-0.500000, 0.309017,-0.809017]'; + obj.symmetry_matrices{34 + 1} = [-0.500000, 0.309017, 0.809017;-0.309017, 0.809017,-0.500000;-0.809017,-0.500000,-0.309017]'; + obj.symmetry_matrices{35 + 1} = [ 0.000000, 0.000000, 1.000000;-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000]'; + obj.symmetry_matrices{36 + 1} = [-0.809017, 0.500000, 0.309017;-0.500000,-0.309017,-0.809017;-0.309017,-0.809017, 0.500000]'; + obj.symmetry_matrices{37 + 1} = [-0.500000,-0.309017, 0.809017;-0.309017,-0.809017,-0.500000; 0.809017,-0.500000, 0.309017]'; + obj.symmetry_matrices{38 + 1} = [ 0.500000,-0.309017, 0.809017; 0.309017,-0.809017,-0.500000; 0.809017, 0.500000,-0.309017]'; + obj.symmetry_matrices{39 + 1} = [ 0.809017, 0.500000, 0.309017; 0.500000,-0.309017,-0.809017;-0.309017, 0.809017,-0.500000]'; + obj.symmetry_matrices{40 + 1} = [ 0.000000, 1.000000, 0.000000; 0.000000, 0.000000,-1.000000;-1.000000, 0.000000, 0.000000]'; + obj.symmetry_matrices{41 + 1} = [ 0.309017, 0.809017,-0.500000; 0.809017,-0.500000,-0.309017;-0.500000,-0.309017,-0.809017]'; + obj.symmetry_matrices{42 + 1} = [-0.809017, 0.500000,-0.309017; 0.500000, 0.309017,-0.809017;-0.309017,-0.809017,-0.500000]'; + obj.symmetry_matrices{43 + 1} = [-0.809017,-0.500000, 0.309017;-0.500000, 0.309017,-0.809017; 0.309017,-0.809017,-0.500000]'; + obj.symmetry_matrices{44 + 1} = [ 0.309017,-0.809017, 0.500000;-0.809017,-0.500000,-0.309017; 0.500000,-0.309017,-0.809017]'; + obj.symmetry_matrices{45 + 1} = [ 1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000,-1.000000]'; + obj.symmetry_matrices{46 + 1} = [ 0.500000, 0.309017, 0.809017; 0.309017, 0.809017,-0.500000;-0.809017, 0.500000, 0.309017]'; + obj.symmetry_matrices{47 + 1} = [ 0.309017, 0.809017, 0.500000;-0.809017, 0.500000,-0.309017;-0.500000,-0.309017, 0.809017]'; + obj.symmetry_matrices{48 + 1} = [-0.309017, 0.809017, 0.500000;-0.809017,-0.500000, 0.309017; 0.500000,-0.309017, 0.809017]'; + obj.symmetry_matrices{49 + 1} = [-0.500000, 0.309017, 0.809017; 0.309017,-0.809017, 0.500000; 0.809017, 0.500000, 0.309017]'; + obj.symmetry_matrices{50 + 1} = [ 0.000000, 0.000000, 1.000000; 1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000]'; + obj.symmetry_matrices{51 + 1} = [-0.809017, 0.500000, 0.309017; 0.500000, 0.309017, 0.809017; 0.309017, 0.809017,-0.500000]'; + obj.symmetry_matrices{52 + 1} = [-0.500000,-0.309017, 0.809017; 0.309017, 0.809017, 0.500000;-0.809017, 0.500000,-0.309017]'; + obj.symmetry_matrices{53 + 1} = [ 0.500000,-0.309017, 0.809017;-0.309017, 0.809017, 0.500000;-0.809017,-0.500000, 0.309017]'; + obj.symmetry_matrices{54 + 1} = [ 0.809017, 0.500000, 0.309017;-0.500000, 0.309017, 0.809017; 0.309017,-0.809017, 0.500000]'; + obj.symmetry_matrices{55 + 1} = [ 0.000000, 1.000000, 0.000000; 0.000000, 0.000000, 1.000000; 1.000000, 0.000000, 0.000000]'; + obj.symmetry_matrices{56 + 1} = [ 0.309017, 0.809017,-0.500000;-0.809017, 0.500000, 0.309017; 0.500000, 0.309017, 0.809017]'; + obj.symmetry_matrices{57 + 1} = [-0.809017, 0.500000,-0.309017;-0.500000,-0.309017, 0.809017; 0.309017, 0.809017, 0.500000]'; + obj.symmetry_matrices{58 + 1} = [-0.809017,-0.500000, 0.309017; 0.500000,-0.309017, 0.809017;-0.309017, 0.809017, 0.500000]'; + obj.symmetry_matrices{59 + 1} = [ 0.309017,-0.809017, 0.500000; 0.809017, 0.500000, 0.309017;-0.500000, 0.309017, 0.809017]'; + elseif strcmp(symmetry(2),'2') + obj.symmetry_matrices{ 0 + 1} = [ 1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; + obj.symmetry_matrices{ 1 + 1} = [ 0.500000,-0.809017,-0.309017;-0.809017,-0.309017,-0.500000; 0.309017, 0.500000,-0.809017]'; + obj.symmetry_matrices{ 2 + 1} = [ 0.309017,-0.500000, 0.809017;-0.500000,-0.809017,-0.309017; 0.809017,-0.309017,-0.500000]'; + obj.symmetry_matrices{ 3 + 1} = [-0.309017, 0.500000, 0.809017;-0.500000,-0.809017, 0.309017; 0.809017,-0.309017, 0.500000]'; + obj.symmetry_matrices{ 4 + 1} = [-0.500000, 0.809017,-0.309017;-0.809017,-0.309017, 0.500000; 0.309017, 0.500000, 0.809017]'; + obj.symmetry_matrices{ 5 + 1} = [ 0.000000, 0.000000,-1.000000;-1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000]'; + obj.symmetry_matrices{ 6 + 1} = [-0.309017,-0.500000, 0.809017; 0.500000,-0.809017,-0.309017; 0.809017, 0.309017, 0.500000]'; + obj.symmetry_matrices{ 7 + 1} = [-0.809017, 0.309017, 0.500000; 0.309017,-0.500000, 0.809017; 0.500000, 0.809017, 0.309017]'; + obj.symmetry_matrices{ 8 + 1} = [-0.809017, 0.309017,-0.500000;-0.309017, 0.500000, 0.809017; 0.500000, 0.809017,-0.309017]'; + obj.symmetry_matrices{ 9 + 1} = [-0.309017,-0.500000,-0.809017;-0.500000, 0.809017,-0.309017; 0.809017, 0.309017,-0.500000]'; + obj.symmetry_matrices{10 + 1} = [ 0.000000,-1.000000, 0.000000; 0.000000, 0.000000,-1.000000; 1.000000, 0.000000, 0.000000]'; + obj.symmetry_matrices{11 + 1} = [-0.809017,-0.309017,-0.500000;-0.309017,-0.500000, 0.809017;-0.500000, 0.809017, 0.309017]'; + obj.symmetry_matrices{12 + 1} = [-0.500000,-0.809017,-0.309017;-0.809017, 0.309017, 0.500000;-0.309017, 0.500000,-0.809017]'; + obj.symmetry_matrices{13 + 1} = [-0.500000,-0.809017, 0.309017;-0.809017, 0.309017,-0.500000; 0.309017,-0.500000,-0.809017]'; + obj.symmetry_matrices{14 + 1} = [-0.809017,-0.309017, 0.500000;-0.309017,-0.500000,-0.809017; 0.500000,-0.809017, 0.309017]'; + obj.symmetry_matrices{15 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; + obj.symmetry_matrices{16 + 1} = [ 0.500000,-0.809017,-0.309017; 0.809017, 0.309017, 0.500000;-0.309017,-0.500000, 0.809017]'; + obj.symmetry_matrices{17 + 1} = [ 0.309017,-0.500000, 0.809017; 0.500000, 0.809017, 0.309017;-0.809017, 0.309017, 0.500000]'; + obj.symmetry_matrices{18 + 1} = [-0.309017, 0.500000, 0.809017; 0.500000, 0.809017,-0.309017;-0.809017, 0.309017,-0.500000]'; + obj.symmetry_matrices{19 + 1} = [-0.500000, 0.809017,-0.309017; 0.809017, 0.309017,-0.500000;-0.309017,-0.500000,-0.809017]'; + obj.symmetry_matrices{20 + 1} = [ 0.000000, 0.000000,-1.000000; 1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000]'; + obj.symmetry_matrices{21 + 1} = [-0.309017,-0.500000, 0.809017;-0.500000, 0.809017, 0.309017;-0.809017,-0.309017,-0.500000]'; + obj.symmetry_matrices{22 + 1} = [-0.809017, 0.309017, 0.500000;-0.309017, 0.500000,-0.809017;-0.500000,-0.809017,-0.309017]'; + obj.symmetry_matrices{23 + 1} = [-0.809017, 0.309017,-0.500000; 0.309017,-0.500000,-0.809017;-0.500000,-0.809017, 0.309017]'; + obj.symmetry_matrices{24 + 1} = [-0.309017,-0.500000,-0.809017; 0.500000,-0.809017, 0.309017;-0.809017,-0.309017, 0.500000]'; + obj.symmetry_matrices{25 + 1} = [ 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000;-1.000000, 0.000000, 0.000000]'; + obj.symmetry_matrices{26 + 1} = [-0.809017,-0.309017,-0.500000; 0.309017, 0.500000,-0.809017; 0.500000,-0.809017,-0.309017]'; + obj.symmetry_matrices{27 + 1} = [-0.500000,-0.809017,-0.309017; 0.809017,-0.309017,-0.500000; 0.309017,-0.500000, 0.809017]'; + obj.symmetry_matrices{28 + 1} = [-0.500000,-0.809017, 0.309017; 0.809017,-0.309017, 0.500000;-0.309017, 0.500000, 0.809017]'; + obj.symmetry_matrices{29 + 1} = [-0.809017,-0.309017, 0.500000; 0.309017, 0.500000, 0.809017;-0.500000, 0.809017,-0.309017]'; + obj.symmetry_matrices{30 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000; 0.000000, 0.000000,-1.000000]'; + obj.symmetry_matrices{31 + 1} = [-0.500000, 0.809017, 0.309017;-0.809017,-0.309017,-0.500000;-0.309017,-0.500000, 0.809017]'; + obj.symmetry_matrices{32 + 1} = [-0.309017, 0.500000,-0.809017;-0.500000,-0.809017,-0.309017;-0.809017, 0.309017, 0.500000]'; + obj.symmetry_matrices{33 + 1} = [ 0.309017,-0.500000,-0.809017;-0.500000,-0.809017, 0.309017;-0.809017, 0.309017,-0.500000]'; + obj.symmetry_matrices{34 + 1} = [ 0.500000,-0.809017, 0.309017;-0.809017,-0.309017, 0.500000;-0.309017,-0.500000,-0.809017]'; + obj.symmetry_matrices{35 + 1} = [ 0.000000, 0.000000, 1.000000;-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000]'; + obj.symmetry_matrices{36 + 1} = [ 0.309017, 0.500000,-0.809017; 0.500000,-0.809017,-0.309017;-0.809017,-0.309017,-0.500000]'; + obj.symmetry_matrices{37 + 1} = [ 0.809017,-0.309017,-0.500000; 0.309017,-0.500000, 0.809017;-0.500000,-0.809017,-0.309017]'; + obj.symmetry_matrices{38 + 1} = [ 0.809017,-0.309017, 0.500000;-0.309017, 0.500000, 0.809017;-0.500000,-0.809017, 0.309017]'; + obj.symmetry_matrices{39 + 1} = [ 0.309017, 0.500000, 0.809017;-0.500000, 0.809017,-0.309017;-0.809017,-0.309017, 0.500000]'; + obj.symmetry_matrices{40 + 1} = [ 0.000000, 1.000000, 0.000000; 0.000000, 0.000000,-1.000000;-1.000000, 0.000000, 0.000000]'; + obj.symmetry_matrices{41 + 1} = [ 0.809017, 0.309017, 0.500000;-0.309017,-0.500000, 0.809017; 0.500000,-0.809017,-0.309017]'; + obj.symmetry_matrices{42 + 1} = [ 0.500000, 0.809017, 0.309017;-0.809017, 0.309017, 0.500000; 0.309017,-0.500000, 0.809017]'; + obj.symmetry_matrices{43 + 1} = [ 0.500000, 0.809017,-0.309017;-0.809017, 0.309017,-0.500000;-0.309017, 0.500000, 0.809017]'; + obj.symmetry_matrices{44 + 1} = [ 0.809017, 0.309017,-0.500000;-0.309017,-0.500000,-0.809017;-0.500000, 0.809017,-0.309017]'; + obj.symmetry_matrices{45 + 1} = [ 1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000,-1.000000]'; + obj.symmetry_matrices{46 + 1} = [-0.500000, 0.809017, 0.309017; 0.809017, 0.309017, 0.500000; 0.309017, 0.500000,-0.809017]'; + obj.symmetry_matrices{47 + 1} = [-0.309017, 0.500000,-0.809017; 0.500000, 0.809017, 0.309017; 0.809017,-0.309017,-0.500000]'; + obj.symmetry_matrices{48 + 1} = [ 0.309017,-0.500000,-0.809017; 0.500000, 0.809017,-0.309017; 0.809017,-0.309017, 0.500000]'; + obj.symmetry_matrices{49 + 1} = [ 0.500000,-0.809017, 0.309017; 0.809017, 0.309017,-0.500000; 0.309017, 0.500000, 0.809017]'; + obj.symmetry_matrices{50 + 1} = [ 0.000000, 0.000000, 1.000000; 1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000]'; + obj.symmetry_matrices{51 + 1} = [ 0.309017, 0.500000,-0.809017;-0.500000, 0.809017, 0.309017; 0.809017, 0.309017, 0.500000]'; + obj.symmetry_matrices{52 + 1} = [ 0.809017,-0.309017,-0.500000;-0.309017, 0.500000,-0.809017; 0.500000, 0.809017, 0.309017]'; + obj.symmetry_matrices{53 + 1} = [ 0.809017,-0.309017, 0.500000; 0.309017,-0.500000,-0.809017; 0.500000, 0.809017,-0.309017]'; + obj.symmetry_matrices{54 + 1} = [ 0.309017, 0.500000, 0.809017; 0.500000,-0.809017, 0.309017; 0.809017, 0.309017,-0.500000]'; + obj.symmetry_matrices{55 + 1} = [ 0.000000, 1.000000, 0.000000; 0.000000, 0.000000, 1.000000; 1.000000, 0.000000, 0.000000]'; + obj.symmetry_matrices{56 + 1} = [ 0.809017, 0.309017, 0.500000; 0.309017, 0.500000,-0.809017;-0.500000, 0.809017, 0.309017]'; + obj.symmetry_matrices{57 + 1} = [ 0.500000, 0.809017, 0.309017; 0.809017,-0.309017,-0.500000;-0.309017, 0.500000,-0.809017]'; + obj.symmetry_matrices{58 + 1} = [ 0.500000, 0.809017,-0.309017; 0.809017,-0.309017, 0.500000; 0.309017,-0.500000,-0.809017]'; + obj.symmetry_matrices{59 + 1} = [ 0.809017, 0.309017,-0.500000; 0.309017, 0.500000, 0.809017; 0.500000,-0.809017, 0.309017]'; + else + error('Icosahderal symmetry may be I or I2'); + end + + otherwise + error('Only CX, DX, O, I(2) symmetry is implemented'); + end - obj.symmetry_matrices{ 0 + 1} = [ 1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; - obj.symmetry_matrices{ 1 + 1} = [-0.500000,-0.309017,-0.809017;-0.309017,-0.809017, 0.500000;-0.809017, 0.500000, 0.309017]'; - obj.symmetry_matrices{ 2 + 1} = [-0.309017,-0.809017,-0.500000; 0.809017,-0.500000, 0.309017;-0.500000,-0.309017, 0.809017]'; - obj.symmetry_matrices{ 3 + 1} = [ 0.309017,-0.809017,-0.500000; 0.809017, 0.500000,-0.309017; 0.500000,-0.309017, 0.809017]'; - obj.symmetry_matrices{ 4 + 1} = [ 0.500000,-0.309017,-0.809017;-0.309017, 0.809017,-0.500000; 0.809017, 0.500000, 0.309017]'; - obj.symmetry_matrices{ 5 + 1} = [ 0.000000, 0.000000,-1.000000;-1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000]'; - obj.symmetry_matrices{ 6 + 1} = [ 0.809017,-0.500000,-0.309017;-0.500000,-0.309017,-0.809017; 0.309017, 0.809017,-0.500000]'; - obj.symmetry_matrices{ 7 + 1} = [ 0.500000, 0.309017,-0.809017;-0.309017,-0.809017,-0.500000;-0.809017, 0.500000,-0.309017]'; - obj.symmetry_matrices{ 8 + 1} = [-0.500000, 0.309017,-0.809017; 0.309017,-0.809017,-0.500000;-0.809017,-0.500000, 0.309017]'; - obj.symmetry_matrices{ 9 + 1} = [-0.809017,-0.500000,-0.309017; 0.500000,-0.309017,-0.809017; 0.309017,-0.809017, 0.500000]'; - obj.symmetry_matrices{10 + 1} = [ 0.000000,-1.000000, 0.000000; 0.000000, 0.000000,-1.000000; 1.000000, 0.000000, 0.000000]'; - obj.symmetry_matrices{11 + 1} = [-0.309017,-0.809017, 0.500000; 0.809017,-0.500000,-0.309017; 0.500000, 0.309017, 0.809017]'; - obj.symmetry_matrices{12 + 1} = [ 0.809017,-0.500000, 0.309017; 0.500000, 0.309017,-0.809017; 0.309017, 0.809017, 0.500000]'; - obj.symmetry_matrices{13 + 1} = [ 0.809017, 0.500000,-0.309017;-0.500000, 0.309017,-0.809017;-0.309017, 0.809017, 0.500000]'; - obj.symmetry_matrices{14 + 1} = [-0.309017, 0.809017,-0.500000;-0.809017,-0.500000,-0.309017;-0.500000, 0.309017, 0.809017]'; - obj.symmetry_matrices{15 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; - obj.symmetry_matrices{16 + 1} = [-0.500000,-0.309017,-0.809017; 0.309017, 0.809017,-0.500000; 0.809017,-0.500000,-0.309017]'; - obj.symmetry_matrices{17 + 1} = [-0.309017,-0.809017,-0.500000;-0.809017, 0.500000,-0.309017; 0.500000, 0.309017,-0.809017]'; - obj.symmetry_matrices{18 + 1} = [ 0.309017,-0.809017,-0.500000;-0.809017,-0.500000, 0.309017;-0.500000, 0.309017,-0.809017]'; - obj.symmetry_matrices{19 + 1} = [ 0.500000,-0.309017,-0.809017; 0.309017,-0.809017, 0.500000;-0.809017,-0.500000,-0.309017]'; - obj.symmetry_matrices{20 + 1} = [ 0.000000, 0.000000,-1.000000; 1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000]'; - obj.symmetry_matrices{21 + 1} = [ 0.809017,-0.500000,-0.309017; 0.500000, 0.309017, 0.809017;-0.309017,-0.809017, 0.500000]'; - obj.symmetry_matrices{22 + 1} = [ 0.500000, 0.309017,-0.809017; 0.309017, 0.809017, 0.500000; 0.809017,-0.500000, 0.309017]'; - obj.symmetry_matrices{23 + 1} = [-0.500000, 0.309017,-0.809017;-0.309017, 0.809017, 0.500000; 0.809017, 0.500000,-0.309017]'; - obj.symmetry_matrices{24 + 1} = [-0.809017,-0.500000,-0.309017;-0.500000, 0.309017, 0.809017;-0.309017, 0.809017,-0.500000]'; - obj.symmetry_matrices{25 + 1} = [ 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000;-1.000000, 0.000000, 0.000000]'; - obj.symmetry_matrices{26 + 1} = [-0.309017,-0.809017, 0.500000;-0.809017, 0.500000, 0.309017;-0.500000,-0.309017,-0.809017]'; - obj.symmetry_matrices{27 + 1} = [ 0.809017,-0.500000, 0.309017;-0.500000,-0.309017, 0.809017;-0.309017,-0.809017,-0.500000]'; - obj.symmetry_matrices{28 + 1} = [ 0.809017, 0.500000,-0.309017; 0.500000,-0.309017, 0.809017; 0.309017,-0.809017,-0.500000]'; - obj.symmetry_matrices{29 + 1} = [-0.309017, 0.809017,-0.500000; 0.809017, 0.500000, 0.309017; 0.500000,-0.309017,-0.809017]'; - obj.symmetry_matrices{30 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000; 0.000000, 0.000000,-1.000000]'; - obj.symmetry_matrices{31 + 1} = [ 0.500000, 0.309017, 0.809017;-0.309017,-0.809017, 0.500000; 0.809017,-0.500000,-0.309017]'; - obj.symmetry_matrices{32 + 1} = [ 0.309017, 0.809017, 0.500000; 0.809017,-0.500000, 0.309017; 0.500000, 0.309017,-0.809017]'; - obj.symmetry_matrices{33 + 1} = [-0.309017, 0.809017, 0.500000; 0.809017, 0.500000,-0.309017;-0.500000, 0.309017,-0.809017]'; - obj.symmetry_matrices{34 + 1} = [-0.500000, 0.309017, 0.809017;-0.309017, 0.809017,-0.500000;-0.809017,-0.500000,-0.309017]'; - obj.symmetry_matrices{35 + 1} = [ 0.000000, 0.000000, 1.000000;-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000]'; - obj.symmetry_matrices{36 + 1} = [-0.809017, 0.500000, 0.309017;-0.500000,-0.309017,-0.809017;-0.309017,-0.809017, 0.500000]'; - obj.symmetry_matrices{37 + 1} = [-0.500000,-0.309017, 0.809017;-0.309017,-0.809017,-0.500000; 0.809017,-0.500000, 0.309017]'; - obj.symmetry_matrices{38 + 1} = [ 0.500000,-0.309017, 0.809017; 0.309017,-0.809017,-0.500000; 0.809017, 0.500000,-0.309017]'; - obj.symmetry_matrices{39 + 1} = [ 0.809017, 0.500000, 0.309017; 0.500000,-0.309017,-0.809017;-0.309017, 0.809017,-0.500000]'; - obj.symmetry_matrices{40 + 1} = [ 0.000000, 1.000000, 0.000000; 0.000000, 0.000000,-1.000000;-1.000000, 0.000000, 0.000000]'; - obj.symmetry_matrices{41 + 1} = [ 0.309017, 0.809017,-0.500000; 0.809017,-0.500000,-0.309017;-0.500000,-0.309017,-0.809017]'; - obj.symmetry_matrices{42 + 1} = [-0.809017, 0.500000,-0.309017; 0.500000, 0.309017,-0.809017;-0.309017,-0.809017,-0.500000]'; - obj.symmetry_matrices{43 + 1} = [-0.809017,-0.500000, 0.309017;-0.500000, 0.309017,-0.809017; 0.309017,-0.809017,-0.500000]'; - obj.symmetry_matrices{44 + 1} = [ 0.309017,-0.809017, 0.500000;-0.809017,-0.500000,-0.309017; 0.500000,-0.309017,-0.809017]'; - obj.symmetry_matrices{45 + 1} = [ 1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000,-1.000000]'; - obj.symmetry_matrices{46 + 1} = [ 0.500000, 0.309017, 0.809017; 0.309017, 0.809017,-0.500000;-0.809017, 0.500000, 0.309017]'; - obj.symmetry_matrices{47 + 1} = [ 0.309017, 0.809017, 0.500000;-0.809017, 0.500000,-0.309017;-0.500000,-0.309017, 0.809017]'; - obj.symmetry_matrices{48 + 1} = [-0.309017, 0.809017, 0.500000;-0.809017,-0.500000, 0.309017; 0.500000,-0.309017, 0.809017]'; - obj.symmetry_matrices{49 + 1} = [-0.500000, 0.309017, 0.809017; 0.309017,-0.809017, 0.500000; 0.809017, 0.500000, 0.309017]'; - obj.symmetry_matrices{50 + 1} = [ 0.000000, 0.000000, 1.000000; 1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000]'; - obj.symmetry_matrices{51 + 1} = [-0.809017, 0.500000, 0.309017; 0.500000, 0.309017, 0.809017; 0.309017, 0.809017,-0.500000]'; - obj.symmetry_matrices{52 + 1} = [-0.500000,-0.309017, 0.809017; 0.309017, 0.809017, 0.500000;-0.809017, 0.500000,-0.309017]'; - obj.symmetry_matrices{53 + 1} = [ 0.500000,-0.309017, 0.809017;-0.309017, 0.809017, 0.500000;-0.809017,-0.500000, 0.309017]'; - obj.symmetry_matrices{54 + 1} = [ 0.809017, 0.500000, 0.309017;-0.500000, 0.309017, 0.809017; 0.309017,-0.809017, 0.500000]'; - obj.symmetry_matrices{55 + 1} = [ 0.000000, 1.000000, 0.000000; 0.000000, 0.000000, 1.000000; 1.000000, 0.000000, 0.000000]'; - obj.symmetry_matrices{56 + 1} = [ 0.309017, 0.809017,-0.500000;-0.809017, 0.500000, 0.309017; 0.500000, 0.309017, 0.809017]'; - obj.symmetry_matrices{57 + 1} = [-0.809017, 0.500000,-0.309017;-0.500000,-0.309017, 0.809017; 0.309017, 0.809017, 0.500000]'; - obj.symmetry_matrices{58 + 1} = [-0.809017,-0.500000, 0.309017; 0.500000,-0.309017, 0.809017;-0.309017, 0.809017, 0.500000]'; - obj.symmetry_matrices{59 + 1} = [ 0.309017,-0.809017, 0.500000; 0.809017, 0.500000, 0.309017;-0.500000, 0.309017, 0.809017]'; - elseif strcmp(symmetry(2),'2') - obj.symmetry_matrices{ 0 + 1} = [ 1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; - obj.symmetry_matrices{ 1 + 1} = [ 0.500000,-0.809017,-0.309017;-0.809017,-0.309017,-0.500000; 0.309017, 0.500000,-0.809017]'; - obj.symmetry_matrices{ 2 + 1} = [ 0.309017,-0.500000, 0.809017;-0.500000,-0.809017,-0.309017; 0.809017,-0.309017,-0.500000]'; - obj.symmetry_matrices{ 3 + 1} = [-0.309017, 0.500000, 0.809017;-0.500000,-0.809017, 0.309017; 0.809017,-0.309017, 0.500000]'; - obj.symmetry_matrices{ 4 + 1} = [-0.500000, 0.809017,-0.309017;-0.809017,-0.309017, 0.500000; 0.309017, 0.500000, 0.809017]'; - obj.symmetry_matrices{ 5 + 1} = [ 0.000000, 0.000000,-1.000000;-1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000]'; - obj.symmetry_matrices{ 6 + 1} = [-0.309017,-0.500000, 0.809017; 0.500000,-0.809017,-0.309017; 0.809017, 0.309017, 0.500000]'; - obj.symmetry_matrices{ 7 + 1} = [-0.809017, 0.309017, 0.500000; 0.309017,-0.500000, 0.809017; 0.500000, 0.809017, 0.309017]'; - obj.symmetry_matrices{ 8 + 1} = [-0.809017, 0.309017,-0.500000;-0.309017, 0.500000, 0.809017; 0.500000, 0.809017,-0.309017]'; - obj.symmetry_matrices{ 9 + 1} = [-0.309017,-0.500000,-0.809017;-0.500000, 0.809017,-0.309017; 0.809017, 0.309017,-0.500000]'; - obj.symmetry_matrices{10 + 1} = [ 0.000000,-1.000000, 0.000000; 0.000000, 0.000000,-1.000000; 1.000000, 0.000000, 0.000000]'; - obj.symmetry_matrices{11 + 1} = [-0.809017,-0.309017,-0.500000;-0.309017,-0.500000, 0.809017;-0.500000, 0.809017, 0.309017]'; - obj.symmetry_matrices{12 + 1} = [-0.500000,-0.809017,-0.309017;-0.809017, 0.309017, 0.500000;-0.309017, 0.500000,-0.809017]'; - obj.symmetry_matrices{13 + 1} = [-0.500000,-0.809017, 0.309017;-0.809017, 0.309017,-0.500000; 0.309017,-0.500000,-0.809017]'; - obj.symmetry_matrices{14 + 1} = [-0.809017,-0.309017, 0.500000;-0.309017,-0.500000,-0.809017; 0.500000,-0.809017, 0.309017]'; - obj.symmetry_matrices{15 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000]'; - obj.symmetry_matrices{16 + 1} = [ 0.500000,-0.809017,-0.309017; 0.809017, 0.309017, 0.500000;-0.309017,-0.500000, 0.809017]'; - obj.symmetry_matrices{17 + 1} = [ 0.309017,-0.500000, 0.809017; 0.500000, 0.809017, 0.309017;-0.809017, 0.309017, 0.500000]'; - obj.symmetry_matrices{18 + 1} = [-0.309017, 0.500000, 0.809017; 0.500000, 0.809017,-0.309017;-0.809017, 0.309017,-0.500000]'; - obj.symmetry_matrices{19 + 1} = [-0.500000, 0.809017,-0.309017; 0.809017, 0.309017,-0.500000;-0.309017,-0.500000,-0.809017]'; - obj.symmetry_matrices{20 + 1} = [ 0.000000, 0.000000,-1.000000; 1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000]'; - obj.symmetry_matrices{21 + 1} = [-0.309017,-0.500000, 0.809017;-0.500000, 0.809017, 0.309017;-0.809017,-0.309017,-0.500000]'; - obj.symmetry_matrices{22 + 1} = [-0.809017, 0.309017, 0.500000;-0.309017, 0.500000,-0.809017;-0.500000,-0.809017,-0.309017]'; - obj.symmetry_matrices{23 + 1} = [-0.809017, 0.309017,-0.500000; 0.309017,-0.500000,-0.809017;-0.500000,-0.809017, 0.309017]'; - obj.symmetry_matrices{24 + 1} = [-0.309017,-0.500000,-0.809017; 0.500000,-0.809017, 0.309017;-0.809017,-0.309017, 0.500000]'; - obj.symmetry_matrices{25 + 1} = [ 0.000000,-1.000000, 0.000000; 0.000000, 0.000000, 1.000000;-1.000000, 0.000000, 0.000000]'; - obj.symmetry_matrices{26 + 1} = [-0.809017,-0.309017,-0.500000; 0.309017, 0.500000,-0.809017; 0.500000,-0.809017,-0.309017]'; - obj.symmetry_matrices{27 + 1} = [-0.500000,-0.809017,-0.309017; 0.809017,-0.309017,-0.500000; 0.309017,-0.500000, 0.809017]'; - obj.symmetry_matrices{28 + 1} = [-0.500000,-0.809017, 0.309017; 0.809017,-0.309017, 0.500000;-0.309017, 0.500000, 0.809017]'; - obj.symmetry_matrices{29 + 1} = [-0.809017,-0.309017, 0.500000; 0.309017, 0.500000, 0.809017;-0.500000, 0.809017,-0.309017]'; - obj.symmetry_matrices{30 + 1} = [-1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000; 0.000000, 0.000000,-1.000000]'; - obj.symmetry_matrices{31 + 1} = [-0.500000, 0.809017, 0.309017;-0.809017,-0.309017,-0.500000;-0.309017,-0.500000, 0.809017]'; - obj.symmetry_matrices{32 + 1} = [-0.309017, 0.500000,-0.809017;-0.500000,-0.809017,-0.309017;-0.809017, 0.309017, 0.500000]'; - obj.symmetry_matrices{33 + 1} = [ 0.309017,-0.500000,-0.809017;-0.500000,-0.809017, 0.309017;-0.809017, 0.309017,-0.500000]'; - obj.symmetry_matrices{34 + 1} = [ 0.500000,-0.809017, 0.309017;-0.809017,-0.309017, 0.500000;-0.309017,-0.500000,-0.809017]'; - obj.symmetry_matrices{35 + 1} = [ 0.000000, 0.000000, 1.000000;-1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000]'; - obj.symmetry_matrices{36 + 1} = [ 0.309017, 0.500000,-0.809017; 0.500000,-0.809017,-0.309017;-0.809017,-0.309017,-0.500000]'; - obj.symmetry_matrices{37 + 1} = [ 0.809017,-0.309017,-0.500000; 0.309017,-0.500000, 0.809017;-0.500000,-0.809017,-0.309017]'; - obj.symmetry_matrices{38 + 1} = [ 0.809017,-0.309017, 0.500000;-0.309017, 0.500000, 0.809017;-0.500000,-0.809017, 0.309017]'; - obj.symmetry_matrices{39 + 1} = [ 0.309017, 0.500000, 0.809017;-0.500000, 0.809017,-0.309017;-0.809017,-0.309017, 0.500000]'; - obj.symmetry_matrices{40 + 1} = [ 0.000000, 1.000000, 0.000000; 0.000000, 0.000000,-1.000000;-1.000000, 0.000000, 0.000000]'; - obj.symmetry_matrices{41 + 1} = [ 0.809017, 0.309017, 0.500000;-0.309017,-0.500000, 0.809017; 0.500000,-0.809017,-0.309017]'; - obj.symmetry_matrices{42 + 1} = [ 0.500000, 0.809017, 0.309017;-0.809017, 0.309017, 0.500000; 0.309017,-0.500000, 0.809017]'; - obj.symmetry_matrices{43 + 1} = [ 0.500000, 0.809017,-0.309017;-0.809017, 0.309017,-0.500000;-0.309017, 0.500000, 0.809017]'; - obj.symmetry_matrices{44 + 1} = [ 0.809017, 0.309017,-0.500000;-0.309017,-0.500000,-0.809017;-0.500000, 0.809017,-0.309017]'; - obj.symmetry_matrices{45 + 1} = [ 1.000000, 0.000000, 0.000000; 0.000000,-1.000000, 0.000000; 0.000000, 0.000000,-1.000000]'; - obj.symmetry_matrices{46 + 1} = [-0.500000, 0.809017, 0.309017; 0.809017, 0.309017, 0.500000; 0.309017, 0.500000,-0.809017]'; - obj.symmetry_matrices{47 + 1} = [-0.309017, 0.500000,-0.809017; 0.500000, 0.809017, 0.309017; 0.809017,-0.309017,-0.500000]'; - obj.symmetry_matrices{48 + 1} = [ 0.309017,-0.500000,-0.809017; 0.500000, 0.809017,-0.309017; 0.809017,-0.309017, 0.500000]'; - obj.symmetry_matrices{49 + 1} = [ 0.500000,-0.809017, 0.309017; 0.809017, 0.309017,-0.500000; 0.309017, 0.500000, 0.809017]'; - obj.symmetry_matrices{50 + 1} = [ 0.000000, 0.000000, 1.000000; 1.000000, 0.000000, 0.000000; 0.000000, 1.000000, 0.000000]'; - obj.symmetry_matrices{51 + 1} = [ 0.309017, 0.500000,-0.809017;-0.500000, 0.809017, 0.309017; 0.809017, 0.309017, 0.500000]'; - obj.symmetry_matrices{52 + 1} = [ 0.809017,-0.309017,-0.500000;-0.309017, 0.500000,-0.809017; 0.500000, 0.809017, 0.309017]'; - obj.symmetry_matrices{53 + 1} = [ 0.809017,-0.309017, 0.500000; 0.309017,-0.500000,-0.809017; 0.500000, 0.809017,-0.309017]'; - obj.symmetry_matrices{54 + 1} = [ 0.309017, 0.500000, 0.809017; 0.500000,-0.809017, 0.309017; 0.809017, 0.309017,-0.500000]'; - obj.symmetry_matrices{55 + 1} = [ 0.000000, 1.000000, 0.000000; 0.000000, 0.000000, 1.000000; 1.000000, 0.000000, 0.000000]'; - obj.symmetry_matrices{56 + 1} = [ 0.809017, 0.309017, 0.500000; 0.309017, 0.500000,-0.809017;-0.500000, 0.809017, 0.309017]'; - obj.symmetry_matrices{57 + 1} = [ 0.500000, 0.809017, 0.309017; 0.809017,-0.309017,-0.500000;-0.309017, 0.500000,-0.809017]'; - obj.symmetry_matrices{58 + 1} = [ 0.500000, 0.809017,-0.309017; 0.809017,-0.309017, 0.500000; 0.309017,-0.500000,-0.809017]'; - obj.symmetry_matrices{59 + 1} = [ 0.809017, 0.309017,-0.500000; 0.309017, 0.500000, 0.809017; 0.500000,-0.809017, 0.309017]'; - else - error('Icosahderal symmetry may be I or I2'); - end - - otherwise - error('Only CX, DX, O, I(2) symmetry is implemented'); - end - - - - - -% switch symmetry_type -% case 0 - -% end - end - + + + + + % switch symmetry_type + % case 0 + + % end + end + + + + end + + function [ ] = delete(obj) - - end - - function [ ] = delete(obj) - free_cuda_objects(obj); - end - - function [ ] = free_cuda_objects(obj) + end + + function [ ] = free_cuda_objects(obj) if ~isempty(obj.texObject) && ~isempty(obj.cuArray) -% fprintf('Deleting the texture obj, it is (%d) a uint64 (%d)\n',isa(obj.texObject,'uint64'),isa(obj.cuArray,'uint64')); - mexXform3d(obj.texObject, obj.cuArray); + % fprintf('Deleting the texture obj, it is (%d) a uint64 (%d)\n',isa(obj.texObject,'uint64'),isa(obj.cuArray,'uint64')); + mexXform3d(obj.texObject, obj.cuArray); obj.texObject = ''; obj.cuArray = ''; end - end - + end + end end diff --git a/testScripts/rotm2eul.m b/testScripts/rotm2eul.m index 86dccb06..eef18419 100644 --- a/testScripts/rotm2eul.m +++ b/testScripts/rotm2eul.m @@ -39,9 +39,9 @@ % The parsed sequence will be in all upper-case letters and validated switch seq case 'ZYX' - % Handle Z-Y-X rotation order + % Handle Z-Y-X rotation order eulShaped = calculateEulerAngles(R, 'ZYX'); - + case 'ZYZ' % Handle Z-Y-Z rotation order eulShaped = calculateEulerAngles(R, 'ZYZ'); @@ -60,7 +60,7 @@ %calculateEulerAngles Calculate Euler angles from rotation matrix % EUL = calculateEulerAngles(R, SEQ) calculates the Euler angles, EUL, % corresponding to the input rotation matrix, R. The Euler angles follow -% the axis order specified in SEQ. +% the axis order specified in SEQ. % Preallocate output eul = zeros(1, 3, size(R,3), 'like', R); %#ok @@ -97,7 +97,7 @@ if repetition % Find special cases of rotation matrix values that correspond to Euler % angle singularities. - sy = sqrt(R(i,j,:).*R(i,j,:) + R(i,k,:).*R(i,k,:)); + sy = sqrt(R(i,j,:).*R(i,j,:) + R(i,k,:).*R(i,k,:)); singular = sy < 10 * eps(class(R)); % Calculate Euler angles @@ -113,8 +113,8 @@ else % Find special cases of rotation matrix values that correspond to Euler - % angle singularities. - sy = sqrt(R(i,i,:).*R(i,i,:) + R(j,i,:).*R(j,i,:)); + % angle singularities. + sy = sqrt(R(i,i,:).*R(i,i,:) + R(j,i,:).*R(j,i,:)); singular = sy < 10 * eps(class(R)); % Calculate Euler angles @@ -126,7 +126,7 @@ if numSingular > 0 eul(:,:,singular) = [atan2(-R(j,k,singular), R(j,j,singular)), ... atan2(-R(k,i,singular), sy(:,:,singular)), zeros(1,1,numSingular,'like',R)]; - end + end end if parity diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 2d777c66..001360a1 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -18,7 +18,7 @@ if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - % Ensure we don't have any duplicates: TODO: add an override flag + % Ensure we don't have any duplicates: TODO: add an override flag % This modifies the RawAlign geometry, so should be cycle -1 if (CYCLE > 0) BH_removeDuplicates(PARAMETER_FILE,sprintf('%d', CYCLE-1)); @@ -53,7 +53,7 @@ end % Experimental downweighting of higher frequency info farther from focus. % Could also consider filtering pre reconstruction -try +try flgFilterDefocus = emc.('filterDefocus'); fprintf('\nFiltering by defocus using exp[-(%d*(argmax(def-1,0,5).*q)^%d)]\n',flgFilterDefocus); catch @@ -68,14 +68,14 @@ try projectVolumes = emc.('flgProjectVolumes'); -catch - projectVolumes = false; +catch + projectVolumes = false; end if (projectVolumes && ~flgCutOutVolumes) - flgCutOutVolumes = true; + flgCutOutVolumes = true; end - + doCut = 0 if (flgCutOutVolumes) if isfield(subTomoMeta,'volumesAreCutOut') @@ -87,7 +87,7 @@ end end -try +try track_stats = emc.('track_stats'); catch track_stats = false; @@ -175,10 +175,10 @@ if isempty(bh_global_ML_angleTolerance) bh_global_ML_angleTolerance = 5; end - + if (emc.nPeaks > 1) fprintf('For ML approach:\nUsing a compression factor %3.3f\nUsing an angulare tolerance of %3.3f degrees\n', ... - bh_global_ML_compressByFactor, bh_global_ML_angleTolerance); + bh_global_ML_compressByFactor, bh_global_ML_angleTolerance); end % for now only turn on (optionally) in reference generation. @@ -200,38 +200,38 @@ if (flgClassify) fieldPrefix = 'Raw' - + else fieldPrefix = 'REF' end - - - classVector{1} = emc.(sprintf('%s_classes_odd','Raw')); - classVector{2} = emc.(sprintf('%s_classes_eve','Raw')); - - className = emc.(sprintf('%s_className','Raw')); - samplingRate = emc.('Ali_samplingRate'); - - if (flgMultiRefAlignment && (test_multi_ref_diffmap || ~flgClassify)) - className = emc.(sprintf('Raw_className')) - saveClassSum = emc.(sprintf('Raw_className')) - elseif (flgMultiRefAlignment && flgClassify) - fprintf('\n\nMutliRef and Classify enabled.\n'); - fprintf('Only creating the global class average for PCA\n\n.'); - className = 0; - saveClassSum = 0; - classVector{1} = [0;1]; - classVector{2} = [0;1]; - end - - + + + classVector{1} = emc.(sprintf('%s_classes_odd','Raw')); + classVector{2} = emc.(sprintf('%s_classes_eve','Raw')); + + className = emc.(sprintf('%s_className','Raw')); + samplingRate = emc.('Ali_samplingRate'); + + if (flgMultiRefAlignment && (test_multi_ref_diffmap || ~flgClassify)) + className = emc.(sprintf('Raw_className')) + saveClassSum = emc.(sprintf('Raw_className')) + elseif (flgMultiRefAlignment && flgClassify) + fprintf('\n\nMutliRef and Classify enabled.\n'); + fprintf('Only creating the global class average for PCA\n\n.'); + className = 0; + saveClassSum = 0; + classVector{1} = [0;1]; + classVector{2} = [0;1]; + end + + case 'FinalAlignment' % Special case for the final cycle. % Assuming RawAlignment already run for this cycle and FSC is calculated % Goal is to re-extract odd-half, applying the xform found in fscGold fieldPrefix = 'Raw' - + classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); @@ -249,7 +249,7 @@ % "global" or whatever requested class iRefPrev = 1; - + try aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','REF',iRefPrev)); @@ -257,32 +257,32 @@ % refine the translation per particle. catch fprintf('\nReverting from %s to Raw in loading fitFSC\n','REF'); - + aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)); oddRot = reshape(aliParams(1,:),3,3); - % refine the translation per particle. + % refine the translation per particle. end clear iRefPrev - - - + + + case 'Cluster_cls' STAGEofALIGNMENT = 'Cluster'; ClusterGeomNAME = 'ClusterClsGeom'; fieldPrefix = 'Cls'; - + classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); - + classCoeffs{1} = emc.('Pca_coeffs'); classCoeffs{2} = emc.('Pca_coeffs'); - + samplingRate = emc.(sprintf('Cls_samplingRate')); className = emc.(sprintf('%s_className',fieldPrefix)); if flgClassify < 0 flgGold = 0; - end + end case 'SnrEstimate' @@ -293,7 +293,7 @@ end classVector{1} = [1:25;ones(1,25)]; classVector{2} = [1:25;ones(1,25)]; - + className = 25; samplingRate = emc.(sprintf('%s_samplingRate','Ali')); @@ -320,7 +320,7 @@ interpOrder = 1; end -try +try flgLimitToOneProcess = emc.('flgLimitToOneProcess'); catch flgLimitToOneProcess = 0; @@ -336,7 +336,7 @@ elseif (flgLimitToOneProcess) limitToOne = flgLimitToOneProcess; else - limitToOne = emc.('nCpuCores'); + limitToOne = emc.('nCpuCores'); interpOrder = 1; end @@ -354,110 +354,110 @@ pixelSize = pixelSize .* samplingRate; peakSearch = floor(0.85.*emc.('particleRadius')./pixelSize) peakCOM = [1,1,1].*3; - - + + if ~(isfield(subTomoMeta,'currentCycle')) subTomoMeta.('currentCycle') = 0; end - + % Somewhere I am saving currentCycle as a string. Haven't taken the time to % track it down, but probably in this function. if isa(subTomoMeta.('currentCycle'), 'char') subTomoMeta.('currentCycle') = EMC_str2double(subTomoMeta.('currentCycle')) end -if subTomoMeta.('currentCycle') == CYCLE - 1 - +if subTomoMeta.('currentCycle') == CYCLE - 1 + cycleRead = sprintf('cycle%0.3u', CYCLE - 1); % Save a backup of the cycles total geometry save(sprintf('%s_%s_backup.mat',cycleRead,emc.('subTomoMeta')), ... - 'subTomoMeta'); + 'subTomoMeta'); end - + if strcmpi(STAGEofALIGNMENT, 'RawAlignment') if ( CYCLE ) cycleRead = sprintf('cycle%0.3u', CYCLE - 1) else - flgShiftEucentric = false; % No possible updates on cycle 0 + flgShiftEucentric = false; % No possible updates on cycle 0 cycleRead = sprintf('cycle%0.3u', CYCLE) end else - flgShiftEucentric = false; % No possible updates for other stages of alignments + flgShiftEucentric = false; % No possible updates for other stages of alignments cycleRead = sprintf('cycle%0.3u', CYCLE ) end - - % leave averages at size appropriate for interpolation when extracting to use - % in class avg alignment. - doNotTrim = false; - eachTomo = false; - flgEstSNR = 0; - switch STAGEofALIGNMENT - case 'RawAlignment' - if ( CYCLE ) - geometry = subTomoMeta.(cycleRead).RawAlign; - else - geometry = subTomoMeta.(cycleRead).geometry; - eachTomo = false;%true; - end - if ~(flgClassify) - doNotTrim = true; - end - - case 'FinalAlignment' - geometry = subTomoMeta.(cycleRead).Avg_geometry; - if ~(flgClassify) - doNotTrim = true; - end - - case 'Cluster' - cN = cell(2,1); - - if (flgGold) - cN{1} = sprintf('%s_%d_%d_nClass_%d_ODD',outputPrefix,classCoeffs{1}(1,1), ... - classCoeffs{1}(1,end), className) - - geometry{1} = subTomoMeta.(cycleRead).ClusterResults.(cN{1}); - cN{2} = sprintf('%s_%d_%d_nClass_%d_EVE',outputPrefix,classCoeffs{2}(1,1), ... - classCoeffs{2}(1,end), className) - geometry{2} = subTomoMeta.(cycleRead).ClusterResults.(cN{2}); - - geometry = BH_mergeClassGeometry(geometry{1}, geometry{2}); - else - if (test_fuzz) - cN{1} = sprintf('%s_%d_%d_nClass_%d_STD','cycle002_full_2',classCoeffs{1}(1,1), ... - classCoeffs{1}(1,end), className) - else - cN{1} = sprintf('%s_%d_%d_nClass_%d_STD',outputPrefix,classCoeffs{1}(1,1), ... - classCoeffs{1}(1,end), className) - end - geometry = subTomoMeta.(cycleRead).ClusterResults.(cN{1}); - end +% leave averages at size appropriate for interpolation when extracting to use +% in class avg alignment. +doNotTrim = false; +eachTomo = false; +flgEstSNR = 0; +switch STAGEofALIGNMENT + case 'RawAlignment' + if ( CYCLE ) + geometry = subTomoMeta.(cycleRead).RawAlign; + else + geometry = subTomoMeta.(cycleRead).geometry; + eachTomo = false;%true; + end + if ~(flgClassify) + doNotTrim = true; + end + + case 'FinalAlignment' + geometry = subTomoMeta.(cycleRead).Avg_geometry; + if ~(flgClassify) + doNotTrim = true; + end + + case 'Cluster' + cN = cell(2,1); + + if (flgGold) + cN{1} = sprintf('%s_%d_%d_nClass_%d_ODD',outputPrefix,classCoeffs{1}(1,1), ... + classCoeffs{1}(1,end), className) + geometry{1} = subTomoMeta.(cycleRead).ClusterResults.(cN{1}); + cN{2} = sprintf('%s_%d_%d_nClass_%d_EVE',outputPrefix,classCoeffs{2}(1,1), ... + classCoeffs{2}(1,end), className) + geometry{2} = subTomoMeta.(cycleRead).ClusterResults.(cN{2}); - subTomoMeta.(cycleRead).('KmsSampling') = samplingRate; - doNotTrim = true; - - case 'SnrEstimate' - flgEstSNR = 1; - if ( CYCLE ) - geometry = subTomoMeta.(cycleRead).RawAlign; + geometry = BH_mergeClassGeometry(geometry{1}, geometry{2}); + else + if (test_fuzz) + cN{1} = sprintf('%s_%d_%d_nClass_%d_STD','cycle002_full_2',classCoeffs{1}(1,1), ... + classCoeffs{1}(1,end), className) else - geometry = subTomoMeta.(cycleRead).geometry; + cN{1} = sprintf('%s_%d_%d_nClass_%d_STD',outputPrefix,classCoeffs{1}(1,1), ... + classCoeffs{1}(1,end), className) end - % randomly dived all currently included subTomos into 10 bins - [geometry, nTotal, snrBinSize] = BH_randomSubset(geometry,'snr',-1,[1]); - [geometry, nTotal, snrBinSize] = BH_randomSubset(geometry,'snr',-1,[2]); - - otherwise - error('STAGE_ALIGNMENTS: [Class,Raw]Alignment, not %s', ... - STAGEofALIGNMENT); - end - + + geometry = subTomoMeta.(cycleRead).ClusterResults.(cN{1}); + end + + + subTomoMeta.(cycleRead).('KmsSampling') = samplingRate; + doNotTrim = true; + + case 'SnrEstimate' + flgEstSNR = 1; + if ( CYCLE ) + geometry = subTomoMeta.(cycleRead).RawAlign; + else + geometry = subTomoMeta.(cycleRead).geometry; + end + % randomly dived all currently included subTomos into 10 bins + [geometry, nTotal, snrBinSize] = BH_randomSubset(geometry,'snr',-1,[1]); + [geometry, nTotal, snrBinSize] = BH_randomSubset(geometry,'snr',-1,[2]); - masterTM = subTomoMeta; clear subTomoMeta + otherwise + error('STAGE_ALIGNMENTS: [Class,Raw]Alignment, not %s', ... + STAGEofALIGNMENT); +end + + +masterTM = subTomoMeta; clear subTomoMeta try symmetry = emc.('symmetry'); @@ -476,12 +476,12 @@ if isfield(masterTM,('tomoCPR_run_in_cycle')) if (flgShiftEucentric && ~isfield(masterTM.(sprintf('%s',cycleRead)), 'eucentric_shifts')) - cycle_to_update = masterTM.('tomoCPR_run_in_cycle')(find(masterTM.('tomoCPR_run_in_cycle')(:,1) == masterTM.currentTomoCPR),2); - if (cycle_to_update == cycleRead) - error('You specified eucentric_fit=1, and you are averaging cycle %d and no shifts are found from cycle %d\n',cycleNumber,cycleRead); - else - flgShiftEucentric = false; - end + cycle_to_update = masterTM.('tomoCPR_run_in_cycle')(find(masterTM.('tomoCPR_run_in_cycle')(:,1) == masterTM.currentTomoCPR),2); + if (cycle_to_update == cycleRead) + error('You specified eucentric_fit=1, and you are averaging cycle %d and no shifts are found from cycle %d\n',cycleNumber,cycleRead); + else + flgShiftEucentric = false; + end end else flgShiftEucentric = false; @@ -493,50 +493,50 @@ -if (flgClassify) - -[ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', pixelSize); +if (flgClassify) + + [ maskType, maskSize, maskRadius, maskCenter ] = ... + BH_multi_maskCheck(emc, 'Ali', pixelSize); % These are used when 'Cluster' is called, to take the masking parameters % from focused PCA/Classification, to produce a montage with reduced % Z-dimension & low pass filtering to be used in decision making but not % alignment. The X,Y dimensions must not change compared to the full % version or else graphical deletion of classes will fail. - - + + [~, pcaMaskSize, pcaMaskRadius, pcaMaskCenter ] = ... - BH_multi_maskCheck(emc, 'Cls', pixelSize); + BH_multi_maskCheck(emc, 'Cls', pixelSize); else -[ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', pixelSize); + [ maskType, maskSize, maskRadius, maskCenter ] = ... + BH_multi_maskCheck(emc, 'Ali', pixelSize); end [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) -padREF = [0,0,0;0,0,0]; + BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) +padREF = [0,0,0;0,0,0]; [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); origMaskSize = sizeMask; %%%%% Considering removing doNotTrim and making this the default. Temporarily %%%%% override here. -% % % %doNotTrim = true; -% This should be moved into BH_multi_validAra +% % % %doNotTrim = true; +% This should be moved into BH_multi_validAra % if (doNotTrim) -% +% if any( (sizeCalc - sizeWindow) < 0 ) sizeCalc = BH_multi_iterator( sizeWindow, 'fourier' ) end - sizeMask = sizeWindow; - % sizeCALC = sizeWindow; +sizeMask = sizeWindow; +% sizeCALC = sizeWindow; % Find the next largest size for fft %[ sizeCalc ] = BH_multi_iterator( sizeCalc, 'fourier' ) - -padCalc = BH_multi_padVal(sizeWindow, sizeCalc); + +padCalc = BH_multi_padVal(sizeWindow, sizeCalc); if (flgFinalAvg) % Load in the even (low-passed) ref and use this only for a (non-iterative) @@ -546,22 +546,22 @@ % that was used to calc the FSC -- that would break things here. imgNAME = sprintf('class_%d_Locations_REF_%s', className, 'EVE'); weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', className, 'EVE'); - + [ refIMG ] = BH_unStackMontage4d(1, ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... - sizeWindow); - + masterTM.(cycleNumber).(imgNAME){1}, ... + masterTM.(cycleNumber).(imgNAME){2},... + sizeWindow); + [ refWDG ] = BH_unStackMontage4d(1, ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... - sizeCalc); - -% % % % % % % [ peakMask] = gather(BH_mask3d('sphere', sizeMask, peakSearch, maskCenter)); + masterTM.(cycleNumber).(weightNAME){1},... + masterTM.(cycleNumber).(weightNAME){2},... + sizeCalc); + + % % % % % % % [ peakMask] = gather(BH_mask3d('sphere', sizeMask, peakSearch, maskCenter)); [ peakMask ] = gather(EMC_maskShape('sphere', sizeMask, peakSearch, 'gpu', {'shift', maskCenter})); - + peakBinary = single(find(peakMask > 0.01)); - + end @@ -607,212 +607,212 @@ spike_info = struct(); spike_info.('std_dev') = nan; if (flgQualityWeight) -%get the average CCC for calculation of particle quality weighting. - - cccVect = []; - wgtVect = []; - angVect = []; - chiVect = []; - - if (spike_prior) - - tiltList_tmp = fieldnames(masterTM.mapBackGeometry); - tiltList_tmp = tiltList_tmp(~ismember(tiltList_tmp,{'viewGroups','tomoName'})); - nST = 1; tiltList = {}; - % First make sure this tilt actualy has tomos. Why is this here/ - for iStack = 1:length(tiltList_tmp) - if masterTM.mapBackGeometry.(tiltList_tmp{iStack}).nTomos - tiltList{nST} = tiltList_tmp{iStack}; - nST = nST +1; - end + %get the average CCC for calculation of particle quality weighting. + + cccVect = []; + wgtVect = []; + angVect = []; + chiVect = []; + + if (spike_prior) + + tiltList_tmp = fieldnames(masterTM.mapBackGeometry); + tiltList_tmp = tiltList_tmp(~ismember(tiltList_tmp,{'viewGroups','tomoName'})); + nST = 1; tiltList = {}; + % First make sure this tilt actualy has tomos. Why is this here/ + for iStack = 1:length(tiltList_tmp) + if masterTM.mapBackGeometry.(tiltList_tmp{iStack}).nTomos + tiltList{nST} = tiltList_tmp{iStack}; + nST = nST +1; end + end + + + % Now loop over all of the tomograms. In the first loop get the + % distribution characterizing the sphericity of the data (if that's a + % word?) i.e. make sure no principle axes are way to big, due to + % points from adjacent virions that were not removed in + % cleanTemplateSearch. + f = fieldnames(masterTM.mapBackGeometry.tomoName); + + for iTomo = 1:length(f) - - % Now loop over all of the tomograms. In the first loop get the - % distribution characterizing the sphericity of the data (if that's a - % word?) i.e. make sure no principle axes are way to big, due to - % points from adjacent virions that were not removed in - % cleanTemplateSearch. - f = fieldnames(masterTM.mapBackGeometry.tomoName); - - for iTomo = 1:length(f) - -% tiltName = masterTM.mapBackGeometry.tomoName.(f{iTomo}).tiltName; -% tomoNumber = masterTM.mapBackGeometry.tomoName.(f{iTomo}).tomoNumber; -% iCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - - tmpTomo = []; - spike_info.(f{iTomo}).('angular_diff') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); - spike_info.(f{iTomo}).('normal_distance') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); - - spike_info.(f{iTomo}).('angular_prob') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); - spike_info.(f{iTomo}).('angular_weight') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); - - nSubTomos = size(geometry.(f{iTomo}) , 1); - particle_coords = zeros(nSubTomos .* emc.nPeaks,8,'single'); - nVol = 1; - for iSubTomo = 1:nSubTomos - for iPeak = 1:emc.nPeaks - particle_coords(nVol,1:5) = geometry.(f{iTomo})(iSubTomo,[26,4,11:13]+(iPeak-1)*26); - nVol = nVol + 1; - end + % tiltName = masterTM.mapBackGeometry.tomoName.(f{iTomo}).tiltName; + % tomoNumber = masterTM.mapBackGeometry.tomoName.(f{iTomo}).tomoNumber; + % iCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); + + tmpTomo = []; + spike_info.(f{iTomo}).('angular_diff') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); + spike_info.(f{iTomo}).('normal_distance') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); + + spike_info.(f{iTomo}).('angular_prob') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); + spike_info.(f{iTomo}).('angular_weight') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); + + nSubTomos = size(geometry.(f{iTomo}) , 1); + particle_coords = zeros(nSubTomos .* emc.nPeaks,8,'single'); + nVol = 1; + for iSubTomo = 1:nSubTomos + for iPeak = 1:emc.nPeaks + particle_coords(nVol,1:5) = geometry.(f{iTomo})(iSubTomo,[26,4,11:13]+(iPeak-1)*26); + nVol = nVol + 1; end - % Logical size nsubtomos x emc.nPeaks - positions_to_analyze = particle_coords(:,1) ~= -9999; - display_fit = false; - radial_shrink_factor = 2; - - - [ normal_vect, chi2 ] = BH_fit_ellipsoidal_prior(pixelSize .* particle_coords(positions_to_analyze,3:5), ... - emc.('particleRadius')(3), ... - radial_shrink_factor, ... - display_fit); - - chiVect = [chiVect chi2]; - particle_coords(positions_to_analyze,[6:8]) = [ normal_vect]; - nVol = 1; - for iSubTomo = 1:nSubTomos - for iPeak = 1:emc.nPeaks - if (particle_coords(nVol,1) ~= -9999) - particleAxis = reshape(geometry.(f{iTomo})(iSubTomo,[17:25]+(iPeak-1)*26),3,3)*[0;0;1]; - angularDiff = dot(particle_coords(nVol,6:8), particleAxis); - if (abs(angularDiff) > 1) - angularDiff = fix(angularDiff); - end - angularDiff = acosd(angularDiff); - spike_info.(f{iTomo}).('angular_diff')(iSubTomo,iPeak) = angularDiff; - tmpTomo = [tmpTomo, angularDiff]; - + end + % Logical size nsubtomos x emc.nPeaks + positions_to_analyze = particle_coords(:,1) ~= -9999; + display_fit = false; + radial_shrink_factor = 2; + + + [ normal_vect, chi2 ] = BH_fit_ellipsoidal_prior(pixelSize .* particle_coords(positions_to_analyze,3:5), ... + emc.('particleRadius')(3), ... + radial_shrink_factor, ... + display_fit); + + chiVect = [chiVect chi2]; + particle_coords(positions_to_analyze,[6:8]) = [ normal_vect]; + nVol = 1; + for iSubTomo = 1:nSubTomos + for iPeak = 1:emc.nPeaks + if (particle_coords(nVol,1) ~= -9999) + particleAxis = reshape(geometry.(f{iTomo})(iSubTomo,[17:25]+(iPeak-1)*26),3,3)*[0;0;1]; + angularDiff = dot(particle_coords(nVol,6:8), particleAxis); + if (abs(angularDiff) > 1) + angularDiff = fix(angularDiff); end - nVol = nVol + 1; - end + angularDiff = acosd(angularDiff); + spike_info.(f{iTomo}).('angular_diff')(iSubTomo,iPeak) = angularDiff; + tmpTomo = [tmpTomo, angularDiff]; + + end + nVol = nVol + 1; end - - angVect = [angVect tmpTomo]; - end - - - spike_info.('std_dev') = std(angVect); - h = histogram(angVect,'Normalization','probability','BinMethod','fd'); - hv = h.Values; -% [~,mc] = max(hv); -% hv(1:mc-1) = hv(mc); - hv = (hv ./ max(hv(:))) .^ 0.5; %(mean(chiVect)./std(chiVect).^2); - x = h.BinWidth:h.BinWidth:h.BinLimits(2); - v = 0:h.BinLimits(2)./1000:h.BinLimits(2); - spike_info.('angular_pdf') = griddedInterpolant(x,hv,'makima','linear'); - figure('Visible','off'), bar(x,h.Values,'w'); hold on - plot(v,spike_info.('angular_pdf')(v),'b','linewidth',2) - sprintf(' %2.3f degrees',spike_info.('std_dev')) - title(sprintf('Spike Angle Prior, std-dev %2.3f degrees',spike_info.('std_dev'))); - file_out = sprintf('%s-spike-angle-prior.pdf', cycleNumber); - saveas(gcf, file_out,'pdf') - hold off; - - - save('spike_hist.mat','angVect','chiVect'); -% figure, + angVect = [angVect tmpTomo]; + end - nVolumes = 0; - addedWeight = 0; - for iParProc = 1:nParProcesses - for iTomo = iterList{iParProc} - if (track_stats) - geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks) = geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks)./geometry.(tomoList{iTomo})(:,2:26:26*emc.nPeaks); - end - - min_weight = 1e-6; - if (spike_prior) - for iSubTomo = 1:size(geometry.(tomoList{iTomo}) , 1) - peakList = false(emc.nPeaks,1); - for iPeak = 1:emc.nPeaks - if (geometry.(tomoList{iTomo})(iSubTomo,26*iPeak)~=-9999) - iWeight = ... - spike_info.('angular_pdf')(spike_info.(tomoList{iTomo}).('angular_diff')(iSubTomo,iPeak)); - peakList(iPeak) = true; - if (iWeight < min_weight || ~isfinite(iWeight)) - iWeight = min_weight; - end - - spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,iPeak) = iWeight; - - nVolumes = nVolumes + 1; - addedWeight = addedWeight + iWeight; + + spike_info.('std_dev') = std(angVect); + h = histogram(angVect,'Normalization','probability','BinMethod','fd'); + hv = h.Values; + % [~,mc] = max(hv); + % hv(1:mc-1) = hv(mc); + hv = (hv ./ max(hv(:))) .^ 0.5; %(mean(chiVect)./std(chiVect).^2); + x = h.BinWidth:h.BinWidth:h.BinLimits(2); + v = 0:h.BinLimits(2)./1000:h.BinLimits(2); + spike_info.('angular_pdf') = griddedInterpolant(x,hv,'makima','linear'); + figure('Visible','off'), bar(x,h.Values,'w'); hold on + plot(v,spike_info.('angular_pdf')(v),'b','linewidth',2) + sprintf(' %2.3f degrees',spike_info.('std_dev')) + + title(sprintf('Spike Angle Prior, std-dev %2.3f degrees',spike_info.('std_dev'))); + file_out = sprintf('%s-spike-angle-prior.pdf', cycleNumber); + saveas(gcf, file_out,'pdf') + hold off; + + + save('spike_hist.mat','angVect','chiVect'); + % figure, + end + + nVolumes = 0; + addedWeight = 0; + for iParProc = 1:nParProcesses + for iTomo = iterList{iParProc} + if (track_stats) + geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks) = geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks)./geometry.(tomoList{iTomo})(:,2:26:26*emc.nPeaks); + end + + min_weight = 1e-6; + if (spike_prior) + for iSubTomo = 1:size(geometry.(tomoList{iTomo}) , 1) + peakList = false(emc.nPeaks,1); + for iPeak = 1:emc.nPeaks + if (geometry.(tomoList{iTomo})(iSubTomo,26*iPeak)~=-9999) + iWeight = ... + spike_info.('angular_pdf')(spike_info.(tomoList{iTomo}).('angular_diff')(iSubTomo,iPeak)); + peakList(iPeak) = true; + if (iWeight < min_weight || ~isfinite(iWeight)) + iWeight = min_weight; end - end - spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList) = ... - spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList) ./ ... - sum(spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList)); - for iScoreMod = 1:emc.nPeaks - if (peakList(iScoreMod)) - geometry.(tomoList{iTomo})(iSubTomo,2 + 26*(iScoreMod-1)) = ... - spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,iScoreMod); - end - end - - - - end + + spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,iPeak) = iWeight; + + nVolumes = nVolumes + 1; + addedWeight = addedWeight + iWeight; + end + end + spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList) = ... + spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList) ./ ... + sum(spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,peakList)); + for iScoreMod = 1:emc.nPeaks + if (peakList(iScoreMod)) + geometry.(tomoList{iTomo})(iSubTomo,2 + 26*(iScoreMod-1)) = ... + spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,iScoreMod); + end + end + + end - - keepVect = geometry.(tomoList{iTomo})(:,26:26:26*emc.nPeaks)~=-9999 ; - - tmpVect = geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks); - - - cccVect = [cccVect ; reshape(tmpVect(keepVect),[],1)]; - tmpVect = geometry.(tomoList{iTomo})(:,2:26:26*emc.nPeaks); - - wgtVect = [wgtVect ; reshape(tmpVect(keepVect),[],1)]; end + + keepVect = geometry.(tomoList{iTomo})(:,26:26:26*emc.nPeaks)~=-9999 ; + + tmpVect = geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks); + + + + cccVect = [cccVect ; reshape(tmpVect(keepVect),[],1)]; + tmpVect = geometry.(tomoList{iTomo})(:,2:26:26*emc.nPeaks); + + wgtVect = [wgtVect ; reshape(tmpVect(keepVect),[],1)]; end - - - if (cccCutOff > 1.0) - sorted_ccc = sort(cccVect); - reqVol = int32(round(cccCutOff)) - length(sorted_ccc) - reqVol - cccCutOff = sorted_ccc(length(sorted_ccc) - reqVol); - fprintf('Removing all volumes with score < %2.2f to return the requested %d volumes\n\n',cccCutOff,reqVol); - elseif (cccCutOff > 0.0) - sorted_ccc = sort(cccVect); - reqVol = cccCutoff; - cccCutOff = sorted_ccc(floor(length(cccVect).*(1 - reqVol))); - fprintf('Removing all volumes with score < %2.2f to return the requested percent %2.2f of possible volumes\n\n',cccCutOff,reqVol); - end - - masterTM.(cycleNumber).('score_sigma') = std(cccVect); - if (spike_prior) -% spike_info.('normalization_factor') = 1;%nVolumes ./ (emc.nPeaks * addedWeight); -% fprintf('From %d possible volumes the total weight is %3.3e\n',nVolumes,addedWeight); - end - avgCCC = mean(cccVect); - - - - mean(wgtVect) - std(wgtVect) - maxCCC = max(cccVect); - mean(wgtVect) - median(wgtVect) - -% figure, hist(cccVect,29) -% figure, hist(wgtVect,29) -% figure, hist((wgtVect./median(wgtVect)).^weightScale,29) -% error('asdf') - - if (track_stats) - fprintf('Avgerage score is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,flgQualityWeight); - else - fprintf('Avgerage CCC is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,flgQualityWeight); - end - - + end + + + if (cccCutOff > 1.0) + sorted_ccc = sort(cccVect); + reqVol = int32(round(cccCutOff)) + length(sorted_ccc) - reqVol + cccCutOff = sorted_ccc(length(sorted_ccc) - reqVol); + fprintf('Removing all volumes with score < %2.2f to return the requested %d volumes\n\n',cccCutOff,reqVol); + elseif (cccCutOff > 0.0) + sorted_ccc = sort(cccVect); + reqVol = cccCutoff; + cccCutOff = sorted_ccc(floor(length(cccVect).*(1 - reqVol))); + fprintf('Removing all volumes with score < %2.2f to return the requested percent %2.2f of possible volumes\n\n',cccCutOff,reqVol); + end + + masterTM.(cycleNumber).('score_sigma') = std(cccVect); + if (spike_prior) + % spike_info.('normalization_factor') = 1;%nVolumes ./ (emc.nPeaks * addedWeight); + % fprintf('From %d possible volumes the total weight is %3.3e\n',nVolumes,addedWeight); + end + avgCCC = mean(cccVect); + + + + mean(wgtVect) + std(wgtVect) + maxCCC = max(cccVect); + mean(wgtVect) + median(wgtVect) + + % figure, hist(cccVect,29) + % figure, hist(wgtVect,29) + % figure, hist((wgtVect./median(wgtVect)).^weightScale,29) + % error('asdf') + + if (track_stats) + fprintf('Avgerage score is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,flgQualityWeight); + else + fprintf('Avgerage CCC is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,flgQualityWeight); + end + + else maxCCC = []; avgCCC = []; @@ -826,44 +826,44 @@ parVect = 1:nParProcesses; parfor iParProc = parVect -% for iParProc = parVect - - % Get the gpuIDX assigned to this process - gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; - iGPUidx = gpuIDXList(iParProc); - gpuDevice(iGPUidx); - fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); - - nExtracted_tmp = zeros(maxClasses,2); - firstLoop = true; - nIgnored = 0; - nSubTomosTotal = 0; - avgVolume_tmp = cell(maxClasses,2); - avgWedge_tmp = cell(maxClasses,2); - geometry_tmp = geometry; - - - - for iRow = 1:maxClasses - for iCol = 1:2 - avgVolume_tmp{iRow, iCol} = zeros(sizeMask, 'single'); - avgWedge_tmp{iRow, iCol}= zeros(sizeCalc, 'single'); - - end + % for iParProc = parVect + + % Get the gpuIDX assigned to this process + gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; + iGPUidx = gpuIDXList(iParProc); + gpuDevice(iGPUidx); + fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); + + nExtracted_tmp = zeros(maxClasses,2); + firstLoop = true; + nIgnored = 0; + nSubTomosTotal = 0; + avgVolume_tmp = cell(maxClasses,2); + avgWedge_tmp = cell(maxClasses,2); + geometry_tmp = geometry; + + + + for iRow = 1:maxClasses + for iCol = 1:2 + avgVolume_tmp{iRow, iCol} = zeros(sizeMask, 'single'); + avgWedge_tmp{iRow, iCol}= zeros(sizeCalc, 'single'); + end + end + + + + if (flgQualityWeight) + [cccWeight,~,~,~,~,~] = BH_multi_gridCoordinates(sizeCalc, ... + 'Cartesian','GPU',... + {'none'},1,0,1); + cccWeight = (cccWeight ./ pixelSize).^2; - - if (flgQualityWeight) - [cccWeight,~,~,~,~,~] = BH_multi_gridCoordinates(sizeCalc, ... - 'Cartesian','GPU',... - {'none'},1,0,1); - cccWeight = (cccWeight ./ pixelSize).^2; - - - - end + end + if (eachTomo) tomoAvgStack = cell(length(iterList{iParProc}),1); tomoWgtStack = cell(length(iterList{iParProc}),1); @@ -871,18 +871,18 @@ tomoAvgStack{iTomo} = zeros(sizeMask, 'single'); tomoWgtStack{iTomo} = zeros(sizeMask, 'single'); end - end + end nTomos = 1; for iTomo = iterList{iParProc} - + if (flgShiftEucentric) - try - geometry_tmp.(tomoList{iTomo})(:,13) = geometry_tmp.(tomoList{iTomo})(:,13) + ... + try + geometry_tmp.(tomoList{iTomo})(:,13) = geometry_tmp.(tomoList{iTomo})(:,13) + ... masterTM.(sprintf('%s',cycleRead)).('eucentric_shifts').(tomoList{iTomo}) ; - catch - fprintf('WARNING, did not find the eucentric shift for tomo %s\n', tomoList{iTomo}); - end + catch + fprintf('WARNING, did not find the eucentric shift for tomo %s\n', tomoList{iTomo}); + end end peakMask_tmp = gpuArray(peakMask); @@ -900,495 +900,495 @@ interpMask_tmp = gpuArray(interpMask); interpMaskWdg_tmp = (gpuArray(interpMaskWdg)); end - + if (eachTomo) tomoAvg = zeros(sizeMask, 'single', 'gpuArray'); tomoWgt = zeros(sizeCalc , 'single', 'gpuArray'); tomoCount = 0; end - + sprintf('gpu %d working on %d/%d volumes\n',iParProc,iTomo,nTomograms) - + tomoName = tomoList{iTomo}; - - - - - tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); + + + + + tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; coords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); [ binShift ] = [0,0,0];%BH_multi_calcBinShift( coords, samplingRate); - + % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); nSubTomos = sum(any(positionList(:,26:26:26*emc.nPeaks) ~= -9999,2)); nSubTomosTotal = nSubTomosTotal + nSubTomos; - - volumeData = []; - %fprintf('loading tomo %d\n',iTomo); - - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - fprintf('Loading tomo %d from tilt %s \n',tomoNumber,tiltName); - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - - if (flgCutOutVolumes && ~doCut) - volumeData = []; - else - - [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,loadTomo); - - if ( loadTomo ) - volHeader = struct(); - volHeader.('nX') = size(volumeData,1); - volHeader.('nY') = size(volumeData,2); - volHeader.('nZ') = size(volumeData,3); - else - volHeader = getHeader(volumeData); - end + + volumeData = []; + %fprintf('loading tomo %d\n',iTomo); + + tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + fprintf('Loading tomo %d from tilt %s \n',tomoNumber,tiltName); + reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); + + if (flgCutOutVolumes && ~doCut) + volumeData = []; + else + + [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... + reconCoords, mapBackIter, ... + samplingRate,iGPUidx,reconScaling,loadTomo); + + if ( loadTomo ) + volHeader = struct(); + volHeader.('nX') = size(volumeData,1); + volHeader.('nY') = size(volumeData,2); + volHeader.('nZ') = size(volumeData,3); + else + volHeader = getHeader(volumeData); + end end - - - - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + + + + iTiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); nCtfGroups = ctfGroupList.(tomoList{1})(1); - - + + % Work on each class seperately pushing to main memory when finished. - for iGold = 1:2-flgFinalAvg - iGold + for iGold = 1:2-flgFinalAvg + iGold size(classVector{iGold},2); for iClassPos = 1:size(classVector{iGold},2) - + iTempParticleODD = zeros(sizeMask,'single','gpuArray'); iTempWedgeODD = zeros(sizeCalc,'single','gpuArray'); - - iTempParticleEVE = zeros(sizeMask,'single','gpuArray'); - iTempWedgeEVE = zeros(sizeCalc,'single','gpuArray'); - - + + iTempParticleEVE = zeros(sizeMask,'single','gpuArray'); + iTempWedgeEVE = zeros(sizeCalc,'single','gpuArray'); + + iClassIDX = classVector{iGold}(1,iClassPos); - % nClassPossible = nClassPossible + ... - % length(find(positionList(:,26) == iClassIDX)); - if classVector{iGold}(1,iClassPos) - % pick just the particular class to work with - + % nClassPossible = nClassPossible + ... + % length(find(positionList(:,26) == iClassIDX)); + if classVector{iGold}(1,iClassPos) + % pick just the particular class to work with + if ( flgEstSNR ) % When the class is for estimating SNR includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... - positionList(:,10) == iClassIDX & ... - positionList(:,7) == iGold ); + positionList(:,10) == iClassIDX & ... + positionList(:,7) == iGold ); else % When the class is from statistical analysis includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... - positionList(:,26) == iClassIDX & ... - positionList(:,7) == iGold ); + positionList(:,26) == iClassIDX & ... + positionList(:,7) == iGold ); end - + else % if class is 0, pick all non-ignored particles - + includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... - any(positionList(:,26:26:26*emc.nPeaks) ~= -9999,2) & ... - positionList(:,7) == iGold ); - + any(positionList(:,26:26:26*emc.nPeaks) ~= -9999,2) & ... + positionList(:,7) == iGold ); + end % Get the position index for each included particle. - + particleIndex = find(includeList); nClassPossible = length(particleIndex); - - - + + + for iSubTomo = particleIndex' - iParticle = []; - iCCCweight = []; - iWedgeMask = []; - % symmetry = classVector{iGold}(2, iClassPos); - - + iParticle = []; + iCCCweight = []; + iWedgeMask = []; + % symmetry = classVector{iGold}(2, iClassPos); + + if ( emc.nPeaks > 1 ) % Calculate a relative weighting, normalize max score to one % and then raise to compressBy factor to downweight lower % scores. - + [ peakWgt, sortedList ] = BH_weightAngCheckPeaks( ... - positionList(iSubTomo,:),... - emc.nPeaks, ... - masterTM.(cycleNumber).('score_sigma') ,... - iSubTomo, tomoList{iTomo},... - track_stats); - % Update any re-ordering or elimination - positionList(iSubTomo,:) = sortedList; + positionList(iSubTomo,:),... + emc.nPeaks, ... + masterTM.(cycleNumber).('score_sigma') ,... + iSubTomo, tomoList{iTomo},... + track_stats); + % Update any re-ordering or elimination + positionList(iSubTomo,:) = sortedList; else peakWgt = 1; end -% % if (spike_prior) -% % peakWgt = peakWgt.*spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,1).*spike_info.('normalization_factor'); -% % end - - make_sf3d = true; - - for iPeak = 1:emc.nPeaks - - if peakWgt(iPeak) == -9999 - positionList(iSubTomo, 26*iPeak) = -9999; - % Skip this peak - continue - end - %Check that the given subTomo is not to be ignored - - - % Get position and rotation info, angles stored as e1,e3,e2 as in AV3 - % and PEET. This also makes inplane shifts easier to see. - - center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; - angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); - wdgIDX = positionList(iSubTomo,9); - - % tmpang = BH_defineMatrix([0,0,-14],'Bah','inv'); - % angles = reshape(angles,3,3)*tmpang; - % if (flgFinalAvg) - % angles = reshape(angles,3,3)*oddRot; - % end - - TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); - - if (make_sf3d) - [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); - make_sf3d = false; - end - - + % % if (spike_prior) + % % peakWgt = peakWgt.*spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,1).*spike_info.('normalization_factor'); + % % end - if (flgQualityWeight) - iCCC = positionList(iSubTomo,[1]+26*(iPeak-1)); + make_sf3d = true; - if (track_stats) - % Downweight higher frequency in all subTomos with iCCC below the mean - iBfactor = (flgQualityWeight.*(iCCC - maxCCC)./4) - iCCCweight = exp(iBfactor.*cccWeight); - else - if iCCC < avgCCC - % Downweight higher frequency in all subTomos with iCCC below the mean - iBfactor = -1.*(flgQualityWeight.*(acosd(iCCC) - acosd(avgCCC)))^2; - iCCCweight = exp(iBfactor.*cccWeight); - - else - iCCCweight=1; - end - - end + for iPeak = 1:emc.nPeaks - if ( any(flgFilterDefocus)) - iDef = abs(mean(tiltGeometry(:,15))*10^6); - iDef = -1.*(flgFilterDefocus(1)*max(iDef-1,0.5))^flgFilterDefocus(2); - fprintf('Using iDef %f\n',iDef); - % Frequency is already squared so adjust to match iDef scale - % factor. - iCCCweight = iCCCweight.*exp(iDef.*cccWeight.^(flgFilterDefocus(2)/2)); + if peakWgt(iPeak) == -9999 + positionList(iSubTomo, 26*iPeak) = -9999; + % Skip this peak + continue end - end - - % Find range to extract, and check for domain error. - if (flgCutOutVolumes && ~doCut) - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(2*CUTPADDING+sizeWindow, ... - sizeWindow, maskRadius, center); - else - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... - sizeWindow, maskRadius, center); - end - - - if ~ischar(indVAL) - + %Check that the given subTomo is not to be ignored - if (flgCutOutVolumes && ~doCut) - try - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iParticle = gpuArray(getVolume(MRCImage(particleOUT_name),... - [indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - catch - fprintf('\n\nDid not load cut out volume subTomo %d\n\n',iSubTomo); - continue; - end - else - - if ( loadTomo ) - iParticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3))); - - else - iParticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - end - - end - - - [ iParticle ] = BH_padZeros3d(iParticle, padVAL(1,1:3), ... - padVAL(2,1:3), 'GPU', 'single'); - - - - if (flgCutOutVolumes && doCut) - % Test with some generic padding , only to be used on bin 1 at - % first!!! TODO add a flag to check this. + % Get position and rotation info, angles stored as e1,e3,e2 as in AV3 + % and PEET. This also makes inplane shifts easier to see. - + center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; + angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); + wdgIDX = positionList(iSubTomo,9); - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - positionList(iSubTomo,[11:13]+26*(iPeak-1)) = shiftVAL+CUTPADDING+ceil((sizeWindow+1)./2); - if (projectVolumes) - SAVE_IMG(sum(iParticle,3),particleOUT_name,pixelSize); - else - particleOUT = BH_padZeros3d(gather(iParticle), CUTPADDING.*[1,1,1], ... - CUTPADDING.*[1,1,1], 'cpu', 'single'); - SAVE_IMG(particleOUT,particleOUT_name,pixelSize); - end + % tmpang = BH_defineMatrix([0,0,-14],'Bah','inv'); + % angles = reshape(angles,3,3)*tmpang; + % if (flgFinalAvg) + % angles = reshape(angles,3,3)*oddRot; + % end - - particleOUT =[]; - end - - - % Each particle in the average is rotate about it's origin and then - % shifted, and the the oddRot and shift is calculated between - % half-sets after the final iteration. Combining those transforms is - % not straightforward, so instead, apply the rotation, but calculate - % each individual shift. - iShift = shiftVAL; - - if (flgFinalAvg) - % Only the odd half needs to be searched - [~, iRefWdg] = interpolator(gpuArray(refWDG{1}),angles',[0,0,0], rotConvention , 'forward', 'C1', true); - [~, iRefIMG] = interpolator(gpuArray(refIMG{1}),angles',iShift, rotConvention , 'forward', 'C1', true); -% iRefIMG = BH_resample3d(refIMG{1},angles', iShift,rotConvention , 'GPU', 'forward'); -% iRefWdg = BH_resample3d(refWDG{1},angles', [0,0,0],rotConvention , 'GPU', 'forward'); + TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); - [ ref_FT ] = BH_bandLimitCenterNormalize(iRefIMG.*peakMask_tmp, ... - ifftshift(iSF3D), ... - peakBinary_tmp,padCalc,... - 'single'); - - [ part_FT ] = BH_bandLimitCenterNormalize(iParticle.*peakMask_tmp, ... - ifftshift(iRefWdg), ... - peakBinary_tmp,padCalc,... - 'single'); - - [ peakCoord ] = BH_multi_xcf_Translational(part_FT, ... - conj(ref_FT),... - peakMask_tmp, peakCOM); - - - fprintf('%d peakShift is %f %f %f\n', gather(iSubTomo), gather(peakCoord)); - iRefIMG = []; - iRefWdg = []; - part_FT = []; - ref_FT = []; - iShift = iShift + peakCoord; - end - - - - - - interpM = 'linear'; - interpU = 'GPU'; + if (make_sf3d) + [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); + make_sf3d = false; + end - - %iParticle is already on GPU if it should be. - [~, iParticle] = interpolator(gpuArray(iParticle), angles, iShift, rotConvention , 'inv', symmetry, true); - -% [ iParticle ] = gpuArray( ... -% BH_resample3d(iParticle, ... -% angles, iShift, ... -% {rotConvention ,symmetry,interpM,1,interpMask_tmpBinary}, ... -% interpU,'inv')); -% -% [ iParticle ] = BH_padZeros3d(iParticle, ... -% -1.*padWindow(1,:),-1.*padWindow(2,:),... -% 'GPU','single'); - - [~, iWedgeMask] = interpolator(gpuArray(iSF3D), angles, [0,0,0], rotConvention , 'inv', symmetry, true); - % For now just leave linear interp, but test with spline -% [ iWedgeMask ] = BH_resample3d(iSF3D, ... -% angles, [0,0,0], ... -% {rotConvention ,symmetry,'linear', ... -% 1,interpMaskWdg_tmp}, ... -% 'GPU','inv'); - - - iParticle = iParticle - mean(iParticle(interpMask_tmpBinary)); - iParticle = iParticle ./ rms(iParticle(interpMask_tmpBinary)); - iParticle = iParticle .* interpMask_tmp; - if (flgQualityWeight && numel(iCCCweight) > 1) - - iParticle = real(ifftn(fftn(BH_padZeros3d(iParticle,... - 'fwd',padCalc,'GPU','singleTaper')).*iCCCweight)); - - iParticle = BH_padZeros3d(iParticle,... - 'inv',padCalc,'GPU','singleTaper'); - iWedgeMask = iWedgeMask .* fftshift(iCCCweight); - end - - - - - trimAvg = mean(iParticle(:)); - - if ~isfinite(trimAvg) - fprintf('SubTomo %d from tomogram %s as Nan mean\n',... - iSubTomo, tomoList{iTomo}); - iParticle(:,:,:) = 0; - % Flag the particle as ignored - positionList(iSubTomo, 26:26:emc.nPeaks*26) = -9999; - else - - if (test_fuzz) - % Find the right row in the weight array - iProb_row = find(class_idx == positionList(iSubTomo,4)); - iProb = class_weights(iProb_row,:); - % iProb_weight = 4; - % iProb = iProb.^iProb_weight ./ sum(iProb.^iProb_weight); - iParticle = gather(iParticle .* peakWgt(iPeak)); - iWedgeMask = gather(iWedgeMask .* peakWgt(iPeak)); - - % Results are weird so override and see if it replicates the "normal" behavior by only taking the class with the minimum - % distance - [tmin,tidx] = min(iProb); - iProb = iProb.*0; - iProb(tidx) = 1; - - for iWeight = 1:length(iProb) - avgVolume_tmp{iWeight, positionList(iSubTomo,7)} = ... - avgVolume_tmp{iWeight, positionList(iSubTomo,7)} + (iParticle .* iProb(iWeight)); - - avgWedge_tmp{ iWeight, positionList(iSubTomo,7)} = ... - avgWedge_tmp{ iWeight, positionList(iSubTomo,7)} + (iWedgeMask .* iProb(iWeight)); - nExtracted_tmp(iWeight, positionList(iSubTomo,7)) = ... - nExtracted_tmp(iWeight, positionList(iSubTomo,7)) + iProb(iWeight); + + if (flgQualityWeight) + iCCC = positionList(iSubTomo,[1]+26*(iPeak-1)); + + if (track_stats) + % Downweight higher frequency in all subTomos with iCCC below the mean + iBfactor = (flgQualityWeight.*(iCCC - maxCCC)./4) + iCCCweight = exp(iBfactor.*cccWeight); + else + if iCCC < avgCCC + % Downweight higher frequency in all subTomos with iCCC below the mean + iBfactor = -1.*(flgQualityWeight.*(acosd(iCCC) - acosd(avgCCC)))^2; + iCCCweight = exp(iBfactor.*cccWeight); + + else + iCCCweight=1; + end + end - + + if ( any(flgFilterDefocus)) + iDef = abs(mean(tiltGeometry(:,15))*10^6); + iDef = -1.*(flgFilterDefocus(1)*max(iDef-1,0.5))^flgFilterDefocus(2); + fprintf('Using iDef %f\n',iDef); + % Frequency is already squared so adjust to match iDef scale + % factor. + iCCCweight = iCCCweight.*exp(iDef.*cccWeight.^(flgFilterDefocus(2)/2)); + end + end + + % Find range to extract, and check for domain error. + if (flgCutOutVolumes && ~doCut) + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid(2*CUTPADDING+sizeWindow, ... + sizeWindow, maskRadius, center); else - if positionList(iSubTomo,7) == 1 %%%%|| (flgGold == 0) - iTempParticleODD = iTempParticleODD + iParticle.*peakWgt(iPeak); - - iTempWedgeODD = iTempWedgeODD + iWedgeMask.*peakWgt(iPeak); - - if (flgEstSNR ) - for iSnr = [iClassPos:5:25] - nExtracted_tmp(iSnr, 1) = nExtracted_tmp(iSnr, 1) + 1; - end + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... + sizeWindow, maskRadius, center); + end + + + if ~ischar(indVAL) + + + if (flgCutOutVolumes && ~doCut) + try + particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); + iParticle = gpuArray(getVolume(MRCImage(particleOUT_name),... + [indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + catch + fprintf('\n\nDid not load cut out volume subTomo %d\n\n',iSubTomo); + continue; + end + else + + if ( loadTomo ) + iParticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... + indVAL(1,2):indVAL(2,2), ... + indVAL(1,3):indVAL(2,3))); + else - nExtracted_tmp(iClassPos, 1) = nExtracted_tmp(iClassPos, 1) + 1; + iParticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); end - elseif positionList(iSubTomo,7) == 2 %%%&& (flgGold) - iTempParticleEVE = iTempParticleEVE + iParticle.*peakWgt(iPeak); - iTempWedgeEVE = iTempWedgeEVE + iWedgeMask.*peakWgt(iPeak); - - if (flgEstSNR ) - for iSnr = [iClassPos:5:25] - nExtracted_tmp(iSnr, 2) = nExtracted_tmp(iSnr, 2) + 1; - end + + end + + + + [ iParticle ] = BH_padZeros3d(iParticle, padVAL(1,1:3), ... + padVAL(2,1:3), 'GPU', 'single'); + + + + if (flgCutOutVolumes && doCut) + % Test with some generic padding , only to be used on bin 1 at + % first!!! TODO add a flag to check this. + + + + particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); + positionList(iSubTomo,[11:13]+26*(iPeak-1)) = shiftVAL+CUTPADDING+ceil((sizeWindow+1)./2); + if (projectVolumes) + SAVE_IMG(sum(iParticle,3),particleOUT_name,pixelSize); else - nExtracted_tmp(iClassPos, 2) = nExtracted_tmp(iClassPos, 2) + 1; + particleOUT = BH_padZeros3d(gather(iParticle), CUTPADDING.*[1,1,1], ... + CUTPADDING.*[1,1,1], 'cpu', 'single'); + SAVE_IMG(particleOUT,particleOUT_name,pixelSize); end - else - error('Position List 7 must be a 1 or a 2') + + + particleOUT =[]; end - - if (eachTomo) - tomoAvg = tomoAvg + iParticle.*peakWgt(iPeak); - tomoWgt = tomoWgt + iWedgeMask.*peakWgt(iPeak); - tomoCount = tomoCount + 1; + + + % Each particle in the average is rotate about it's origin and then + % shifted, and the the oddRot and shift is calculated between + % half-sets after the final iteration. Combining those transforms is + % not straightforward, so instead, apply the rotation, but calculate + % each individual shift. + iShift = shiftVAL; + + if (flgFinalAvg) + % Only the odd half needs to be searched + [~, iRefWdg] = interpolator(gpuArray(refWDG{1}),angles',[0,0,0], rotConvention , 'forward', 'C1', true); + [~, iRefIMG] = interpolator(gpuArray(refIMG{1}),angles',iShift, rotConvention , 'forward', 'C1', true); + % iRefIMG = BH_resample3d(refIMG{1},angles', iShift,rotConvention , 'GPU', 'forward'); + % iRefWdg = BH_resample3d(refWDG{1},angles', [0,0,0],rotConvention , 'GPU', 'forward'); + + [ ref_FT ] = BH_bandLimitCenterNormalize(iRefIMG.*peakMask_tmp, ... + ifftshift(iSF3D), ... + peakBinary_tmp,padCalc,... + 'single'); + + [ part_FT ] = BH_bandLimitCenterNormalize(iParticle.*peakMask_tmp, ... + ifftshift(iRefWdg), ... + peakBinary_tmp,padCalc,... + 'single'); + + [ peakCoord ] = BH_multi_xcf_Translational(part_FT, ... + conj(ref_FT),... + peakMask_tmp, peakCOM); + + + fprintf('%d peakShift is %f %f %f\n', gather(iSubTomo), gather(peakCoord)); + iRefIMG = []; + iRefWdg = []; + part_FT = []; + ref_FT = []; + iShift = iShift + peakCoord; end - - end % end test fuzz - - end - - else - - fprintf('SubTomo %d from tomogram %s only sampled at %f\n',... - iSubTomo, tomoList{iTomo}, 1-padVAL); - - % Flag the particle as ignored - positionList(iSubTomo, 26:26:26*emc.nPeaks) = -9999; - nIgnored = nIgnored + 1; - peakWgt(1:emc.nPeaks) = -9999; - - end - % end - - end % loop over peaks + + + + + + interpM = 'linear'; + interpU = 'GPU'; + + + %iParticle is already on GPU if it should be. + [~, iParticle] = interpolator(gpuArray(iParticle), angles, iShift, rotConvention , 'inv', symmetry, true); + + % [ iParticle ] = gpuArray( ... + % BH_resample3d(iParticle, ... + % angles, iShift, ... + % {rotConvention ,symmetry,interpM,1,interpMask_tmpBinary}, ... + % interpU,'inv')); + % + % [ iParticle ] = BH_padZeros3d(iParticle, ... + % -1.*padWindow(1,:),-1.*padWindow(2,:),... + % 'GPU','single'); + + [~, iWedgeMask] = interpolator(gpuArray(iSF3D), angles, [0,0,0], rotConvention , 'inv', symmetry, true); + + % For now just leave linear interp, but test with spline + % [ iWedgeMask ] = BH_resample3d(iSF3D, ... + % angles, [0,0,0], ... + % {rotConvention ,symmetry,'linear', ... + % 1,interpMaskWdg_tmp}, ... + % 'GPU','inv'); + + + iParticle = iParticle - mean(iParticle(interpMask_tmpBinary)); + iParticle = iParticle ./ rms(iParticle(interpMask_tmpBinary)); + iParticle = iParticle .* interpMask_tmp; + if (flgQualityWeight && numel(iCCCweight) > 1) + + iParticle = real(ifftn(fftn(BH_padZeros3d(iParticle,... + 'fwd',padCalc,'GPU','singleTaper')).*iCCCweight)); + + iParticle = BH_padZeros3d(iParticle,... + 'inv',padCalc,'GPU','singleTaper'); + iWedgeMask = iWedgeMask .* fftshift(iCCCweight); + end + + + + + trimAvg = mean(iParticle(:)); + + if ~isfinite(trimAvg) + fprintf('SubTomo %d from tomogram %s as Nan mean\n',... + iSubTomo, tomoList{iTomo}); + iParticle(:,:,:) = 0; + % Flag the particle as ignored + positionList(iSubTomo, 26:26:emc.nPeaks*26) = -9999; + else + + if (test_fuzz) + % Find the right row in the weight array + iProb_row = find(class_idx == positionList(iSubTomo,4)); + iProb = class_weights(iProb_row,:); + % iProb_weight = 4; + % iProb = iProb.^iProb_weight ./ sum(iProb.^iProb_weight); + iParticle = gather(iParticle .* peakWgt(iPeak)); + iWedgeMask = gather(iWedgeMask .* peakWgt(iPeak)); + + % Results are weird so override and see if it replicates the "normal" behavior by only taking the class with the minimum + % distance + [tmin,tidx] = min(iProb); + iProb = iProb.*0; + iProb(tidx) = 1; + + for iWeight = 1:length(iProb) + avgVolume_tmp{iWeight, positionList(iSubTomo,7)} = ... + avgVolume_tmp{iWeight, positionList(iSubTomo,7)} + (iParticle .* iProb(iWeight)); + + avgWedge_tmp{ iWeight, positionList(iSubTomo,7)} = ... + avgWedge_tmp{ iWeight, positionList(iSubTomo,7)} + (iWedgeMask .* iProb(iWeight)); + nExtracted_tmp(iWeight, positionList(iSubTomo,7)) = ... + nExtracted_tmp(iWeight, positionList(iSubTomo,7)) + iProb(iWeight); + end + + else + if positionList(iSubTomo,7) == 1 %%%%|| (flgGold == 0) + iTempParticleODD = iTempParticleODD + iParticle.*peakWgt(iPeak); + + iTempWedgeODD = iTempWedgeODD + iWedgeMask.*peakWgt(iPeak); + + if (flgEstSNR ) + for iSnr = [iClassPos:5:25] + nExtracted_tmp(iSnr, 1) = nExtracted_tmp(iSnr, 1) + 1; + end + else + nExtracted_tmp(iClassPos, 1) = nExtracted_tmp(iClassPos, 1) + 1; + end + elseif positionList(iSubTomo,7) == 2 %%%&& (flgGold) + iTempParticleEVE = iTempParticleEVE + iParticle.*peakWgt(iPeak); + iTempWedgeEVE = iTempWedgeEVE + iWedgeMask.*peakWgt(iPeak); + + if (flgEstSNR ) + for iSnr = [iClassPos:5:25] + nExtracted_tmp(iSnr, 2) = nExtracted_tmp(iSnr, 2) + 1; + end + else + nExtracted_tmp(iClassPos, 2) = nExtracted_tmp(iClassPos, 2) + 1; + end + else + error('Position List 7 must be a 1 or a 2') + end + + if (eachTomo) + tomoAvg = tomoAvg + iParticle.*peakWgt(iPeak); + tomoWgt = tomoWgt + iWedgeMask.*peakWgt(iPeak); + tomoCount = tomoCount + 1; + end + + end % end test fuzz + + end + + else + + fprintf('SubTomo %d from tomogram %s only sampled at %f\n',... + iSubTomo, tomoList{iTomo}, 1-padVAL); + + % Flag the particle as ignored + positionList(iSubTomo, 26:26:26*emc.nPeaks) = -9999; + nIgnored = nIgnored + 1; + peakWgt(1:emc.nPeaks) = -9999; + + end + % end + + end % loop over peaks end % end of the loop over subTomos - - + + if ( flgEstSNR ) for iSnr = [iClassPos:5:25] avgVolume_tmp{ iSnr, 1} = avgVolume_tmp{ iSnr, 1} + ... - gather(iTempParticleODD); + gather(iTempParticleODD); avgVolume_tmp{ iSnr, 2} = avgVolume_tmp{ iSnr, 2} + ... - gather(iTempParticleEVE); - + gather(iTempParticleEVE); + avgWedge_tmp{ iSnr, 1} = avgWedge_tmp{ iSnr, 1} + ... - gather(iTempWedgeODD); + gather(iTempWedgeODD); avgWedge_tmp{ iSnr, 2} = avgWedge_tmp{ iSnr, 2} + ... - gather(iTempWedgeEVE); + gather(iTempWedgeEVE); end else % for the fuzz each particle goes into every class, so we need to pull earlier if ~(test_fuzz) avgVolume_tmp{ iClassPos, 1} = avgVolume_tmp{ iClassPos, 1} + ... - gather(iTempParticleODD); + gather(iTempParticleODD); avgVolume_tmp{ iClassPos, 2} = avgVolume_tmp{ iClassPos, 2} + ... - gather(iTempParticleEVE); - avgWedge_tmp{ iClassPos, 1} = avgWedge_tmp{ iClassPos, 1} + ... - gather(iTempWedgeODD); - avgWedge_tmp{ iClassPos, 2} = avgWedge_tmp{ iClassPos, 2} + ... - gather(iTempWedgeEVE); + gather(iTempParticleEVE); + avgWedge_tmp{ iClassPos, 1} = avgWedge_tmp{ iClassPos, 1} + ... + gather(iTempWedgeODD); + avgWedge_tmp{ iClassPos, 2} = avgWedge_tmp{ iClassPos, 2} + ... + gather(iTempWedgeEVE); end end end % end of loop over classes end% Update geometry to include information on ignored particles. geometry_tmp.(tomoList{iTomo})= positionList; - + if (eachTomo) tomoAvgStack{nTomos} = gather(tomoAvg./tomoCount); tomoWgtStack{nTomos} = gather(tomoWgt); nTomos = nTomos +1; end - + end % end of the loop over Tomograms - - + + avgResults{iParProc} = avgVolume_tmp; extResults{iParProc} = nExtracted_tmp; geoResults{iParProc} = geometry_tmp; cntResults{iParProc} = [nSubTomosTotal, nIgnored]; wgtResults{iParProc} = avgWedge_tmp; - + if (eachTomo) avgTomoResults{iParProc} = tomoAvgStack; @@ -1398,16 +1398,16 @@ end % end parfor loop on gpus - + nExtracted = zeros(maxClasses,2); avgVolume = cell(maxClasses,2); -avgWedge = cell(maxClasses,4); +avgWedge = cell(maxClasses,4); nSubTomosTotal = 0; nIgnored = 0; - -for iParProc = 1:nParProcesses +for iParProc = 1:nParProcesses + nExtracted = nExtracted + gather(extResults{iParProc}); nSubTomosTotal = nSubTomosTotal + cntResults{iParProc}(1,1); nIgnored = nIgnored + cntResults{iParProc}(1,2); @@ -1417,15 +1417,15 @@ for iHalf = 1:2-flgFinalAvg iHalf if iParProc == 1 - avgVolume{iVol, iHalf} = avgResults{iParProc}{iVol, iHalf}; - avgWedge{iVol, iHalf} = wgtResults{iParProc}{iVol, iHalf}; - - else + avgVolume{iVol, iHalf} = avgResults{iParProc}{iVol, iHalf}; + avgWedge{iVol, iHalf} = wgtResults{iParProc}{iVol, iHalf}; + + else avgVolume{iVol, iHalf} = avgVolume{iVol, iHalf} + ... - avgResults{iParProc}{iVol, iHalf}; + avgResults{iParProc}{iVol, iHalf}; avgWedge{iVol, iHalf} = avgWedge{iVol, iHalf} + ... - wgtResults{iParProc}{iVol, iHalf}; - + wgtResults{iParProc}{iVol, iHalf}; + end end end @@ -1438,25 +1438,25 @@ fprintf('\n%d / %d subTomos extracted\n',sum(nExtracted(:)), nSubTomosTotal) - + masterTM.(cycleNumber).('nSubTomoAveraged') = gather(sum(nExtracted(:))); masterTM.(cycleNumber).(sprintf('newIgnored_Avg%s',fieldPrefix)) = ... - gather(nIgnored); + gather(nIgnored); + - % get the total class average by combining eve/odd classStorage = cell(maxClasses,2); if (doNotTrim) && (flgClassify) filteredClass= cell(maxClasses,2); % low-pass to see class averages more clearly. -% % [ bandpassFilt ] = BH_bandpass3d( sizeMask, 0.2, 300, 30, 'GPU',pixelSize); + % % [ bandpassFilt ] = BH_bandpass3d( sizeMask, 0.2, 300, 30, 'GPU',pixelSize); end - + % % % if (doNotTrim) && (flgClassify) % % % % reduce z dimension to area focused on in classification % % % % taken from PCA mask values. @@ -1479,17 +1479,17 @@ for iClass = 1:maxClasses classStorage{iClass,1} = zeros(sizeMask, 'single'); classStorage{iClass,2} = zeros(sizeMask, 'single'); -% % % if (doNotTrim) && (flgClassify) -% % % filteredClass{iClass,1} = zeros(sizeFilteredClass, 'single'); -% % % filteredClass{iClass,2} = zeros(sizeFilteredClass, 'single'); -% % % end + % % % if (doNotTrim) && (flgClassify) + % % % filteredClass{iClass,1} = zeros(sizeFilteredClass, 'single'); + % % % filteredClass{iClass,2} = zeros(sizeFilteredClass, 'single'); + % % % end end if (eachTomo) [o,c] = system('mkdir -p initialTomoAvgs'); % sizeWeight mask is sizeMask or 128^3 whichever is larger -% bandpassFiltTomo = BH_bandpass3d( sizeCalc, lpTomo(1), lpTomo(2), lpTomo(3), 'GPU',pixelSize); - + % bandpassFiltTomo = BH_bandpass3d( sizeCalc, lpTomo(1), lpTomo(2), lpTomo(3), 'GPU',pixelSize); + for iParProc = 1:nParProcesses nTomos = 1; @@ -1497,10 +1497,10 @@ tomoName = sprintf('initialTomoAvgs/%s.mrc',tomoList{iTomo}); classTmp = avgTomoResults{iParProc}{nTomos}; avgTomoResults{iParProc}{nTomos} = []; - + SAVE_IMG(classTmp,tomoName,pixelSize); clear classTmp - nTomos = nTomos + 1; + nTomos = nTomos + 1; end end end @@ -1515,86 +1515,86 @@ end for iClassPos = 1:maxClasses - - if (doNotTrim) && (flgClassify) -% % % % % % % m = BH_mask3d('sphere',sizeMask,floor(sizeMask./2-6),pcaMaskCenter); - [ m ] = EMC_maskShape('sphere', sizeMask,floor(sizeMask./2-6), 'gpu', {'shift', pcaMaskCenter}); + + if (doNotTrim) && (flgClassify) + % % % % % % % m = BH_mask3d('sphere',sizeMask,floor(sizeMask./2-6),pcaMaskCenter); + [ m ] = EMC_maskShape('sphere', sizeMask,floor(sizeMask./2-6), 'gpu', {'shift', pcaMaskCenter}); - + + else + m = 1; + end + + % Re-weight both halves whether flgGold or not. + + for iGold = 1:2-flgFinalAvg + + avgVolume{iClassPos,iGold} = avgVolume{iClassPos,iGold} ./ ... + sum(nExtracted(iClassPos,iGold)); + + + end + + fprintf('flgGold = %d\n',flgGold); + for iGold = 1:2-flgFinalAvg + + % % % if( flgGold ) + if iGold == 1 + halfSet = 'ODD'; else - m = 1; + halfSet = 'EVE'; end + % % % else + % % % halfSet = 'STD'; + % % % end - % Re-weight both halves whether flgGold or not. - for iGold = 1:2-flgFinalAvg - - avgVolume{iClassPos,iGold} = avgVolume{iClassPos,iGold} ./ ... - sum(nExtracted(iClassPos,iGold)); - - - end + classStorage{iClassPos,iGold} = gather(avgVolume{iClassPos,iGold} ); - fprintf('flgGold = %d\n',flgGold); - for iGold = 1:2-flgFinalAvg - -% % % if( flgGold ) - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end -% % % else -% % % halfSet = 'STD'; -% % % end - - - classStorage{iClassPos,iGold} = gather(avgVolume{iClassPos,iGold} ); - - - if isnan( mean(classStorage{iClassPos,iGold}(:)) ) - clear classAVG - fprintf('zeroing out classavg because of NaN values detected.\n') - else -% % % if (doNotTrim) && (flgClassify) -% % % % lowpass according to Kms bandpass -% % % bandpassFilt = BH_bandpass3d( sizeMask, 0.1,300,30, 'GPU',pixelSize); -% % % -% % % tmpFilt = BH_bandLimitCenterNormalize( ... -% % % classStorage{iClassPos,iGold}.* ... -% % % m, bandpassFilt, (m>0.95), ... -% % % [0,0,0;0,0,0],'single'); -% % % tmpFilt = m.*real(ifftn(tmpFilt)); -% % % tmpFilt = tmpFilt(:,:,zLow:zTop ); -% % % tmpFilt = tmpFilt - mean(tmpFilt(:)); -% % % tmpFilt = tmpFilt ./ rms(tmpFilt(:)); -% % % -% % % filteredClass{iClassPos,iGold} = gather(tmpFilt); -% % % end - - % Normalize the regular averages - classStorage{iClassPos,iGold} = classStorage{iClassPos,iGold} - ... - mean(classStorage{iClassPos,iGold}(:)); - - classStorage{iClassPos,iGold} = classStorage{iClassPos,iGold} ./ ... - rms(classStorage{iClassPos,iGold}(:)); - classStorage{iClassPos,iGold} = gather(classStorage{iClassPos,iGold}); - - if (saveClassSum > -1) - fprintf('adding class %d to %d\n',iClassPos,iGold); - classSum{iGold} = classSum{iGold} + classStorage{iClassPos,iGold}; - classWgtSum{iGold} = classWgtSum{iGold} + avgWedge{iClassPos,iGold}; - end - end + + if isnan( mean(classStorage{iClassPos,iGold}(:)) ) + clear classAVG + fprintf('zeroing out classavg because of NaN values detected.\n') + else + % % % if (doNotTrim) && (flgClassify) + % % % % lowpass according to Kms bandpass + % % % bandpassFilt = BH_bandpass3d( sizeMask, 0.1,300,30, 'GPU',pixelSize); + % % % + % % % tmpFilt = BH_bandLimitCenterNormalize( ... + % % % classStorage{iClassPos,iGold}.* ... + % % % m, bandpassFilt, (m>0.95), ... + % % % [0,0,0;0,0,0],'single'); + % % % tmpFilt = m.*real(ifftn(tmpFilt)); + % % % tmpFilt = tmpFilt(:,:,zLow:zTop ); + % % % tmpFilt = tmpFilt - mean(tmpFilt(:)); + % % % tmpFilt = tmpFilt ./ rms(tmpFilt(:)); + % % % + % % % filteredClass{iClassPos,iGold} = gather(tmpFilt); + % % % end + % Normalize the regular averages + classStorage{iClassPos,iGold} = classStorage{iClassPos,iGold} - ... + mean(classStorage{iClassPos,iGold}(:)); - - + classStorage{iClassPos,iGold} = classStorage{iClassPos,iGold} ./ ... + rms(classStorage{iClassPos,iGold}(:)); + classStorage{iClassPos,iGold} = gather(classStorage{iClassPos,iGold}); + + if (saveClassSum > -1) + fprintf('adding class %d to %d\n',iClassPos,iGold); + classSum{iGold} = classSum{iGold} + classStorage{iClassPos,iGold}; + classWgtSum{iGold} = classWgtSum{iGold} + avgWedge{iClassPos,iGold}; + end end + + + + end + end @@ -1605,30 +1605,30 @@ classListOut = 0; % % % if (flgGold) && strcmpi(STAGEofALIGNMENT, 'Cluster') if strcmpi(STAGEofALIGNMENT, 'Cluster') && (flgClassify ~= -1) -% % % [classListOut, geometry] = reorder_classes(filteredClass(:,1),filteredClass(:,2),maxClasses, geometry); - -% % % filteredClass(:,2) = filteredClass(classListOut(:,2), 2); + % % % [classListOut, geometry] = reorder_classes(filteredClass(:,1),filteredClass(:,2),maxClasses, geometry); + + % % % filteredClass(:,2) = filteredClass(classListOut(:,2), 2); [classListOut, geometry] = reorder_classes(avgVolume(:,1),avgVolume(:,2),maxClasses, geometry); - + avgVolume(:,2) = avgVolume(classListOut(:,2), 2); avgWedge(:,2) = avgWedge(classListOut(:,2), 2); classMatches = fopen(sprintf('%s_class%d_%s_matches.txt', ... - outputPrefix,className,fieldPrefix),'w'); - fprintf(classMatches,'Score\t\tOdd\tEVE\n'); - fprintf(classMatches,'%d\t%d\t%2.6f\n', classListOut'); + outputPrefix,className,fieldPrefix),'w'); + fprintf(classMatches,'Score\t\tOdd\tEVE\n'); + fprintf(classMatches,'%d\t%d\t%2.6f\n', classListOut'); fclose(classMatches); masterTM.(cycleNumber).(sprintf('class_%d_%s_EveOddIdx',className,fieldPrefix)) = classListOut; - + end % Second option allows re-use of class designations to generate % multi-reference alignment. if strcmpi(STAGEofALIGNMENT, 'Cluster') masterTM.(cycleNumber).(ClusterGeomNAME) = geometry; -elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') && flgMultiRefAlignment +elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') && flgMultiRefAlignment if (flgClassify) masterTM.(cycleNumber).('ClusterClsGeom') = geometry; else @@ -1640,95 +1640,95 @@ % Should this save differently depending on the stage of alignment?? I % think so but leave alone for nw. - -for iGold = 1:2-flgFinalAvg -%%% if( flgGold ) - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end -%%% else -%%% halfSet = 'STD'; -%%% end - imgCounts = gather([classVector{iGold}(1,:) ; nExtracted(:,iGold)']); - - - [montOUT, imgLocations] = BH_montage4d(avgVolume(:,iGold), ''); - - - imout = sprintf('%s_class%d_%s_%s_NoWgt.mrc',outputPrefix, ... - className, fieldPrefix, halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', className,fieldPrefix, halfSet); - - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; - - if (flgFinalAvg) - system(sprintf('mv %s preHalfSetAli_%s',imout,imout)); - end - SAVE_IMG(montOUT, imout, pixelSize); - %%%%%%%% - [montOUT, imgLocations] = BH_montage4d(avgWedge(:,iGold), ''); - - imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... - className, fieldPrefix, halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_Wgt', className,fieldPrefix, halfSet); - - % For the weight, instead of imgCounts save the padValues - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,fscPAD}; - - if (flgFinalAvg) - system(sprintf('mv %s preHalfSetAli_%s',imout,imout)); - end - SAVE_IMG(montOUT, imout); - %%%%%%%% -% % % if (doNotTrim) && (flgClassify) -% % % [montOUT, ~] = BH_montage4d(filteredClass(:,iGold), ''); -% % % imout = sprintf('%s_filtered%d_%s_%s.mrc',outputPrefix, ... -% % % className, fieldPrefix, halfSet); -% % % SAVE_IMG(MRCImage(gather(montOUT)), imout,pixelSize); -% % % -% % % end - - if (saveClassSum > -1) - imgCounts = gather([classVector{iGold}(1,:) ; nExtracted(:,iGold)']); - - - [montOUT, imgLocations] = BH_montage4d(classSum(iGold), ''); - - imout = sprintf('%s_class%d_%s_%s_NoWgt.mrc',outputPrefix, ... - saveClassSum, 'Raw', halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'Raw', halfSet); - SAVE_IMG(montOUT, imout,pixelSize); - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; - [montOUT, imgLocations] = BH_montage4d(classWgtSum(iGold), ''); - - imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... - saveClassSum, 'Raw', halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'Raw', halfSet); - SAVE_IMG(montOUT, imout,pixelSize); - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; - - end +for iGold = 1:2-flgFinalAvg + %%% if( flgGold ) + if iGold == 1 + halfSet = 'ODD'; + else + halfSet = 'EVE'; + end + %%% else + %%% halfSet = 'STD'; + %%% end + imgCounts = gather([classVector{iGold}(1,:) ; nExtracted(:,iGold)']); + + + [montOUT, imgLocations] = BH_montage4d(avgVolume(:,iGold), ''); + + + imout = sprintf('%s_class%d_%s_%s_NoWgt.mrc',outputPrefix, ... + className, fieldPrefix, halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', className,fieldPrefix, halfSet); + + masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + + if (flgFinalAvg) + system(sprintf('mv %s preHalfSetAli_%s',imout,imout)); + end + SAVE_IMG(montOUT, imout, pixelSize); + %%%%%%%% + [montOUT, imgLocations] = BH_montage4d(avgWedge(:,iGold), ''); + + imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... + className, fieldPrefix, halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_Wgt', className,fieldPrefix, halfSet); + + % For the weight, instead of imgCounts save the padValues + masterTM.(cycleNumber).(classOut) = {imout,imgLocations,fscPAD}; + + if (flgFinalAvg) + system(sprintf('mv %s preHalfSetAli_%s',imout,imout)); + end + SAVE_IMG(montOUT, imout); + %%%%%%%% + % % % if (doNotTrim) && (flgClassify) + % % % [montOUT, ~] = BH_montage4d(filteredClass(:,iGold), ''); + % % % imout = sprintf('%s_filtered%d_%s_%s.mrc',outputPrefix, ... + % % % className, fieldPrefix, halfSet); + % % % SAVE_IMG(MRCImage(gather(montOUT)), imout,pixelSize); + % % % + % % % end + + if (saveClassSum > -1) + imgCounts = gather([classVector{iGold}(1,:) ; nExtracted(:,iGold)']); + + + [montOUT, imgLocations] = BH_montage4d(classSum(iGold), ''); + + imout = sprintf('%s_class%d_%s_%s_NoWgt.mrc',outputPrefix, ... + saveClassSum, 'Raw', halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'Raw', halfSet); + SAVE_IMG(montOUT, imout,pixelSize); + masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + + [montOUT, imgLocations] = BH_montage4d(classWgtSum(iGold), ''); + + imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... + saveClassSum, 'Raw', halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'Raw', halfSet); + SAVE_IMG(montOUT, imout,pixelSize); + masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + + end end - masterTM = gather(masterTM); - classVector = gather(classVector); - nExtracted = gather(nExtracted); - +masterTM = gather(masterTM); +classVector = gather(classVector); +nExtracted = gather(nExtracted); + subTomoMeta = gather(masterTM); subTomoMeta.(cycleNumber).('SymmetryApplied').(STAGEofALIGNMENT) = symmetry; subTomoMeta.(cycleNumber).('ClassVector').(STAGEofALIGNMENT) = classVector; -cycleNumber = gather(cycleNumber); +cycleNumber = gather(cycleNumber); subTomoMeta.('currentCycle') = gather(CYCLE); -save(emc.('subTomoMeta'), 'subTomoMeta'); +save(emc.('subTomoMeta'), 'subTomoMeta'); @@ -1743,8 +1743,8 @@ end -if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - BH_fscGold_class(PARAMETER_FILE, num2str(CYCLE), STAGEofALIGNMENT); +if strcmpi(STAGEofALIGNMENT, 'RawAlignment') + BH_fscGold_class(PARAMETER_FILE, num2str(CYCLE), STAGEofALIGNMENT); else error('This block should not be reached'); end @@ -1765,164 +1765,164 @@ if ~( flgEstSNR ) - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - masterTM = subTomoMeta; - %%%%%%%%%%%%%55 Reweight now that the FSC is calculated - - for iGold = 1:2 - %%% if( flgGold ) - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - %%% else - %%% halfSet = 'STD'; - %%% end - imgIN = sprintf('class_%d_Locations_%s_%s_NoWgt', ... - className, fieldPrefix, halfSet); - wgtIN = sprintf('class_%d_Locations_%s_%s_Wgt', ... - className, fieldPrefix, halfSet); - - [ refIMG{iGold} ] = BH_unStackMontage4d(1:maxClasses, ... - masterTM.(cycleNumber).(imgIN){1}, ... - masterTM.(cycleNumber).(imgIN){2},... - sizeWindow); - - [ refWGT{iGold} ] = BH_unStackMontage4d(1:maxClasses, ... - masterTM.(cycleNumber).(wgtIN){1},... - masterTM.(cycleNumber).(wgtIN){2},... - sizeCalc); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); + masterTM = subTomoMeta; + %%%%%%%%%%%%%55 Reweight now that the FSC is calculated + + for iGold = 1:2 + %%% if( flgGold ) + if iGold == 1 + halfSet = 'ODD'; + else + halfSet = 'EVE'; end - - - + %%% else + %%% halfSet = 'STD'; + %%% end + imgIN = sprintf('class_%d_Locations_%s_%s_NoWgt', ... + className, fieldPrefix, halfSet); + wgtIN = sprintf('class_%d_Locations_%s_%s_Wgt', ... + className, fieldPrefix, halfSet); + + [ refIMG{iGold} ] = BH_unStackMontage4d(1:maxClasses, ... + masterTM.(cycleNumber).(imgIN){1}, ... + masterTM.(cycleNumber).(imgIN){2},... + sizeWindow); + + [ refWGT{iGold} ] = BH_unStackMontage4d(1:maxClasses, ... + masterTM.(cycleNumber).(wgtIN){1},... + masterTM.(cycleNumber).(wgtIN){2},... + sizeCalc); + end + + + % This is slow ass when using cones and class averages and wouldn't be too % hard to put into parallel. Do that once the next manuscript is finished. - if (~flgMultiRefAlignment && ~flgClassify ) - nClassesReWgt = 1; + if (~flgMultiRefAlignment && ~flgClassify ) + nClassesReWgt = 1; + else + nClassesReWgt = maxClasses; + end + + for iRef = 1:nClassesReWgt + fprintf('Stage of alignment %s\niRef %d\n',STAGEofALIGNMENT,iRef); + if strcmpi(STAGEofALIGNMENT, 'RawAlignment') + savePrefix = 'Raw' else - nClassesReWgt = maxClasses; + savePrefix = fieldPrefix end - - for iRef = 1:nClassesReWgt - fprintf('Stage of alignment %s\niRef %d\n',STAGEofALIGNMENT,iRef); - if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - savePrefix = 'Raw' - else - savePrefix = fieldPrefix - end - - % When extracting class averages only a single FSC is initially available for - % calculation of the SPW filter. - if strcmpi(STAGEofALIGNMENT, 'Cluster') - iRefPrev = 1; - else - iRefPrev = iRef; - end - - if (flgGold) || (flgClassify < 0) - flgCombine = 0; - flgRefCutOff = 1; - else - flgCombine = 1; - flgRefCutOff = 0; - end - - - - - - % When switching from no Classification to classification, there will be no - % previous savePrefix ref, it will be Raw. Another reason to get rid of the - % (artificial) distinction. For now, use a try catch . - - try - fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRefPrev)); - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); - mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); - catch - fprintf('\nReverting from %s to Raw in loading fitFSC\n',savePrefix); - fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d','Raw',iRefPrev)); - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)); - mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d','Raw',iRefPrev)); - end - - % if (flgGold) && strcmpi(STAGEofALIGNMENT, 'Cluster') - % iOdd = cccFinal(iRef,2); - % iEve = cccFinal(iRef,3); - % else - iOdd = iRef; - iEve = iRef; - % end - - if (flgFinalAvg) - % negative to combine but NOT apply the xform to the odd set - flgCombine = -1; - flgRefCutoff = 0; - - end - - % Only send the lowest Bfactor if not flgFinalAvg - if (flgFinalAvg) - bFactorSend = fscBfactor; - else - bFactorSend = fscBfactor(1); - end + + % When extracting class averages only a single FSC is initially available for + % calculation of the SPW filter. + if strcmpi(STAGEofALIGNMENT, 'Cluster') + iRefPrev = 1; + else + iRefPrev = iRef; + end + + if (flgGold) || (flgClassify < 0) + flgCombine = 0; + flgRefCutOff = 1; + else + flgCombine = 1; + flgRefCutOff = 0; + end + + + + + + % When switching from no Classification to classification, there will be no + % previous savePrefix ref, it will be Raw. Another reason to get rid of the + % (artificial) distinction. For now, use a try catch . + + try + fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRefPrev)); + aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); + mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); + catch + fprintf('\nReverting from %s to Raw in loading fitFSC\n',savePrefix); + fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d','Raw',iRefPrev)); + aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)); + mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d','Raw',iRefPrev)); + end + + % if (flgGold) && strcmpi(STAGEofALIGNMENT, 'Cluster') + % iOdd = cccFinal(iRef,2); + % iEve = cccFinal(iRef,3); + % else + iOdd = iRef; + iEve = iRef; + % end + + if (flgFinalAvg) + % negative to combine but NOT apply the xform to the odd set + flgCombine = -1; + flgRefCutoff = 0; - refTMP = gather(BH_multi_cRef_Vnorm(fscParams, aliParams, mskParams,... - {refIMG{1}{iOdd},refIMG{2}{iEve}}, ... - {refWGT{1}{iOdd},refWGT{2}{iEve}}, ... - flgCombine,flgRefCutOff, pixelSize, bFactorSend)); - - if ~(flgFinalAvg) - refIMG{1}{iOdd} = refTMP{1,1}; - if (flgGold) || (flgClassify < 0) - refIMG{2}{iEve} = refTMP{1,2}; - end - clear refTMP + end + + % Only send the lowest Bfactor if not flgFinalAvg + if (flgFinalAvg) + bFactorSend = fscBfactor; + else + bFactorSend = fscBfactor(1); + end + + refTMP = gather(BH_multi_cRef_Vnorm(fscParams, aliParams, mskParams,... + {refIMG{1}{iOdd},refIMG{2}{iEve}}, ... + {refWGT{1}{iOdd},refWGT{2}{iEve}}, ... + flgCombine,flgRefCutOff, pixelSize, bFactorSend)); + + if ~(flgFinalAvg) + refIMG{1}{iOdd} = refTMP{1,1}; + if (flgGold) || (flgClassify < 0) + refIMG{2}{iEve} = refTMP{1,2}; end - + clear refTMP end - - - for iGold = 1:2-flgFinalAvg - if( flgGold ) || (flgClassify < 0) - if iGold == 1 - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end + + end + + + for iGold = 1:2-flgFinalAvg + if( flgGold ) || (flgClassify < 0) + if iGold == 1 + halfSet = 'ODD'; else - halfSet = 'STD'; + halfSet = 'EVE'; end - iRef - imgIN = sprintf('class_%d_Locations_%s_%s_NoWgt', ... - className, fieldPrefix, halfSet); - imgCounts = masterTM.(cycleNumber).(imgIN){3}; - % Save the unweighted, weighted imgs, weightes, optionally filtered. - - if (flgFinalAvg) - for iBfactor = 1:length(fscBfactor) - imout = sprintf('%s_class%d_%s_bFact-%d.mrc',outputPrefix, ... - className, 'final',fscBfactor(iBfactor)); - - SAVE_IMG(refTMP{iBfactor}, imout, pixelSize); - end - else - - [montOUT, imgLocations] = BH_montage4d(refIMG{iGold}(:), ''); - imout = sprintf('%s_class%d_%s_%s.mrc',outputPrefix, ... - className, fieldPrefix, halfSet); - classOut = sprintf('class_%d_Locations_%s_%s', className,fieldPrefix, halfSet); - - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; - SAVE_IMG(montOUT, imout, pixelSize); + else + halfSet = 'STD'; + end + iRef + imgIN = sprintf('class_%d_Locations_%s_%s_NoWgt', ... + className, fieldPrefix, halfSet); + imgCounts = masterTM.(cycleNumber).(imgIN){3}; + % Save the unweighted, weighted imgs, weightes, optionally filtered. + + if (flgFinalAvg) + for iBfactor = 1:length(fscBfactor) + imout = sprintf('%s_class%d_%s_bFact-%d.mrc',outputPrefix, ... + className, 'final',fscBfactor(iBfactor)); + + SAVE_IMG(refTMP{iBfactor}, imout, pixelSize); end - %%%%%%% + else + + [montOUT, imgLocations] = BH_montage4d(refIMG{iGold}(:), ''); + imout = sprintf('%s_class%d_%s_%s.mrc',outputPrefix, ... + className, fieldPrefix, halfSet); + classOut = sprintf('class_%d_Locations_%s_%s', className,fieldPrefix, halfSet); + + masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + SAVE_IMG(montOUT, imout, pixelSize); end - - + %%%%%%% + end + + subTomoMeta = masterTM; subTomoMeta.('CUTPADDING') = CUTPADDING; if (flgCutOutVolumes && doCut) @@ -1950,18 +1950,18 @@ eveNorm(iClass) = rms(eve{iClass}(:)); end -for iOdd = 1:maxClasses - ccc = zeros(1,length(classRemain)); - nCCC = 1; - for iEve = classRemain - ccc(nCCC) = sum(sum(sum(odd{iOdd}.*eve{iEve})))/(oddNorm(iOdd).*eveNorm(iEve)); - nCCC = nCCC + 1; - end - - [v,c] = max(ccc); - classListOut(iOdd,:) = [iOdd,classRemain(c),v]; - classRemain(c) = 0; - classRemain = classRemain(classRemain ~= 0); +for iOdd = 1:maxClasses + ccc = zeros(1,length(classRemain)); + nCCC = 1; + for iEve = classRemain + ccc(nCCC) = sum(sum(sum(odd{iOdd}.*eve{iEve})))/(oddNorm(iOdd).*eveNorm(iEve)); + nCCC = nCCC + 1; + end + + [v,c] = max(ccc); + classListOut(iOdd,:) = [iOdd,classRemain(c),v]; + classRemain(c) = 0; + classRemain = classRemain(classRemain ~= 0); end @@ -1971,8 +1971,8 @@ for iTomo = 1:length(tomoList) % select included even half includeList = (geometry.(tomoList{iTomo})(:,26) ~= -9999) & ... - geometry.(tomoList{iTomo})(:,7) == 2; - + geometry.(tomoList{iTomo})(:,7) == 2; + % Get positions for all classes classIDX = zeros(length(includeList),maxClasses); for iClass = 1:maxClasses @@ -1983,9 +1983,9 @@ classListOut(iClass,2) newClass = classIDX(:,classListOut(iClass,2)); geometry.(tomoList{iTomo})(logical(newClass),26) = iClass; - + end - + end end diff --git a/transformations/BH_axialSymmetry.m b/transformations/BH_axialSymmetry.m index 6e1dcc9b..9a4b18d8 100755 --- a/transformations/BH_axialSymmetry.m +++ b/transformations/BH_axialSymmetry.m @@ -1,5 +1,5 @@ function [ symIMG ] = BH_axialSymmetry( IMG, SYMMETRY, ANG_SHIFT, ... - METHOD, XYZ_SHIFT ) + METHOD, XYZ_SHIFT ) %Apply axial (z) symmetry to an image. % Goal is to generate either a symmetric reference that is more planar, or a % rotationally smeared out reference to use to speed up a grid search by first @@ -15,38 +15,38 @@ if SYMMETRY > 60 if ~(overrideRadial) flgRadial = true; - else + else flgRadial = true; end % Generating a rotationally smeared out reference, not just applying axial symmetry % Create a radial mask similar to ramp weight for WBP [ radialCylinder ,~,~,~,~,~ ] = BH_multi_gridCoordinates( size(IMG), ... - 'Cylindrical',... - METHOD, ... - {'none'}, ... - 0, 0, 0 ); + 'Cylindrical',... + METHOD, ... + {'none'}, ... + 0, 0, 0 ); else flgRadial = false; end - + symInc = 360/SYMMETRY; % An additional axial rotation to orient the reference, use positive sense % for indexing if ANG_SHIFT < 0 ANG_SHIFT = 360 + ANG_SHIFT; end - - if SYMMETRY > 6 - % Old slower way - symIMG = zeros(size(IMG), 'single','gpuArray'); - for iSym = 0:SYMMETRY-1 - symIMG = symIMG + BH_resample3d(IMG,[ANG_SHIFT+(iSym.*symInc),0,0],XYZ_SHIFT, 'Bah', 'GPU', 'forward'); - end - else - % More memory but faster - symIMG = BH_resample3d(IMG,[ANG_SHIFT,0,0],XYZ_SHIFT, {'Bah',SYMMETRY,'linear'}, 'GPU', 'forward'); - end - + + if SYMMETRY > 6 + % Old slower way + symIMG = zeros(size(IMG), 'single','gpuArray'); + for iSym = 0:SYMMETRY-1 + symIMG = symIMG + BH_resample3d(IMG,[ANG_SHIFT+(iSym.*symInc),0,0],XYZ_SHIFT, 'Bah', 'GPU', 'forward'); + end + else + % More memory but faster + symIMG = BH_resample3d(IMG,[ANG_SHIFT,0,0],XYZ_SHIFT, {'Bah',SYMMETRY,'linear'}, 'GPU', 'forward'); + end + else if SYMMETRY == -1 || SYMMETRY == -2 || SYMMETRY == -3 @@ -54,9 +54,9 @@ else error('mirror symmetry must be -1,-2,-3') end - -end +end + if (flgRadial) symIMG = real(ifftn(fftn(symIMG).*radialCylinder)); end diff --git a/transformations/BH_montage4d.m b/transformations/BH_montage4d.m index c90ad106..ecdd56c1 100755 --- a/transformations/BH_montage4d.m +++ b/transformations/BH_montage4d.m @@ -1,6 +1,6 @@ function [ MONTAGE, IMG_LOC ] = BH_montage4d( IMAGES , OUTPUT_PREFIX) %Create a 4d stack of 3d images saved as one 3d volume. -% +% % % Input variables: % @@ -22,7 +22,7 @@ % % TODO: % - check on image sizes -% - option for padding, for non-square montages, +% - option for padding, for non-square montages, % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -39,39 +39,39 @@ img1 = getVolume(MRCImage(IMAGES{1})); else img1 = IMAGES{1}; - + end imgSize = size(img1); -padSize = 0; % don't change, this is used to access class info, might could store it +padSize = 0; % don't change, this is used to access class info, might could store it clear img1 dx = imgSize(1); dy = imgSize(2); dz = imgSize(3); -ox = 1 + padSize; oy = 1 + padSize; oz = 1; +ox = 1 + padSize; oy = 1 + padSize; oz = 1; MONTAGE(nDim.*(dx+padSize)+padSize, nDim.*(dy+padSize)+padSize, dz) = single(0); % Track image locations. Return and store in the subTomoMeta file. imageLocations = cell(nVolumes,1); for iIMG = 1:nVolumes - - + + if isnumeric(IMAGES{iIMG}) img = IMAGES{iIMG}; - + else img = getVolume(MRCImage(IMAGES{iIMG})); - + end - - + + if size(img) ~= imgSize error('All subimages must have the same size.)') end - + if iIMG <= nVolumes - if mod(iIMG, nDim) + if mod(iIMG, nDim) imageLocations{iIMG} = [ox;ox + dx-1;oy;oy + dy-1;oz; oz + dz-1]; MONTAGE(ox: ox + dx-1, oy: oy + dy-1, oz: oz + dz-1) = img; ox = ox + dx + padSize; @@ -81,18 +81,18 @@ ox = 1 + padSize ; oy = oy + dy + padSize; end - + end end - - + + if ischar(IMAGES{1}) save(MRCImage(MONTAGE), sprintf('%s-mont.mrc', OUTPUT_PREFIX)) MONTAGE = '' end - -IMG_LOC = imageLocations; + +IMG_LOC = imageLocations; end % end of montage4d function diff --git a/transformations/BH_montage4d_halfGrid.m b/transformations/BH_montage4d_halfGrid.m index 294d2691..9901214b 100644 --- a/transformations/BH_montage4d_halfGrid.m +++ b/transformations/BH_montage4d_halfGrid.m @@ -1,6 +1,6 @@ function [ MONTAGE, IMG_LOC ] = BH_montage4d( IMAGES , OUTPUT_PREFIX, varargin) %Create a 4d stack of 3d images saved as one 3d volume. -% +% % % Input variables: % @@ -22,7 +22,7 @@ % % TODO: % - check on image sizes -% - option for padding, for non-square montages, +% - option for padding, for non-square montages, % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -52,27 +52,27 @@ % Track image locations. Return and store in the subTomoMeta file. imageLocations = cell(nVolumes,1); for iIMG = 1:nVolumes - + if isnumeric(IMAGES{iIMG}) MONTAGE(1 + (iIMG-1)*dx:dx + (iIMG-1)*dx,:,:) = gather(IMAGES{iIMG}); - IMAGES{iIMG} = []; + IMAGES{iIMG} = []; else MONTAGE(1 + (iIMG-1)*dx:dx + (iIMG-1)*dx,:,:) = getVolume(MRCImage(IMAGES{iIMG})); - + end - - - + + + imageLocations{iIMG} = [1 + (iIMG-1)*dx ;dx + (iIMG-1)*dx; ... - 1;dy;1;dz]; + 1;dy;1;dz]; end - + if ischar(IMAGES{1}) save(MRCImage(MONTAGE), sprintf('%s-mont.mrc', OUTPUT_PREFIX)) MONTAGE = ''; end - -IMG_LOC = imageLocations; + +IMG_LOC = imageLocations; end % end of montage4d function diff --git a/transformations/BH_multi_combineLowResInfo.m b/transformations/BH_multi_combineLowResInfo.m index 832d917f..51df4bca 100755 --- a/transformations/BH_multi_combineLowResInfo.m +++ b/transformations/BH_multi_combineLowResInfo.m @@ -6,7 +6,7 @@ nRefs = length(refIDX) [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates([512,512,512],... - 'Cartesian','cpu',{'none'},1,0,1); + 'Cartesian','cpu',{'none'},1,0,1); radialGrid = radialGrid ./ pixelSize; outputRefs = cell(2,1); @@ -16,37 +16,37 @@ for iRef = refIDX' % For today assume equal contributions, I think I already save this in the % meta data, so add this in soon. -% oddWeight = sum(nExtracted(iClassPos,1)) ./ sum(nExtracted(iClassPos,1:2)) -% eveWeight = sum(nExtracted(iClassPos,2)) ./ sum(nExtracted(iClassPos,1:2)) + % oddWeight = sum(nExtracted(iClassPos,1)) ./ sum(nExtracted(iClassPos,1:2)) + % eveWeight = sum(nExtracted(iClassPos,2)) ./ sum(nExtracted(iClassPos,1:2)) oddWeight = inputCounts{1}(2,iRef) ./ (inputCounts{1}(2,iRef) + inputCounts{2}(2,iRef)) eveWeight = inputCounts{2}(2,iRef) ./ (inputCounts{1}(2,iRef) + inputCounts{2}(2,iRef)) - + [ combPAD ] = BH_multi_padVal( size(inputRefs{1}{iRef}), 512 ); % Oversample so the cutoff is more accurate, and use double precision for the % same reason. oddPAD = fftn(BH_padZeros3d(inputRefs{1}{iRef}, ... - combPAD(1,:),combPAD(2,:),'cpu','doubleTaper')); + combPAD(1,:),combPAD(2,:),'cpu','doubleTaper')); evePAD = fftn(BH_padZeros3d(inputRefs{2}{iRef}, ... - combPAD(1,:),combPAD(2,:),'cpu','doubleTaper')); - + combPAD(1,:),combPAD(2,:),'cpu','doubleTaper')); + sharedInfo = (radialGrid < 1/resCutOff) .* (oddWeight.*oddPAD + eveWeight.*evePAD); - + oddPAD = real(ifftn(sharedInfo + (radialGrid >= 1/resCutOff).*oddPAD)); evePAD = real(ifftn(sharedInfo + (radialGrid >= 1/resCutOff).*evePAD)); clear sharedInfo oddWeight eveWeight - + outputRefs{1}{iRef} = single(oddPAD(1+combPAD(1,1):end-combPAD(2,1), ... - 1+combPAD(1,2):end-combPAD(2,2), ... - 1+combPAD(1,3):end-combPAD(2,3))); - clear oddPAD - + 1+combPAD(1,2):end-combPAD(2,2), ... + 1+combPAD(1,3):end-combPAD(2,3))); + clear oddPAD + outputRefs{2}{iRef} = single(evePAD(1+combPAD(1,1):end-combPAD(2,1), ... - 1+combPAD(1,2):end-combPAD(2,2), ... - 1+combPAD(1,3):end-combPAD(2,3))); + 1+combPAD(1,2):end-combPAD(2,2), ... + 1+combPAD(1,3):end-combPAD(2,3))); clear evePAD -end +end clear radialGrid inputRefs end diff --git a/transformations/BH_multi_loadOrBin.m b/transformations/BH_multi_loadOrBin.m index 3b9ca4ef..a8e7b820 100755 --- a/transformations/BH_multi_loadOrBin.m +++ b/transformations/BH_multi_loadOrBin.m @@ -1,5 +1,5 @@ function [ IMG_OUT, iPixelHeader, iOriginHeader, imgExt ] = ... - BH_multi_loadOrBin( IMG, SAMPLING,DIMENSION, varargin ) + BH_multi_loadOrBin( IMG, SAMPLING,DIMENSION, varargin ) %Check to see if a cached binned image exists, either load or bin and load. % Switched to using imod's newstack and binvol to create binning and % removed inline binning from my workflow. @@ -49,16 +49,16 @@ else doCalc = 1; end - + if (doCalc) - !mkdir -p cache - - + !mkdir -p cache + + switch DIMENSION case 3 system(sprintf('binvol -BinningFactor %d -antialias 6 %s cache/%s_bin%d%s > /dev/null', ... - samplingRate,IMG, imgName, samplingRate,imgExt)); + samplingRate,IMG, imgName, samplingRate,imgExt)); case 2 sprintf('%s',IMG) try @@ -70,17 +70,17 @@ end iHeader = getHeader(tiltObj); - outputName = (sprintf('cache/%s_bin%d%s',imgName, samplingRate,imgExt)); + outputName = (sprintf('cache/%s_bin%d%s',imgName, samplingRate,imgExt)); iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* samplingRate , ... - iHeader.cellDimensionY/iHeader.nY .* samplingRate, ... - iHeader.cellDimensionZ/iHeader.nZ .* samplingRate]; - + iHeader.cellDimensionY/iHeader.nY .* samplingRate, ... + iHeader.cellDimensionZ/iHeader.nZ .* samplingRate]; + iOriginHeader= [iHeader.xOrigin ./ samplingRate, ... - iHeader.yOrigin ./ samplingRate, ... - iHeader.zOrigin ./ samplingRate]; - - pixelSize = iHeader.cellDimensionX/iHeader.nX; % Assuming X/Y the same and Z might be incorrect. - + iHeader.yOrigin ./ samplingRate, ... + iHeader.zOrigin ./ samplingRate]; + + pixelSize = iHeader.cellDimensionX/iHeader.nX; % Assuming X/Y the same and Z might be incorrect. + [binSize,binShift] = BH_multi_calcBinShift([iHeader.nX,iHeader.nY],1,samplingRate); % Gridding correction for the interpolation in the binning. Not @@ -100,28 +100,28 @@ iProjection = bhF.invFFT(bhF.fwdFFT(R.*iProjection,0,0,[1e-6,600,samplingRate*pixelSize,pixelSize]),2); iProjection = BH_resample2d(iProjection,[0,0,0],binShift,'Bah','GPU','forward',1/samplingRate,binSize(1:2),bhF); - -% iProjection = real(ifftn(ifftshift(BH_padZeros3d(fftshift(fftn(iProjection)),'fwd',padVal,'GPU','singleTaper')))); -% - - + + % iProjection = real(ifftn(ifftshift(BH_padZeros3d(fftshift(fftn(iProjection)),'fwd',padVal,'GPU','singleTaper')))); + % + + newStack(:,:,iPrj) = gather(iProjection); end SAVE_IMG(newStack,outputName,iPixelHeader,iOriginHeader); clear newStack bpFilt iProjection -% system(sprintf('newstack -shrink %d -antialias 6 %s cache/%s_bin%d%s > /dev/null', ... -% samplingRate,IMG, imgName, samplingRate,imgExt)); + % system(sprintf('newstack -shrink %d -antialias 6 %s cache/%s_bin%d%s > /dev/null', ... + % samplingRate,IMG, imgName, samplingRate,imgExt)); otherwise error('DIMENSION should be 2 or 3\n.') end - - - end - + + + end + if (flgLoad) failedLoads = 0; @@ -129,9 +129,9 @@ try fprintf('pwd is %s\n', pwd); fprintf(... - 'attempting to load cache/%s_bin%d%s\n', imgName, samplingRate,imgExt); + 'attempting to load cache/%s_bin%d%s\n', imgName, samplingRate,imgExt); m = MRCImage(sprintf(... - 'cache/%s_bin%d%s', imgName, samplingRate,imgExt)); + 'cache/%s_bin%d%s', imgName, samplingRate,imgExt)); fprintf('Loaded the MRCImage\n'); IMG_OUT =getVolume(m); fprintf('Loaded the volume\n'); @@ -149,11 +149,11 @@ % syntax is only intended when resampling is required, so ignore the flag here % but throw a warning. fprintf('\n\nYou requested a sampling of -1 Nonsense!! loading anyway.\n\n'); - + IMG_OUT = single(getVolume(MRCImage(IMG))); end - + end diff --git a/transformations/BH_multi_loadOrBuild.m b/transformations/BH_multi_loadOrBuild.m index 1ff4dfc4..9cd567d1 100755 --- a/transformations/BH_multi_loadOrBuild.m +++ b/transformations/BH_multi_loadOrBuild.m @@ -1,7 +1,7 @@ function [ IMG_OUT, reconGeom ] = BH_multi_loadOrBuild(tomoName, ... - rCoords, mapBackIter, ... - SAMPLING, gpuIDX,... - reconScaling, varargin) + rCoords, mapBackIter, ... + SAMPLING, gpuIDX,... + reconScaling, varargin) %Check to see if a cached binned image exists, either load or reconstruct % Switched to using imod's newstack and binvol to create binning and % removed inline binning from my workflow. @@ -36,10 +36,10 @@ expand_lines = ''; if nargin > 8 if ~isempty(varargin{3}) - super_sample = varargin{3}; + super_sample = varargin{3}; end -% expand_lines = varargin{4} + % expand_lines = varargin{4} end !mkdir -p cache @@ -64,70 +64,70 @@ if SAMPLING > 1 - stack = sprintf('cache/%s_ali%d%s_bin%d.fixed',tomoName,mapBackIter+1,ctf,SAMPLING); - if ~exist(stack, 'file') - BH_multi_loadOrBin(checkStack,SAMPLING, 2); %%%%% med filt flag - end + stack = sprintf('cache/%s_ali%d%s_bin%d.fixed',tomoName,mapBackIter+1,ctf,SAMPLING); + if ~exist(stack, 'file') + BH_multi_loadOrBin(checkStack,SAMPLING, 2); %%%%% med filt flag + end else - - stack = sprintf('%sStacks/%s_ali%d%s.fixed',ali,tomoName,mapBackIter+1,ctf); -end + stack = sprintf('%sStacks/%s_ali%d%s.fixed',ali,tomoName,mapBackIter+1,ctf); +end + if exist(recon,'file') || ~doRecon header = getHeader(MRCImage(stack,0)); [ reconGeom ] = calc_rg( header, rCoords ); elseif (doRecon) - + if (mapBackIter) TLT = sprintf('mapBack%d/%s_ali%d_ctf.tlt',mapBackIter,tomoName,... - mapBackIter); + mapBackIter); LOCAL = sprintf('mapBack%d/%s_ali%d_ctf.local',mapBackIter,tomoName, ... - mapBackIter); - else + mapBackIter); + else TLT = sprintf('fixedStacks/%s.tlt',tomoName); LOCAL = sprintf('fixedStacks/%s.local',tomoName); end - + if exist(LOCAL,'file') flgLocal = 1; else fprintf('Did not find local alignment information at %s\n',LOCAL); flgLocal = 0; end - + % check to see if the binned stack exists and is readable [initialCheckFail,~] = system(sprintf('header %s',stack)); - + if (initialCheckFail) % See if the file exists but is being written by another process. if exist( stack, 'file') % It is there but possibly being written, run imod wait which throws an error if not growing. BH_imodWait(stack) else - + error('Did not find the full aligned stack at %s\n',stack); - + end - + else - + fprintf('Reconstructing from cached stack %s\n', stack); - - end - + + end + header = getHeader(MRCImage(stack)); - + if exist(recon,'file') fprintf('Using cached file %s\n', recon); [ reconGeom ] = calc_rg( header, rCoords ); else - - + + % Check that no out of bounds occur on slices - + if (rCoords(2) == 0) fprintf('shifting slices up 1\n') rCoords(2) = 1; @@ -139,33 +139,33 @@ rCoords(3) = header.nY-1; end end - + [ reconGeom ] = calc_rg( header, rCoords ); - + rCMD = sprintf(['-input %s -output %s -TILTFILE %s -UseGPU %d ', ... - '-WIDTH %d -SLICE %d,%d -THICKNESS %d -SHIFT %f,%f '],... - stack, recon, TLT, gpuIDX, rCoords(1:6)); - - - % Explicitly set Radial to Nyquist + '-WIDTH %d -SLICE %d,%d -THICKNESS %d -SHIFT %f,%f '],... + stack, recon, TLT, gpuIDX, rCoords(1:6)); + + + % Explicitly set Radial to Nyquist if (flgLocal) rCMD = [rCMD sprintf('-LOCALFILE %s -RotateBy90 -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%f',LOCAL,reconScaling)]; else rCMD = [rCMD sprintf('-RotateBy90 -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%f',reconScaling)]; - end - + end + if system('which tilt') error('Did not find IMOD tilt funciton on path') else - fprintf('Reconstructing from newly cached stack %s\n', stack); + fprintf('Reconstructing from newly cached stack %s\n', stack); fprintf('tilt %s %s %s\n',rCMD,super_sample,expand_lines) system(sprintf('tilt %s %s %s',rCMD,super_sample,expand_lines)); - end - + end + end else error('An appropriate reconstruction was not found for %s\n',recon) - + end if (strcmpi(recon,'tomoCPR')) @@ -176,7 +176,7 @@ try % fprintf('pwd is %s\n', pwd); % fprintf(... - % 'attempting to load %s\n', recon); + % 'attempting to load %s\n', recon); m = MRCImage(sprintf('%s', recon),0); % fprintf('Loaded the MRCImage\n'); if ( flgLoad ) @@ -201,10 +201,10 @@ function [ reconGeom ] = calc_rg( header, rCoords ) - + oY = ceil((header.nY +1)./2); nY = rCoords(3) - rCoords(2) + 1; - dY = floor(rCoords(2)+nY/2) -oY; + dY = floor(rCoords(2)+nY/2) -oY; reconGeom = zeros(2,3); reconGeom(1,1:3) = [rCoords(1), nY, rCoords(4)]; % value specify location of origin, but SHIFT in IMOD's tilt takes the diff --git a/transformations/BH_multi_loadOrCalcWeight.m b/transformations/BH_multi_loadOrCalcWeight.m index 44b02514..0f694695 100644 --- a/transformations/BH_multi_loadOrCalcWeight.m +++ b/transformations/BH_multi_loadOrCalcWeight.m @@ -1,73 +1,73 @@ function [ ] = BH_multi_loadOrCalcWeight(masterTM, ... - ctfGroupList, ... - tomoName, ... - samplingRate, ... - sizeCalc,... - geometry_tmp,... - cutPrecision,... - gpuIDX) + ctfGroupList, ... + tomoName, ... + samplingRate, ... + sizeCalc,... + geometry_tmp,... + cutPrecision,... + gpuIDX) %UNTITLED Summary of this function goes here % Detailed explanation goes here - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; - wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); - - try - calcWeight = 0; - iTiltMRCobj = MRCImage(wgtName,0); - iTiltHeader = getHeader(iTiltMRCobj); - - % Explicitly assuming the weights are cubic, st the z dimension of - % the montage corresponds to each weight dimension - if any(sizeCalc - iTiltHeader.nZ) - calcWeight = 1; - fprintf('the current weight with z = %d is not equal %d %d %d\n',... - iTiltHeader.NZ,sizeCalc); - else - fprintf('the wgt %s exists and will be used as is\n',wgtName); - end - catch - calcWeight = 1; - gDev = gpuDevice(gpuIDX); - - nCtfGroups = ctfGroupList.(tomoName)(1); - - system('mkdir -p cache'); - - fprintf('Reweighting.\n\n') - % Make a wedge mask that can be interpolated with no extrapolation for - % calculating wedge weighting in class average alignment. - kVal = 0; - - end - +iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; +wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); +try + calcWeight = 0; + iTiltMRCobj = MRCImage(wgtName,0); + iTiltHeader = getHeader(iTiltMRCobj); + + % Explicitly assuming the weights are cubic, st the z dimension of + % the montage corresponds to each weight dimension + if any(sizeCalc - iTiltHeader.nZ) + calcWeight = 1; + fprintf('the current weight with z = %d is not equal %d %d %d\n',... + iTiltHeader.NZ,sizeCalc); + else + fprintf('the wgt %s exists and will be used as is\n',wgtName); + end +catch + calcWeight = 1; + gDev = gpuDevice(gpuIDX); + + nCtfGroups = ctfGroupList.(tomoName)(1); + + system('mkdir -p cache'); + + fprintf('Reweighting.\n\n') + % Make a wedge mask that can be interpolated with no extrapolation for + % calculating wedge weighting in class average alignment. + kVal = 0; + +end - if (calcWeight) - wedgeMask = []; - - - fprintf('sizeCalc %d %d %d\n sizeWgt %d %d %d\n', sizeCalc, size(wedgeMask)); - % Keep tomo name as cell, so switching to explicit calc makes - % sense to me later. -% % % [ wedgeMask ] = BH_weightMask_dp(masterTM, OUTPUT(1,:), samplingRate,... -% % % {{tomoName},geometry_tmp}, ... -% % % cutPrecision, 'GPU'); - [ wedgeMask ] = BH_weightMask_dp(masterTM, sizeCalc.*[1,1,1], samplingRate,... - {{tomoName},geometry_tmp}, ... - cutPrecision, 'GPU'); - -% [montOUT, ~] = BH_montage4d_halfGrid(wedgeMask, ''); - [montOUT, ~] = BH_montage4d(wedgeMask, ''); +if (calcWeight) + wedgeMask = []; + + + fprintf('sizeCalc %d %d %d\n sizeWgt %d %d %d\n', sizeCalc, size(wedgeMask)); + % Keep tomo name as cell, so switching to explicit calc makes + % sense to me later. + + % % % [ wedgeMask ] = BH_weightMask_dp(masterTM, OUTPUT(1,:), samplingRate,... + % % % {{tomoName},geometry_tmp}, ... + % % % cutPrecision, 'GPU'); + [ wedgeMask ] = BH_weightMask_dp(masterTM, sizeCalc.*[1,1,1], samplingRate,... + {{tomoName},geometry_tmp}, ... + cutPrecision, 'GPU'); + + + % [montOUT, ~] = BH_montage4d_halfGrid(wedgeMask, ''); + [montOUT, ~] = BH_montage4d(wedgeMask, ''); + + SAVE_IMG(MRCImage(montOUT),wgtName); + montOUT = []; + for iCtfGroup = 1:nCtfGroups + wedgeMask{iCtfGroup} = []; + end +end - SAVE_IMG(MRCImage(montOUT),wgtName); - montOUT = []; - for iCtfGroup = 1:nCtfGroups - wedgeMask{iCtfGroup} = []; - end - end - end diff --git a/transformations/BH_reScale3d.m b/transformations/BH_reScale3d.m index f58f356d..f4374f17 100755 --- a/transformations/BH_reScale3d.m +++ b/transformations/BH_reScale3d.m @@ -3,7 +3,7 @@ % Detailed explanation goes here if (isnumeric(MAG)) - mag = MAG; + mag = MAG; else mag = EMC_str2double(MAG); end @@ -29,7 +29,7 @@ if isempty(imgPath) imgPath = '.'; end - % Read in the image + % Read in the image nVols = 1; mrcImage = MRCImage(inputVol,0); header = getHeader(mrcImage); @@ -43,7 +43,7 @@ inputVol = {getVolume(mrcImage')}; writeOut = true; outPutArray = false; - + end outputVol = cell(nVols,1); @@ -60,12 +60,12 @@ % % % end sizeOut = round(sizeVol.*mag); -% Bandlimit first and clear mask to save memory +% Bandlimit first and clear mask to save memory freqCutoff = mag.*0.475; bandPass = BH_bandpass3d(sizeVol,0,0,1/freqCutoff,METHOD,1); - + for iVol = 1:nVols - inputVol{iVol} = real(ifftn(fftn(inputVol{iVol}).*bandPass)); + inputVol{iVol} = real(ifftn(fftn(inputVol{iVol}).*bandPass)); end clear bandPass @@ -74,23 +74,23 @@ [~,~,~,x,y,z] = BH_multi_gridCoordinates(sizeVol,'Cartesian',METHOD,... - {'single',[1,0,0;0,1,0;0,0,1],... - xyzShift','forward',1,mag},0,1,0); + {'single',[1,0,0;0,1,0;0,0,1],... + xyzShift','forward',1,mag},0,1,0); [X,Y,Z,~,~,~] = BH_multi_gridCoordinates(sizeOut,'Cartesian',METHOD,... - {'single',[1,0,0;0,1,0;0,0,1],... - xyzShift','forward',1,mag},0,1,0); + {'single',[1,0,0;0,1,0;0,0,1],... + xyzShift','forward',1,mag},0,1,0); + - for iVol = 1:nVols if strcmp(METHOD,'GPU') outputVol{iVol} = interpn(x,y,z, inputVol{iVol},X,Y,Z ,'linear',0); else - outputVol{iVol} = interpn(x,y,z, inputVol{iVol},X,Y,Z ,'spline',0); + outputVol{iVol} = interpn(x,y,z, inputVol{iVol},X,Y,Z ,'spline',0); outputVol{iVol}(isnan(outputVol{iVol})) = 0; end - inputVol{iVol} = []; + inputVol{iVol} = []; end clear x y z X Y Z diff --git a/transformations/BH_resample2d.m b/transformations/BH_resample2d.m index 08479991..6c1e09f8 100755 --- a/transformations/BH_resample2d.m +++ b/transformations/BH_resample2d.m @@ -1,11 +1,11 @@ function [ TRANS_IMAGE ] = BH_resample2d( IMAGE, ANGLES, SHIFTS, ... - CONVENTION, METHOD, DIRECTION, ... - MAG, SIZEOUT, varargin) + CONVENTION, METHOD, DIRECTION, ... + MAG, SIZEOUT, varargin) %Transform an image in 3d. % -% +% % Input Variables: -% +% % IMAGE = 2d volume, or a string specifing a volume to read in. % % ANGLES = Euler angles defining the desired transformation @@ -29,7 +29,7 @@ % % Output Variables: % -% TRANS_IMAGE = the transformed image. +% TRANS_IMAGE = the transformed image. % Independent of the input image being passed in or read in from disk, the % output is an image in memory. % @@ -86,10 +86,10 @@ end if ischar(IMAGE) - % Read in the image - stackIN = getVolume(MRCImage(IMAGE),[],[],[]); + % Read in the image + stackIN = getVolume(MRCImage(IMAGE),[],[],[]); else - stackIN = IMAGE; clear IMAGE + stackIN = IMAGE; clear IMAGE end @@ -128,8 +128,8 @@ % The transformation from IMOD is a rotation matrix scaled by the mag, but the % shift values are also already scaled. transformation = {'sequential',R,SHIFTS(1,:)',DIRECTION,1,MAG(1) ; ... - 'sequential',R2,SHIFTS(2,:)',DIRECTION,1,MAG(2)}; - + 'sequential',R2,SHIFTS(2,:)',DIRECTION,1,MAG(2)}; + else error('ANGLES must be either three eulers or 9 rot matrix') end @@ -152,7 +152,7 @@ stackIN = BH_padZeros3d(stackIN,padLow,padHigh,'GPU','single'); else stackIN = BH_padZeros3d(stackIN,padLow,padHigh,'cpu','single'); - end + end end if doHalfGrid @@ -160,38 +160,38 @@ % grid indices. The latter makes getting the values along the origin % easier. [ Xnew,Ynew,~,x1,y1,~ ] = BH_multi_gridCoordinates( varargin{1}.inputSize, ... - 'Cartesian', ... - METHOD,transformation,... - 0, 1, 0,{'halfgrid'}); - - - + 'Cartesian', ... + METHOD,transformation,... + 0, 1, 0,{'halfgrid'}); + + + else [ Xnew,Ynew,~,x1,y1,~ ] = BH_multi_gridCoordinates( size(stackIN), ... - 'Cartesian', ... - METHOD,transformation,... - 0, shiftOrigin, 0 ); + 'Cartesian', ... + METHOD,transformation,... + 0, shiftOrigin, 0 ); end -if ~doHalfGrid +if ~doHalfGrid [ padVal ] = BH_multi_padVal( SIZEOUT, size(stackIN) ); cutLow = padVal(1,:); cutHigh= padVal(2,:); - + cutLow = cutLow .* (cutLow > 0); cutHigh = cutHigh.* (cutHigh > 0); Xnew = Xnew(cutLow(1) + 1:end - cutHigh(1), ... - cutLow(2) + 1:end - cutHigh(2)); - + cutLow(2) + 1:end - cutHigh(2)); + Ynew = Ynew(cutLow(1) + 1:end - cutHigh(1), ... - cutLow(2) + 1:end - cutHigh(2)); + cutLow(2) + 1:end - cutHigh(2)); end if (flgSeq) Xnew = Xnew{1}; Ynew = Ynew{1}; -end +end % Interpolate and write out the image. @@ -202,9 +202,9 @@ Xnew = Xnew ./ hgMAG; Ynew = Ynew ./ hgMAG; - + % Values from X < 0 can simply be conj(X > 0) - hermitianMates = Xnew < 0; + hermitianMates = Xnew < 0; % Values coming from X = 0 will not be correct if simply flipped. x_border_mask = Xnew < 1 & Xnew > -1; % Create an interpolant that also has +/- 2 @@ -224,41 +224,41 @@ % mates Xnew_border = Xnew(x_border_mask); Ynew_border = Ynew(x_border_mask); - + % Invert the coordinates, take conjugate after interpolating Xnew(hermitianMates) = -1.*Xnew(hermitianMates); Ynew(hermitianMates) = -1.*Ynew(hermitianMates); - - + + TRANS_IMAGE = interpn(x1,y1,stackIN,Xnew,Ynew,'linear',0); TRANS_IMAGE(hermitianMates) = conj(TRANS_IMAGE(hermitianMates)); % Now go back and replace the values that came from locations near x = % 0. TRANS_IMAGE(x_border_mask) = interpn(Xborder,Yborder,values_on_origin,Xnew_border, Ynew_border,'linear',0); - + clear values_on_origin Xborder Yborder x_border_mask - + if (hgMAG ~= 1 || hgSHIFTS(1) || hgSHIFTS(2)) isCentered=1; TRANS_IMAGE = varargin{1}.shiftStretch(TRANS_IMAGE,hgSHIFTS,hgMAG,isCentered); end - + if (returnComplex) - TRANS_IMAGE = varargin{1}.invSwap(TRANS_IMAGE); - + TRANS_IMAGE = varargin{1}.invSwap(TRANS_IMAGE); + else - TRANS_IMAGE = BH_padZeros3d( varargin{1}.invFFT(varargin{1}.invSwap(TRANS_IMAGE),2),... - trimVal(1,:),trimVal(2,:),'GPU','single'); + TRANS_IMAGE = BH_padZeros3d( varargin{1}.invFFT(varargin{1}.invSwap(TRANS_IMAGE),2),... + trimVal(1,:),trimVal(2,:),'GPU','single'); end - + else - + TRANS_IMAGE = interpn(x1,y1,stackIN,Xnew,Ynew,'linear',extrapVal); - + end else diff --git a/transformations/BH_resample3d.m b/transformations/BH_resample3d.m index 52c682e5..d0c75659 100755 --- a/transformations/BH_resample3d.m +++ b/transformations/BH_resample3d.m @@ -1,11 +1,11 @@ function [ TRANS_IMAGE, x1, y1, z1 ] = BH_resample3d( IMAGE, ANGLES, SHIFTS, ... - CONVENTION, METHOD, DIRECTION, ... - varargin) + CONVENTION, METHOD, DIRECTION, ... + varargin) %Transform an image in 3d. % -% +% % Input Variables: -% +% % IMAGE = 3d volume, or a string specifing a volume to read in. % % ANGLES = Single Volume: Euler angles defining the desired transformation @@ -13,7 +13,7 @@ % symmetrizing. % % Multi Volumes: Will return a stack of resampled images if ANGLES is -% a cell. In this case, the cell should be +% a cell. In this case, the cell should be % {nImages,2} = {Angles, shifts} % % then the SHIFTS variable should be a cell{1,1} with the 2x3 matrix @@ -38,7 +38,7 @@ % % Output Variables: % -% TRANS_IMAGE = the transformed image. +% TRANS_IMAGE = the transformed image. % Independent of the input image being passed in or read in from disk, the % output is an image in memory. % @@ -79,8 +79,8 @@ if ischar(IMAGE) - % Read in the image - IMAGE = getVolume(MRCImage(IMAGE)); + % Read in the image + IMAGE = getVolume(MRCImage(IMAGE)); end flgComplex = 0; @@ -93,19 +93,19 @@ preCalc = 0; flgComplexShift = 1; [dU,dV,dW] = BH_multi_gridCoordinates(size(IMAGE),'Cartesian',METHOD, ... - {'none'},1,1,0); + {'none'},1,1,0); elseif any(SHIFTS) preCalc = 1; flgComplexShift = 1; else preCalc = 0; flgComplexShift = 0; - end + end - % Set the direction of the shift by the sign of the phase change. + % Set the direction of the shift by the sign of the phase change. if strcmpi(DIRECTION,'forward') phaseDir = -1; - + elseif strcmpi(DIRECTION, 'inv') phaseDir = 1; else @@ -120,14 +120,14 @@ if ( flgComplexShift && preCalc) % Extra lines rather than re-assigning varargin to dU,dV,dW. IMAGE = IMAGE .* exp((phaseDir*2i*pi).*(varargin{2}{1}.*phaseShifts(1) + ... - varargin{2}{2}.*phaseShifts(2) + ... - varargin{2}{3}.*phaseShifts(3))); - + varargin{2}{2}.*phaseShifts(2) + ... + varargin{2}{3}.*phaseShifts(3))); + elseif ( flgComplexShift ) IMAGE = IMAGE .* exp((phaseDir*2i*pi).*(dU.*phaseShifts(1) + ... - dV.*phaseShifts(2) + ... - dW.*phaseShifts(3))); - + dV.*phaseShifts(2) + ... + dW.*phaseShifts(3))); + end end end @@ -144,7 +144,7 @@ symmetry = 1; mag = 1; if isa(CONVENTION, 'cell') - interpMethod = CONVENTION{3}; + interpMethod = CONVENTION{3}; symmetry = CONVENTION{2}; if (symmetry > 1) flgSymmetry = true; @@ -156,26 +156,26 @@ mag = CONVENTION{4}; end - + if length(CONVENTION) == 5 % a binary mask to limit the region searched volBinary = CONVENTION{5}; flgMask = 1; -% % % This is a wasted pre-allocation. Just zero outside interp mask in -% the output vol. -% % % if (useGPU) -% % % imOUT = zeros(size(IMAGE),'single','gpuArray'); -% % % else -% % % imOUT = zeros(size(IMAGE),'single'); -% % % end -% % % if (flgComplex) -% % % imOUT = complex(imOUT); -% % % end + % % % This is a wasted pre-allocation. Just zero outside interp mask in + % the output vol. + % % % if (useGPU) + % % % imOUT = zeros(size(IMAGE),'single','gpuArray'); + % % % else + % % % imOUT = zeros(size(IMAGE),'single'); + % % % end + % % % if (flgComplex) + % % % imOUT = complex(imOUT); + % % % end end CONVENTION = CONVENTION{1}; - - + + end @@ -198,44 +198,44 @@ end if (inputVectors) - if (flgSymmetry) - + if (flgSymmetry) + [ Xnew,Ynew,Znew,x1,y1,z1 ] = BH_multi_gridCoordinates( size(IMAGE), 'Cartesian', METHOD, ... - {'single',R(:),SHIFTS',DIRECTION,symmetry,mag,volBinary},... - 0, 1, 0, varargin{1} ); - + {'single',R(:),SHIFTS',DIRECTION,symmetry,mag,volBinary},... + 0, 1, 0, varargin{1} ); + else [ Xnew,Ynew,Znew,x1,y1,z1 ] = BH_multi_gridCoordinates( size(IMAGE), 'Cartesian', METHOD, ... - {'single',R(:),SHIFTS',DIRECTION,1,mag,volBinary},... - 0, 1, 0, varargin{1}); - end + {'single',R(:),SHIFTS',DIRECTION,1,mag,volBinary},... + 0, 1, 0, varargin{1}); + end else - if (flgSymmetry) - + if (flgSymmetry) + [ Xnew,Ynew,Znew,x1,y1,z1 ] = BH_multi_gridCoordinates( size(IMAGE), 'Cartesian', METHOD, ... - {'single',R(:),SHIFTS',DIRECTION,symmetry,mag,volBinary},... - 0, 1, 0 ); - + {'single',R(:),SHIFTS',DIRECTION,symmetry,mag,volBinary},... + 0, 1, 0 ); + else [ Xnew,Ynew,Znew,x1,y1,z1 ] = BH_multi_gridCoordinates( size(IMAGE), 'Cartesian', METHOD, ... - {'single',R(:),SHIFTS',DIRECTION,1,mag,volBinary},... - 0, 1, 0 ); - end + {'single',R(:),SHIFTS',DIRECTION,1,mag,volBinary},... + 0, 1, 0 ); + end end %%%%%%%%%%%% First interpolate the real part. If complex then also the %%%%%%%%%%%% imaginary. if (flgSymmetry) - + if strcmpi(interpMethod, 'spline') - fgrid = griddedInterpolant({x1,y1,z1},real(IMAGE), 'spline', 'none'); - TRANS_IMAGE = fgrid(Xnew{1}, Ynew{1},Znew{1}); - - TRANS_IMAGE(isnan(TRANS_IMAGE)) = 0; + fgrid = griddedInterpolant({x1,y1,z1},real(IMAGE), 'spline', 'none'); + TRANS_IMAGE = fgrid(Xnew{1}, Ynew{1},Znew{1}); + + TRANS_IMAGE(isnan(TRANS_IMAGE)) = 0; else - TRANS_IMAGE = interpn(x1,y1,z1,real(IMAGE),Xnew{1},Ynew{1},Znew{1},'linear',0); - + TRANS_IMAGE = interpn(x1,y1,z1,real(IMAGE),Xnew{1},Ynew{1},Znew{1},'linear',0); + end else @@ -246,26 +246,26 @@ symInc = 360/symmetry; for iSym = 2:symmetry - -% Rsym = R * BH_defineMatrix([0,0,iSym*symInc], CONVENTION, DIRECTION); -% [ Xnew,Ynew,Znew,x1,y1,z1 ] = BH_multi_gridCoordinates( size(stackIN), ... -% 'Cartesian', METHOD, ... -% {Rsym(:);SHIFTS';DIRECTION},... -% 0, 1, 0 ); - if strcmpi(interpMethod, 'spline') - symVol = fgrid(Xnew{iSym}, Ynew{iSym},Znew{iSym}); - - symVol(isnan(symVol)) = 0; - TRANS_IMAGE = TRANS_IMAGE + symVol; + + % Rsym = R * BH_defineMatrix([0,0,iSym*symInc], CONVENTION, DIRECTION); + % [ Xnew,Ynew,Znew,x1,y1,z1 ] = BH_multi_gridCoordinates( size(stackIN), ... + % 'Cartesian', METHOD, ... + % {Rsym(:);SHIFTS';DIRECTION},... + % 0, 1, 0 ); + if strcmpi(interpMethod, 'spline') + symVol = fgrid(Xnew{iSym}, Ynew{iSym},Znew{iSym}); + + symVol(isnan(symVol)) = 0; + TRANS_IMAGE = TRANS_IMAGE + symVol; else - - TRANS_IMAGE = TRANS_IMAGE + ... - interpn(x1,y1,z1,real(IMAGE),Xnew{iSym},Ynew{iSym},Znew{iSym},'linear',0); - end + TRANS_IMAGE = TRANS_IMAGE + ... + interpn(x1,y1,z1,real(IMAGE),Xnew{iSym},Ynew{iSym},Znew{iSym},'linear',0); + + end end -TRANS_IMAGE = TRANS_IMAGE ./ symmetry; clear firstVol symVol fgrid + TRANS_IMAGE = TRANS_IMAGE ./ symmetry; clear firstVol symVol fgrid end %%%%%%%%%%%% Imaginary @@ -276,66 +276,66 @@ % if symmetry applied, return a cell, with the first being the asymmetric, and % second being the symmetrized volume if strcmpi(interpMethod, 'spline') - fgrid = griddedInterpolant({x1,y1,z1},imag(IMAGE), 'spline', 'none'); - TRANS_IMAGE_Imag = fgrid(Xnew{1}, Ynew{1},Znew{1}); - TRANS_IMAGE_Imag(isnan(TRANS_IMAGE_Imag)) = 0; + fgrid = griddedInterpolant({x1,y1,z1},imag(IMAGE), 'spline', 'none'); + TRANS_IMAGE_Imag = fgrid(Xnew{1}, Ynew{1},Znew{1}); + TRANS_IMAGE_Imag(isnan(TRANS_IMAGE_Imag)) = 0; else - TRANS_IMAGE_Imag = interpn(x1,y1,z1,imag(IMAGE),Xnew{1},Ynew{1},Znew{1},'linear',0); - + TRANS_IMAGE_Imag = interpn(x1,y1,z1,imag(IMAGE),Xnew{1},Ynew{1},Znew{1},'linear',0); + end else - + TRANS_IMAGE_Imag = interpn(x1,y1,z1,imag(IMAGE),Xnew,Ynew,Znew,'linear',0); end - + if (flgSymmetry) symInc = 360/symmetry; - + for iSym = 2:symmetry - - % Rsym = R * BH_defineMatrix([0,0,iSym*symInc], CONVENTION, DIRECTION); - % [ Xnew,Ynew,Znew,x1,y1,z1 ] = BH_multi_gridCoordinates( size(stackIN), ... - % 'Cartesian', METHOD, ... - % {Rsym(:);SHIFTS';DIRECTION},... - % 0, 1, 0 ); - if strcmpi(interpMethod, 'spline') - symVol = fgrid(Xnew{iSym}, Ynew{iSym},Znew{iSym}); - symVol(isnan(symVol)) = 0; - TRANS_IMAGE_Imag = TRANS_IMAGE_Imag + symVol; + + % Rsym = R * BH_defineMatrix([0,0,iSym*symInc], CONVENTION, DIRECTION); + % [ Xnew,Ynew,Znew,x1,y1,z1 ] = BH_multi_gridCoordinates( size(stackIN), ... + % 'Cartesian', METHOD, ... + % {Rsym(:);SHIFTS';DIRECTION},... + % 0, 1, 0 ); + if strcmpi(interpMethod, 'spline') + symVol = fgrid(Xnew{iSym}, Ynew{iSym},Znew{iSym}); + symVol(isnan(symVol)) = 0; + TRANS_IMAGE_Imag = TRANS_IMAGE_Imag + symVol; else - - TRANS_IMAGE_Imag = TRANS_IMAGE_Imag + ... - interpn(x1,y1,z1,imag(IMAGE),Xnew{iSym},Ynew{iSym},Znew{iSym},'linear',0); - end + + TRANS_IMAGE_Imag = TRANS_IMAGE_Imag + ... + interpn(x1,y1,z1,imag(IMAGE),Xnew{iSym},Ynew{iSym},Znew{iSym},'linear',0); + end end - - TRANS_IMAGE_Imag = TRANS_IMAGE_Imag ./ symmetry; clear firstVol symVol fgrid + + TRANS_IMAGE_Imag = TRANS_IMAGE_Imag ./ symmetry; clear firstVol symVol fgrid end - + %for iSymImg = 1:1+flgSymmetry - - TRANS_IMAGE = complex(TRANS_IMAGE,TRANS_IMAGE_Imag); - TRANS_IMAGE_Imag = []; + + TRANS_IMAGE = complex(TRANS_IMAGE,TRANS_IMAGE_Imag); + TRANS_IMAGE_Imag = []; %end - + % A forward transform is shifted after rotation. if strcmpi(DIRECTION,'forward') for iSymImg = 1:1+flgSymmetry if ( flgComplexShift && preCalc) % Extra lines rather than re-assigning varargin to dU,dV,dW. TRANS_IMAGE = TRANS_IMAGE .* ... - exp((phaseDir*2i*pi).*(varargin{2}{1}.*phaseShifts(1) + ... - varargin{2}{2}.*phaseShifts(2) + ... - varargin{2}{3}.*phaseShifts(3))); - + exp((phaseDir*2i*pi).*(varargin{2}{1}.*phaseShifts(1) + ... + varargin{2}{2}.*phaseShifts(2) + ... + varargin{2}{3}.*phaseShifts(3))); + elseif ( flgComplexShift ) - - + + TRANS_IMAGE = TRANS_IMAGE .* ... - exp((phaseDir*2i*pi).*(dU.*phaseShifts(1) + ... - dV.*phaseShifts(2) + ... - dW.*phaseShifts(3))); - + exp((phaseDir*2i*pi).*(dU.*phaseShifts(1) + ... + dV.*phaseShifts(2) + ... + dW.*phaseShifts(3))); + end end end @@ -346,9 +346,9 @@ % needed and not in the padded area. In this case, return only a single image % and not a cell, and let it be the symmetric one if that is what is requested. if (flgMask) - - TRANS_IMAGE(~volBinary) = 0; - + + TRANS_IMAGE(~volBinary) = 0; + end dU = []; dV = []; dW = []; varargin = []; IMAGE = []; Xnew = []; Ynew = []; Znew = []; volBinary = []; imOUT = []; diff --git a/transformations/BH_unStackMontage4d.m b/transformations/BH_unStackMontage4d.m index bbfbc206..dbf91944 100755 --- a/transformations/BH_unStackMontage4d.m +++ b/transformations/BH_unStackMontage4d.m @@ -1,6 +1,6 @@ function [ STACK_OUT ] = BH_unStackMontage4d( IMAGES , NAME, LOCATIONS, sizeWINDOW ) %Unstack a 4d stack of 3d images saved as one 3d volume. -% +% % % Input variables: % @@ -42,11 +42,11 @@ if isa(ind, 'cell') nVolumes = length(ind); sizeRef = ind{1}(2:2:6)'; - + if any(sizeRef - sizeWINDOW < 0) fprintf('SIZE_REF %d %d %d\nSIZE_WINDOW %d %d %d', sizeRef, sizeWINDOW); error(['The stored reference is too small to be masked appopriately,' , ... - 'choose a smaller mask radius or find a better solution.\n']) + 'choose a smaller mask radius or find a better solution.\n']) else refTrim = BH_multi_padVal( sizeWINDOW, sizeRef ); end @@ -65,9 +65,9 @@ for iY = 1:nY for iX = 1:nX ind{iVol} = [1 + (iX-1).* iDim,(iX) .* iDim, ... - 1 + (iY-1).* iDim,(iY) .* jDim, ... - 1,size(montage,3) ]; - + 1 + (iY-1).* iDim,(iY) .* jDim, ... + 1,size(montage,3) ]; + iVol = iVol + 1; end end @@ -78,30 +78,30 @@ STACK_OUT = cell(nVolumes,1); for iPos = 1:nVolumes - + if ismember(iPos, IMAGES) - + iIMG = iPos; - + STACK_OUT{iPos} = montage(ind{iIMG}(1)+refTrim(1,1) : ind{iIMG}(2)-refTrim(2,1), ... - ind{iIMG}(3)+refTrim(1,2) : ind{iIMG}(4)-refTrim(2,2), ... - ind{iIMG}(5)+refTrim(1,3) : ind{iIMG}(6)-refTrim(2,3)); - + ind{iIMG}(3)+refTrim(1,2) : ind{iIMG}(4)-refTrim(2,2), ... + ind{iIMG}(5)+refTrim(1,3) : ind{iIMG}(6)-refTrim(2,3)); + %figure, imshow3D(gather(STACK_OUT{iPos})), pause(5); close(gcf) else - + STACK_OUT{iPos} = []; end - - -end - +end + + + + + - - end % end of unStackMontage4d function diff --git a/transformations/BH_unStackMontage4d_halfGrid.m b/transformations/BH_unStackMontage4d_halfGrid.m index ea40ade5..c0a62d62 100644 --- a/transformations/BH_unStackMontage4d_halfGrid.m +++ b/transformations/BH_unStackMontage4d_halfGrid.m @@ -1,6 +1,6 @@ function [ STACK_OUT ] = BH_unStackMontage4d_halfGrid( IMAGES , NAME, totalNumber) %Unstack a 4d stack of 3d images saved as one 3d volume. -% +% % % Input variables: % @@ -45,16 +45,16 @@ error('The first dimension of the montage is not evenly divisible by the expected number of images.'); else d1 = d1 / totalNumber; -end +end nIMG = 1; for iPos = IMAGES - + STACK_OUT{nIMG} = montage(1+(iPos-1)*d1:d1 +(iPos-1)*d1,:,:); nIMG = nIMG + 1; - -end +end + clear montage end % end of unStackMontage4d function From a202e1bb4fd9a56a6e9996715fde9776b2f04a1a Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 10:07:17 -0500 Subject: [PATCH 011/151] Continue moving optional parameters and also create EMC_assert functions to check for correctness. --- alignment/BH_alignRaw3d_v2.m | 51 +--- logicals/EMC_assert_boolean.m | 15 ++ logicals/EMC_assert_integer.m | 29 +++ logicals/EMC_assert_numeric.m | 29 +++ masking/BH_weightMask_dp.m | 422 ------------------------------- metaData/BH_combineProjects.m | 2 - metaData/BH_geometryAnalysis.m | 4 +- metaData/BH_geometryInitialize.m | 24 +- metaData/BH_parseParameterFile.m | 63 ++++- statistics/BH_pcaPub.m | 19 +- transformations/BH_average3d.m | 124 ++++----- 11 files changed, 190 insertions(+), 592 deletions(-) create mode 100644 logicals/EMC_assert_boolean.m create mode 100644 logicals/EMC_assert_integer.m create mode 100644 logicals/EMC_assert_numeric.m delete mode 100755 masking/BH_weightMask_dp.m diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index ba0be5d6..eb037ffd 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -70,19 +70,6 @@ reconScaling = 1; -try - track_stats = emc.('track_stats'); -catch - track_stats = false; -end - -try - flgCutOutVolumes=emc.('flgCutOutVolumes') -catch - flgCutOutVolumes=0 -end - - % TODO decide on a "reasonable" padding based on expected shifts. try CUTPADDING = subTomoMeta.('CUTPADDING') @@ -190,17 +177,6 @@ rotConvention = 'Helical' end -rotConvention - -try - bFactor = emc.('Fsc_bfactor'); -catch - bFactor = 0; -end -if length(bFactor) > 1 - fprintf('multiple bFactors specified, using the first for alignment.\n'); - bFactor = bFactor(1); -end try scaleCalcSize = emc.('scaleCalcSize'); @@ -279,7 +255,6 @@ tomoList = fieldnames(geometry); nTomograms = length(tomoList); tiltList = masterTM.tiltGeometry; -ctfGroupList = masterTM.('ctfGroupSize'); % % Sort the list by number of active subtomos to improve parallelism % sortedTomoList = zeros(nTomograms,1); @@ -482,12 +457,12 @@ if (eraseMask) peakMask = EMC_maskShape(eraseMaskType,sizeCalc,floor(eraseMaskRadius),'cpu',{'kernel',false}); - if track_stats + if ( emc.track_stats ) stat_mask = single(find(peakMask > 0.95)); end else - if track_stats + if ( emc.track_stats ) stat_mask = EMC_maskShape('sphere', sizeCalc, [1,1,1].*floor(max(peakSearch)), 'cpu', {'shift', maskCenter}); stat_mask = single(find(stat_mask > 0.95)); end @@ -527,9 +502,9 @@ radialGrid = single(radialGrid./pixelSize); % returns a cpu array if (flgWeightCCC) - [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1); + [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1, 1); else - [ bandpassFilt{iRef}, ~] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1); + [ bandpassFilt{iRef}, ~] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1); end @@ -547,9 +522,9 @@ radialGrid = single(radialGrid./pixelSize); % returns a cpu array if (flgWeightCCC) - [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1, 1 ); + [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1, 1 ); else - [ bandpassFilt{iRef},~ ] = BH_multi_cRef( fscINFO, radialGrid, bFactor, 1 ); + [ bandpassFilt{iRef},~ ] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1 ); end bandpassFiltREF{iRef} = 1; @@ -795,7 +770,7 @@ peakMaskInterpolator = ''; peakMaskInterpolator = interpolator(gpuArray(peakMask),[0,0,0],[0,0,0], rotConvention , 'forward', 'C1', false); - if (track_stats) + if (emc.track_stats) mip = struct(); mip.('mask') = gpuArray(stat_mask); end @@ -869,7 +844,7 @@ reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); - if (flgCutOutVolumes) + if (emc.flgCutOutVolumes) volumeData = []; else [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... @@ -911,7 +886,7 @@ for iPeak = 1:emc.nPeaks - if (track_stats) + if (emc.track_stats) measure_noise = true; mip.('x') = {}; mip.('x2') = {}; @@ -954,7 +929,7 @@ angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); % Find range to extract, and check for domain error. - if (flgCutOutVolumes) + if (emc.flgCutOutVolumes) % Need some check that the windowsize has not changed! TODO TODO [ indVAL, padVAL, shiftVAL ] = ... @@ -976,7 +951,7 @@ else - if (flgCutOutVolumes) + if (emc.flgCutOutVolumes) % Test with some generic padding , only to be used on bin 1 at % first!!! TODO add a flag to check this. try @@ -1223,7 +1198,7 @@ iTrimParticle,... bandpassFilt_tmp{iRef} ,'',padCalc,flgPrecision); - if (track_stats && measure_noise) + if (emc.track_stats && measure_noise) [ ~, mip ] = BH_multi_xcf_Translational_2( ... @@ -1687,7 +1662,7 @@ end - if (track_stats) + if (emc.track_stats) if thetaInc > 0 cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); diff --git a/logicals/EMC_assert_boolean.m b/logicals/EMC_assert_boolean.m new file mode 100644 index 00000000..d08cf4f0 --- /dev/null +++ b/logicals/EMC_assert_boolean.m @@ -0,0 +1,15 @@ +function EMC_assert_boolean(input_val) + + if ( isa(input_val, 'logical') ) + return; + else + if ( isa(input_val, 'numeric') ) + if ( (input_val == 0) || (input_val == 1) ) + return; + else + error('EMC_assert_boolean: input_val is numeric but not 0 or 1 to represent false or true'); + end + end + end + +end \ No newline at end of file diff --git a/logicals/EMC_assert_integer.m b/logicals/EMC_assert_integer.m new file mode 100644 index 00000000..e5733a25 --- /dev/null +++ b/logicals/EMC_assert_integer.m @@ -0,0 +1,29 @@ +function EMC_assert_integer(input_val, varargin) + + % Use the default for any length + assert_length = false; + assert_passed = false; + if ( nargin == 2 ) + assert_length = true; + wanted_numel = varargin{1}; + elseif ( nargin > 2 ) + error('EMC_assert_numeric: too many input arguments'); + end + + if ( isa(input_val, 'integer') ) + if ( assert_length ) + if ( numel(input_val) == wanted_numel ) + assert_passed = true; + end + else + assert_passed = true; + end + return; + end + + if ( assert_passed == false ) + error('EMC_assert_integer: input is not an integer or has wrong number of elements'); + end + + +end \ No newline at end of file diff --git a/logicals/EMC_assert_numeric.m b/logicals/EMC_assert_numeric.m new file mode 100644 index 00000000..dbde3505 --- /dev/null +++ b/logicals/EMC_assert_numeric.m @@ -0,0 +1,29 @@ +function EMC_assert_numeric(input_val, varargin) + + % Use the default for any length + assert_length = false; + assert_passed = false; + if ( nargin == 2 ) + assert_length = true; + wanted_numel = varargin{1}; + elseif ( nargin > 2 ) + error('EMC_assert_numeric: too many input arguments'); + end + + if ( isa(input_val, 'numeric') ) + if ( assert_length ) + if ( numel(input_val) == wanted_numel ) + assert_passed = true; + end + else + assert_passed = true; + end + return; + end + + if ( assert_passed == false ) + error('EMC_assert_numeric: input is not numeric or has wrong number of elements'); + end + + +end \ No newline at end of file diff --git a/masking/BH_weightMask_dp.m b/masking/BH_weightMask_dp.m deleted file mode 100755 index c1287281..00000000 --- a/masking/BH_weightMask_dp.m +++ /dev/null @@ -1,422 +0,0 @@ -function [ ctfWeights ] = BH_weightMask_dp(subTomoMeta, SIZE, SAMPLING,... - GEOMETRY, PRECISION, METHOD,... - varargin) -% Calculate the sampling of a number of subtomograms. -% -% If tiltGeometry, reconGeometry, and sTgeometry are structs, then calculate the -% weights for the full data set. Otherwise calculate a subset of weights -ctfScaleFactor = 1 -if nargin > 6 - ctfScaleFactor = varargin{1}; -end -OverSIZE = 512; - -%512 -% % % if all(SIZE > 0) -% % % outputScaling = SIZE(1)/(512); -% % % SIZE = OverSIZE.*[1,1,1];%[512,512,512] ; -% % % else -% % % error('Use regular weightmask for template matching for now'); -% % % %optional override for template matching which may not always be cubic, wich -% % % %trades a little accuracy in the ctf mask for speed. -% % % % SIZE = abs(SIZE); -% % % % outputScaling = 1; -% % % end -outputScaling = false; -flgFirstPass = 1; - -% Use this to control whether a group of ctfs for a single tomogram are made, or -% alternatively weights for the whole data set are calculated. -tomoList = GEOMETRY{1}; -geometryFull = GEOMETRY{2}; -nTomos = length(tomoList); - -if (nTomos > 1) - calcAllWeights = 1 - nCtfGroups = 1; - ctfWeights = cell(2,1); -else - calcAllWeights = 0; - nCtfGroups = subTomoMeta.('ctfGroupSize').(tomoList{1})(1); - ctfWeights = cell(1,nCtfGroups); - -end - -% Calculate at 512^3 and then reduce, so loop over nCtfGroups, pulling to main -% memory as you go. -% % % [ rWeight ] = calc_rWeight( SIZE, PRECISION, METHOD); - -TLT = subTomoMeta.('tiltGeometry').(tomoList{1}); -pixelSize = TLT(1,16).*SAMPLING; - - -% % % [radialCTFCalc, phiCTFCalc, ~, ~, ~, ~ ] = ... -% % % BH_multi_gridCoordinates(SIZE(1:2),'Cylindrical', ... -% % % METHOD, {'none'},1,1,0); -% % % -% % % -% % % % This should be the default. -% % % radialCTFCalc = {radialCTFCalc ./ pixelSize,1,phiCTFCalc}; - - -for iCtfGroup = 1:nCtfGroups - rec = cell(1+calcAllWeights,1); - if strcmp(METHOD, 'GPU') - rec{1} = zeros(SIZE,PRECISION,'gpuArray'); - if (calcAllWeights) - rec{2} = zeros(SIZE,PRECISION,'gpuArray'); - end - useGPU = 1; - SIZE = gpuArray(SIZE); - elseif strcmp(METHOD, 'cpu') - rec{1} = zeros(SIZE,PRECISION); - if (calcAllWeights) - rec{2} = zeros(SIZE,PRECISION); - end - useGPU = 0; - else - error('METHOD must be GPU or %s\n', 'cpu'); - end - for iTomo = 1:1+(nTomos-1)*(calcAllWeights) - - - % % % reconGeometry = subTomoMeta.('reconGeometry').(tomoList{iTomo})./SAMPLING; - reconGeometry = subTomoMeta.('reconGeometry').(tomoList{iTomo}); - - reconShift = reconGeometry(2,:); - - % Also there are no offsets here, but this should be considered as in - % syntheticMapback - also need to update things to save the tilt header. - % Possibly just calculate the respective origins and lowerLeft vol as part of - % the initialization. - - % % originPrj = subTomoMeta.('tiltGeometry').(tomoList{iTomo})(1,20:22)./SAMPLING; - % % % originPrj = ceil((originPrj+1)./2); - originPrj = subTomoMeta.('tiltGeometry').(tomoList{iTomo})(1,20:22); - - % % % originPrj(3) = 1; - - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - - - positionList = geometryFull.(tomoList{iTomo}); - positionList = positionList(positionList(:,26)~=-9999,:); - TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); - nPrjs = size(TLT,1); - nSubTomos = size(positionList,1) ; - - zCoords = sort(positionList(:,13)); - - - % The thickness in the tomogram is NOT the thickness in the projection - % if the specimen is tilted at zero-tilt angle. Need to think of a - % better way to calculate this. - % % if (length(zCoords) < 5) - % % iThickness = 100 - % % else - % % iThickness = iqr(zCoords).*pixelSize*10^9 - % % % meanMax = mean(zCoords(1:5)).*pixelSize*10^9; - % % % meanMin = mean(zCoords(end-5:end)).*pixelSize*10^9; - % % end - % % - % % if iThickness > 400 - % % fprintf('capping thickness to 400\n'); - % % iThickness = 400; - % % end - iThickness = 75; - - - - % % % [ exposureFilter ] = BH_exposureFilter( SIZE(1:2), TLT, METHOD, SAMPLING, 1 ); - - if strcmp(PRECISION,'double') - exposureFilter = double(exposureFilter); - end - for iSubTomo = 1:1+(nSubTomos-1)*(calcAllWeights) - - fprintf('%d/%d tomo %d/%d subtomo\n',iTomo,nTomos,iSubTomo,nSubTomos); - % xyzSubTomo = (positionList(iSubTomo,11:13)./1 + lowerLeftVol); - % prjVector = (xyzSubTomo- originPrj); - - if (calcAllWeights) - xyzSubTomo = positionList(iSubTomo,11:13)./SAMPLING; - prjVector = xyzSubTomo - originVol + reconShift; - else - ctfGroupSize = subTomoMeta.('ctfGroupSize').(tomoList{iTomo})(2); - - xyzSubTomo = [(ctfGroupSize .* iCtfGroup) - ctfGroupSize/2, 0,0]./SAMPLING; - prjVector = xyzSubTomo - originVol + reconShift; - prjVector(2:3) = 0; - end - - - if ( calcAllWeights ) - iGold = positionList(iSubTomo,7); - else - iGold = 1; - end - % % % for iPrj = 1:nPrjs - % % % - % % % - % % % iCs = TLT(iPrj,17); - % % % iWavelength = TLT(iPrj,18); - % % % iPhaseShift = TLT(iPrj,19); - % % % iDefocus = TLT(iPrj,15); - % % % iddF = TLT(iPrj,12); - % % % idPHI = TLT(iPrj,13); - % % % - % % % data = rWeight; - % % % - % % % % The central section is located at -1.*tiltAngle in the 3dFT since we - % % % % rotate the specimen back by this amount in reconstruction. - % % % rTilt = BH_defineMatrix([90,TLT(iPrj,4),-90],'Bah','invVector'); - if (calcAllWeights) - rSubTomo = reshape(positionList(iSubTomo,17:25),3,3); - else - rSubTomo = eye(3); - end - % % % % To calculate the defocus, we need to rotate to where the projection is - % % % % in real space TLT(iPrj,4) - % % % - % % % rProjection = BH_defineMatrix([90,TLT(iPrj,4),-90],'Bah','forwardVector'); - % % % prjCoords = rProjection*prjVector'; - % % % - % % % iDf = prjCoords(3).*pixelSize + iDefocus; - % % % defVect = [iDf - iddF, iDf + iddF, idPHI]; - % % % % Note the transpose (=inverse since the rotation matrix is orthogonal) is - % % % % taken because the stored matrix is for interpolation - % % % r = rSubTomo'*rTilt; - % % % - % % % % By default these are single PRECISION, but we truncate them anyhow, so - % % % % leave as single. - % % % [X,Y,Z,~,~,~] = BH_multi_gridCoordinates([SIZE(1:2),1],'Cartesian',METHOD,... - % % % {'single',r,[0,0,0]','invVector',1,1},0,1,0); - % % % - % % % - % % % % assuming ampContrast = 0.1, using -0.15 results in a weight with - % % % % (0.1^0.15)^2~ 0.5 at zero freqency. Allows some recovery of low freq without - % % % % creating too severe a blur - % % % [Hqz, ~] = BH_ctfCalc(radialCTFCalc,iCs,iWavelength,defVect,SIZE(1:2),iPhaseShift,-1.0); - % % % - % % % - % % % - % % % % %Default is double - % % % % if strcmpi(PRECISION,'single') - % % % % Hqz = single(abs(Hqz.*HqzUnMod)).^ctfScaleFactor; - % % % % elseif strcmpi(PRECISION, 'double') - % % % % Hqz = double(abs(Hqz.*HqzUnMod)).^ctfScaleFactor; - % % % % end - % % % - % % % %Default is double - % % % if strcmpi(PRECISION,'single') - % % % Hqz = single(abs(Hqz).^2); - % % % elseif strcmpi(PRECISION, 'double') - % % % Hqz = double(abs(Hqz).^2); - % % % end - % % % if SAMPLING > 1 - % % % Hqz = Hqz .^ (1*(SAMPLING - 1)^-3); - % % % end - - - % % % fractionOfDose = TLT(iPrj,14)/mean(TLT(:,14)); - % % % fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(iPrj,4))*400 )); - % - % fprintf('fractionOfDose %2.2f fractionOfElastic %2.2f at angle %2.2f\n',... - % fractionOfDose, fractionOfElastics,TLT(iPrj,4)); - - % TODO does this make sense to run as power of one or two? - exposureFilterPower = 1; - % % % data = data.*Hqz.*exposureFilter(:,:,TLT(iPrj,1)).^exposureFilterPower.*(fractionOfDose*fractionOfElastics); - - - % % % data = data(:); - % % % - % % % % Shift from image to array coordinates and set any out of bounds values to - % % % % the origin. - % % % originXYZ = ceil(((SIZE + 1)./2)); - % % % X = X(:)+originXYZ(1); - % % % Y = Y(:)+originXYZ(2); - % % % Z = Z(:)+originXYZ(3); - % % % - % % % - % % % outOfBounds = logical(( X < 1 | X > SIZE(1) ) + (Y < 1 | Y > SIZE(2)) + (Z < 1 | Z > SIZE(3))); - % % % X(outOfBounds) = 1;%originXYZ(1); - % % % Y(outOfBounds) = 1;%originXYZ(2); - % % % Z(outOfBounds) = 1;%originXYZ(3); - % % % - % % % - % % % - % % % l = sub2ind(SIZE,round(X),round(Y),round(Z)); - % % % clear X Y Z - % % % % much faster and effecitive for this simple sort. - % % % [B,I] = sort(l); - % % % idu = I(logical(B(1:end-1) - B(2:end))); - % % % - % % % tiltScale = 1;%- ( abs(sind(TLT(iPrj,4))).*0.2 ); - % % % rec{iGold}(l(idu)) = rec{iGold}(l(idu)) + tiltScale.*data(idu); - % % % clear l idu - % % % - % % % end % end loop over projections - [maxSize,maxCoord] = max(SIZE); - if (2.*maxSize > 512) - padScaling = maxSize ./ 512; - paddedSize = 512.*[1,1,1]; - else - paddedSize = ceil(2.*maxSize).*[1,1,1]; - padScaling = 1/2; - end - - [ SF3D ] = fourierCtfRecTex(paddedSize, [positionList(iSubTomo,1:10),... - xyzSubTomo.*SAMPLING,... - positionList(iSubTomo,14:16),... - rSubTomo(1:9),positionList(iSubTomo,26)],... - TLT, reconGeometry, originPrj, ... - SAMPLING); - - - - % % % - % % % % For now just recreate the Hermitian pair - % % % nX = paddedSize(1); - % % % nZ = paddedSize(3); - % % % oX = floor(nX/2)+1; - % % % isOdd = mod(nX,2); - [ rec{iGold} ] = BH_multi_makeHermitian(SF3D, paddedSize, padScaling); - - - - % % % try - % % % tmpArray = zeros(paddedSize,'single','gpuArray'); - % % % tmpArray(oX-1+isOdd:end,:,:) = SF3D; - % % % tmpArray(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); - % % % clear SF3D - % % % - % % % rec{iGold} = BH_reScale3d(tmpArray,'',padScaling,'GPU'); - % % % clear tmpArray - % % % catch - % % % fprintf('SIZE %d paddedSize %d scaleFactor %d\n',SIZE(1),paddedSize(1),padScale); - % % % error('I broke in generating the Hermitian mates!'); - % % % end - % % % % rec{iGold}(oX-1+isOdd:end,:,:) = SF3D; - % % % % rec{iGold}(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); - % % % % clear SF3D - - - - % % For now just recreate the Hermitian pair - % nX = size(rec{iGold} ,1); - % nZ = size(rec{iGold} ,3); - % oX = floor(nX/2)+1; - % isOdd = mod(nX,2); - - % rec{iGold}(oX-1+isOdd:end,:,:) = SF3D; - % rec{iGold}(1:oX-2+isOdd,:,:) = SF3D(oX-1+isOdd:-1:2,:,nZ:-1:1); - % clear SF3D - - - % % % for iGold = 1:1 + calcAllWeights - % % % % Zero out the large value from out of bounds conditions - % % % rec{iGold}(1) = 0; - % % % end - - end % end loop over subTomos - end %end loop over Tomos - - - - % % % g = BH_multi_gaussian3d(16.*[1,1,1],1.25); - % % % gf=fftshift(fftn(ifftshift(g))); - % % % g=real(fftshift(ifftn(ifftshift(gf.^3)))); clear gf - % % % g = g ./ sum(g(:)); - % % % - % % % if (useGPU) - % % % g = gpuArray(g); - % % % end - - - %SAVE_IMG(MRCImage(gather(rec{1})),'tmpR1.mrc'); - %SAVE_IMG(MRCImage(gather(rec{2})),'tmpR2.mrc'); - - for iGold = 1:1+calcAllWeights - - if (outputScaling) - ctfWeights{iGold,iCtfGroup} = gather(BH_reScale3d( rec{iGold}... - ,'', sprintf('%f',outputScaling), METHOD)); rec{iGold} = []; - else - ctfWeights{iGold,iCtfGroup} = gather(rec{iGold}); rec{iGold} = []; - end - - - - % % % % % ctfWeights{iGold,iCtfGroup} = convn(single(rec{iGold}),g,'same'); rec{iGold} = []; - - - - % % % % if (outputScaling ~= 1) - % % % % ctfWeights{iGold,iCtfGroup} = BH_reScale3d( ctfWeights{iGold,iCtfGroup}... - % % % % ,'', sprintf('%f',outputScaling), METHOD); - % % % % end - % % % % if (flgFirstPass) - % % % % % Only calc this once. - % % % % rad = fftshift(BH_bandpass3d(size(ctfWeights{iGold,iCtfGroup}),... - % % % % 0,0,0,'cpu','nyquist')); - % % % % flgFirstPass = 0; - % % % % end - % % % % - % % % % ctfWeights{iGold,iCtfGroup} = gather(ctfWeights{iGold,iCtfGroup}) .* rad; - % % % % - % % % % - % % % % ctfWeights{iGold,iCtfGroup} = (ctfWeights{iGold,iCtfGroup} - ... - % % % % min(ctfWeights{iGold,iCtfGroup}(rad > 0.0)))./((nPrjs)./2-0.5); - % % % % - % % % % % The mean value should probably be < 0.5 due to exposure filtering and - % % % % % inelastic losses. Figure out how to put this on an absolute scale. - % % % % %FIXME - % % % % - % % % % m = BH_movingAverage(ctfWeights{iGold,iCtfGroup},[7,7,7]); - % % % % - % % % % ctfWeights{iGold,iCtfGroup}(m < (mean(m(m>0)) + 0.5*std(m(m>0)))) = 0; - % % % % - % % % % meanPositiveValues = mean(mean(mean(ctfWeights{iGold,iCtfGroup}(ctfWeights{iGold,iCtfGroup} > 1e-2)))); - % % % % - % % % % ctfWeights{iGold,iCtfGroup} = gather(ctfWeights{iGold,iCtfGroup} .* (0.5/meanPositiveValues)); - - - end - - -end % end loop over ctfGroups -clear rad g rec Hqz HqzUnMod B I data outOfBounds -end - -function [ rWeight ] = calc_rWeight( SIZE, PRECISION, METHOD) - -rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); -if strcmp(METHOD,'GPU') - rWeight = gpuArray(rWeight); -end -rOrig = ceil((SIZE(1)+1)./2); -% % % % imod tilt zero freq = 0.2 * first non zero component -rWeight(rOrig ) = 0.2; -[rCut] = find(rWeight == floor(0.45*SIZE(1))); -pixelFallOff = rCut(1) ; -taperLow = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); - -pixelFallOff = SIZE(1)-rCut(2)+1 ; -taperTop = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); -rWeight(1:rCut(1)) = rWeight(1:rCut(1)).*flip(taperLow)'; -rWeight(rCut(2):end) = rWeight(rCut(2):end).*taperTop'; - - -%rWeight = rWeight + 1./rWeight.^2; -% resample2d only handles scaling right now, so pad to z=3 -rWeight = repmat((rWeight), 1, SIZE(2),1); -if strcmpi(PRECISION,'single') - rWeight = single(rWeight); -else - % This should be the default. - rWeight = double(rWeight); -end -end - diff --git a/metaData/BH_combineProjects.m b/metaData/BH_combineProjects.m index a5d63d75..dd380a92 100644 --- a/metaData/BH_combineProjects.m +++ b/metaData/BH_combineProjects.m @@ -86,8 +86,6 @@ masterTM.subTomoMeta.tiltGeometry.(tomoNames{iField}) = ... tmpTM.subTomoMeta.tiltGeometry.(tomoNames{iField}); - masterTM.subTomoMeta.ctfGroupSize.(tomoNames{iField}) = ... - tmpTM.subTomoMeta.ctfGroupSize.(tomoNames{iField}); end diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 46e6f01c..99337c5b 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -397,9 +397,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... if isfield(masterTM.tiltGeometry.(f{iOrig})) masterTM.tiltGeometry = rmfield(masterTM.tiltGeometry,f{iOrig}); end - if isfield(masterTM.ctfGroupSize.(f{iOrig})) - masterTM.ctfGroupSize = rmfield(masterTM.ctfGroupSize,f{iOrig}); - end + if isfield(masterTM.(cycleNumber).RawAlign.f{iOrig}) masterTM.(cycleNumber).RawAlign = rmfield(masterTM.(cycleNumber).RawAlign,(f{iOrig})); end diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index 8bd349f8..955a4a15 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -272,21 +272,12 @@ end - subTomoMeta.('ctfGroupSize').(fileInfo{iTomo,2}) = [nCTFgroups,0]; - iX = subTomoMeta.('mapBackGeometry').(fileInfo{iTomo,1}).('coords')(tomoNumber,1); - subTomoMeta.('ctfGroupSize').(fileInfo{iTomo,2})(2) = floor(iX./... - nCTFgroups); - % % % subTomoMeta.('mapExt').(fileInfo{iTomo,2}) = fileInfo{iTomo,3}; - % % % subTomoMeta.('mapPath').(fileInfo{iTomo,1}) = fileInfo{iTomo,2}; - end % For now, just assuming all of the maps are in the same place and have the same % suffix - generalize later. - - if nGPUs > nTomogramsTotal nGPUs = nTomogramsTotal end @@ -493,9 +484,6 @@ mapName = fileInfo{iTomo,2}; tmpGeom = parResults{iGPU}.(mapName); - tmpGeom(:,9:26:26*emc.nPeaks) = repmat(ceil(tmpGeom(:,11)./ ... - subTomoMeta.('ctfGroupSize').(mapName)(2)),1,emc.nPeaks); - % Sort so that CTFs can be left in main mem, and only pulled when needed and only % once per round of alignment. tmpGeom = sortrows(tmpGeom,9); @@ -506,17 +494,7 @@ nIDX = nIDX +1; end - - - %tmpGeom(:,9) = ceil(tmpGeom(:,11)./ ... - % subTomoMeta.('ctfGroupSize').(mapName)(2)); - - % Using my template matching, there should never be a tomo so close to - % the edge for this to be problem, but when working with coordinates - % from relion, I've noticed out of bounds conditions. Check explicitly - % here. - tmpGeom( tmpGeom(:,9)> nCTFgroups, 9 ) = nCTFgroups; - + subTomoMeta.('cycle000').('geometry').(mapName) = tmpGeom; end end diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 83bc0362..24b5412c 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -44,7 +44,9 @@ % Early development parameter, used to store more than one orientation during template matching % and use for further refinement. -if ~isfield(emc, 'nPeaks') +if isfield(emc, 'nPeaks') + EMC_assert_integer(emc.nPeaks, 1); +else emc.('nPeaks') = 1; end @@ -52,21 +54,68 @@ % This has not been well tested % When used in average3d, this value is stored in the subTomoMeta. -if ~isfield(emc, 'CUTPADDING') +if isfield(emc, 'CUTPADDING') + EMC_assert_integer(emc.CUTPADDING, 1); +else emc.('CUTPADDING') = 20; end if isfield(emc, 'whitenPS') - if (numel(emc.whitenPS) == 3) - emc.('wiener_constant') = emc.whitenPS(3); - else - error('whitenPS should be a 3 element vector'); - end + EMC_assert_numeric(emc.whitenPS, 3) + emc.('wiener_constant') = emc.whitenPS(3); else emc.('whitenPS') = [0.0,0.0,0.0]; emc.('wiener_constant') = 0.0; end +% Default bfactor applied to the re-weighting when generating the fully corrected volumes. +% positive corresponds to a sharpening, negative to a low-pass. +if isfield(emc, 'Fsc_bfactor') + EMC_assert_numeric(emc.Fsc_bfactor) +else + emc.('Fsc_bfactor') = 40.0; +end + +% Used to downweight higher frequencies based on relative CCC scores. +% Based on one of Niko's papers, but catching some edge cases for tomo. +% Overwritten if cycle == 0 as the scores from template matching do not work for this metric as they are SNR not CCC. +% TODO: get rid of the flg prefix +if isfield(emc, 'flgQualityWeight') + EMC_assert_numeric(emc.flgQualityWeight, 1) +else + emc.('flgQualityWeight') = 5.0; +end + +% Experimental downweighting of higher frequency info farther from focus. +% Could also consider filtering pre reconstruction +% Filtering by defocus using exp[-(%d*(argmax(def-1,0,5).*q)^%d)]\n',flgFilterDefocus); +if isfield(emc,'filterDefocus') + EMC_assert_numeric(emc.filterDefocus, 2) +else + emc.filterDefocus = [0.0, 0.0]; +end + +if isfield(emc,'flgCutOutVolumes') + EMC_assert_boolean(emc.flgCutOutVolumes) +else + emc.flgCutOutVolumes = false; +end + + +if isfield(emc,'track_stats') + EMC_assert_boolean(emc.track_stats) +else + emc.track_stats = false; +end + + +% Check and override the rotational convention to get helical averaging. +% Replaces the former hack of adding a fifth dummy value to the angular search +if isfield(emc,'doHelical') + EMC_assert_boolean(emc.doHelical) +else + emc.doHelical = false; +end end diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 4b1d9522..b411169a 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -264,13 +264,6 @@ geometry = subTomoMeta.(cycleNumber).(geom_name); - -try - flgCutOutVolumes = emc.('flgCutOutVolumes'); -catch - flgCutOutVolumes = 0; -end - try CUTPADDING = subTomoMeta.('CUTPADDING') catch @@ -719,7 +712,7 @@ TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); - if (flgCutOutVolumes) + if (emc.flgCutOutVolumes) volumeData = []; else [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... @@ -729,13 +722,7 @@ end - nCtfGroups = masterTM.('ctfGroupSize').(tomoList{iTomo})(1); iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; - wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); - % wgtName = sprintf('cache/%s_bin%d.wgt', tomoList{iTomo},... - - - tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); @@ -829,7 +816,7 @@ wedgeMask = wedgeMask .* wdgBP; % Find range to extract, and check for domain error. - if (flgCutOutVolumes) + if (emc.flgCutOutVolumes) [ indVAL, padVAL, shiftVAL ] = ... BH_isWindowValid(2*CUTPADDING+sizeWindow, ... sizeWindow, maskRadius, center); @@ -849,7 +836,7 @@ particleIDX = positionList(iSubTomo, 4); % Same for all peaks - if (flgCutOutVolumes) + if (emc.flgCutOutVolumes) particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak+1); iParticle = gpuArray(getVolume(MRCImage(particleOUT_name),... diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 001360a1..7357f765 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -32,39 +32,15 @@ reconScaling = 1; - -try - fscBfactor = emc.('Fsc_bfactor'); -catch - fscBfactor = 40; -end - mapBackIter = subTomoMeta.currentTomoCPR; -if (CYCLE) - try - flgQualityWeight = emc.('flgQualityWeight'); - catch - flgQualityWeight = 5; - end -else +if (CYCLE == 0) fprintf('No quality weighting in the initial cycle after template matching\n'); - flgQualityWeight = 0; -end -% Experimental downweighting of higher frequency info farther from focus. -% Could also consider filtering pre reconstruction -try - flgFilterDefocus = emc.('filterDefocus'); - fprintf('\nFiltering by defocus using exp[-(%d*(argmax(def-1,0,5).*q)^%d)]\n',flgFilterDefocus); -catch - flgFilterDefocus = 0; + emc.flgQualityWeight = 0; end -try - flgCutOutVolumes = emc.('flgCutOutVolumes'); -catch - flgCutOutVolumes = 0; -end + + try projectVolumes = emc.('flgProjectVolumes'); @@ -72,26 +48,21 @@ projectVolumes = false; end -if (projectVolumes && ~flgCutOutVolumes) - flgCutOutVolumes = true; +if (projectVolumes && ~emc.flgCutOutVolumes) + emc.flgCutOutVolumes = true; end -doCut = 0 -if (flgCutOutVolumes) +volumesNeedToBeExtracted = 0; +if (emc.flgCutOutVolumes) if isfield(subTomoMeta,'volumesAreCutOut') if ~(subTomoMeta.volumesAreCutOut) - doCut = 1 + volumesNeedToBeExtracted = 1; end else - doCut = 1 + volumesNeedToBeExtracted = 1; end end -try - track_stats = emc.('track_stats'); -catch - track_stats = false; -end % Note this will be set to false unless we are averging after an update try @@ -101,13 +72,7 @@ end rotConvention = 'Bah'; -% Check and override the rotational convention to get helical averaging. -% Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = emc.('doHelical'); -catch - doHelical = 0; -end + if ( doHelical ) rotConvention = 'Helical'; end @@ -117,16 +82,16 @@ % The weights are only re-estimated for an out of plane search. Until this % happens, they are not valid. -if (track_stats) +if (emc.track_stats) if isfield(subTomoMeta,'updatedWeights') - if subTomoMeta.updatedWeights == false - track_stats = false; + if ~(subTomoMeta.updatedWeights) + emc.track_stats = false; end else - track_stats = false; + emc.track_stats = false; end end -fprintf('track stats is %d\n',track_stats) +fprintf('track stats is %d\n',emc.track_stats) flgClassify= emc.('flgClassify'); %%% For general release, I've disabled class average alignment and @@ -489,13 +454,11 @@ % Get the number of tomograms to process. tomoList = fieldnames(geometry); nTomograms = length(tomoList); -ctfGroupList = masterTM.('ctfGroupSize'); - if (flgClassify) - [ maskType, maskSize, maskRadius, maskCenter ] = ... + [ ~, maskSize, maskRadius, maskCenter ] = ... BH_multi_maskCheck(emc, 'Ali', pixelSize); % These are used when 'Cluster' is called, to take the masking parameters % from focused PCA/Classification, to produce a montage with reduced @@ -504,11 +467,11 @@ % version or else graphical deletion of classes will fail. - [~, pcaMaskSize, pcaMaskRadius, pcaMaskCenter ] = ... + [~, ~, ~, pcaMaskCenter ] = ... BH_multi_maskCheck(emc, 'Cls', pixelSize); else - [ maskType, maskSize, maskRadius, maskCenter ] = ... + [ ~, maskSize, maskRadius, maskCenter ] = ... BH_multi_maskCheck(emc, 'Ali', pixelSize); end @@ -606,7 +569,7 @@ end spike_info = struct(); spike_info.('std_dev') = nan; -if (flgQualityWeight) +if (emc.flgQualityWeight) %get the average CCC for calculation of particle quality weighting. cccVect = []; @@ -720,7 +683,7 @@ addedWeight = 0; for iParProc = 1:nParProcesses for iTomo = iterList{iParProc} - if (track_stats) + if (emc.track_stats) geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks) = geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks)./geometry.(tomoList{iTomo})(:,2:26:26*emc.nPeaks); end @@ -806,10 +769,10 @@ % figure, hist((wgtVect./median(wgtVect)).^weightScale,29) % error('asdf') - if (track_stats) - fprintf('Avgerage score is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,flgQualityWeight); + if (emc.track_stats) + fprintf('Avgerage score is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,emc.flgQualityWeight); else - fprintf('Avgerage CCC is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,flgQualityWeight); + fprintf('Avgerage CCC is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,emc.flgQualityWeight); end @@ -854,7 +817,7 @@ - if (flgQualityWeight) + if (emc.flgQualityWeight) [cccWeight,~,~,~,~,~] = BH_multi_gridCoordinates(sizeCalc, ... 'Cartesian','GPU',... {'none'},1,0,1); @@ -934,7 +897,7 @@ fprintf('Loading tomo %d from tilt %s \n',tomoNumber,tiltName); reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - if (flgCutOutVolumes && ~doCut) + if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) volumeData = []; else @@ -956,7 +919,6 @@ iTiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); - nCtfGroups = ctfGroupList.(tomoList{1})(1); % Work on each class seperately pushing to main memory when finished. @@ -1025,7 +987,7 @@ emc.nPeaks, ... masterTM.(cycleNumber).('score_sigma') ,... iSubTomo, tomoList{iTomo},... - track_stats); + emc.track_stats); % Update any re-ordering or elimination positionList(iSubTomo,:) = sortedList; else @@ -1070,17 +1032,17 @@ - if (flgQualityWeight) + if (emc.flgQualityWeight) iCCC = positionList(iSubTomo,[1]+26*(iPeak-1)); - if (track_stats) + if (emc.track_stats) % Downweight higher frequency in all subTomos with iCCC below the mean - iBfactor = (flgQualityWeight.*(iCCC - maxCCC)./4) + iBfactor = (emc.flgQualityWeight.*(iCCC - maxCCC)./4) iCCCweight = exp(iBfactor.*cccWeight); else if iCCC < avgCCC % Downweight higher frequency in all subTomos with iCCC below the mean - iBfactor = -1.*(flgQualityWeight.*(acosd(iCCC) - acosd(avgCCC)))^2; + iBfactor = -1.*(emc.flgQualityWeight.*(acosd(iCCC) - acosd(avgCCC)))^2; iCCCweight = exp(iBfactor.*cccWeight); else @@ -1089,18 +1051,18 @@ end - if ( any(flgFilterDefocus)) + if ( any(emc.filterDefocus)) iDef = abs(mean(tiltGeometry(:,15))*10^6); - iDef = -1.*(flgFilterDefocus(1)*max(iDef-1,0.5))^flgFilterDefocus(2); + iDef = -1.*(emc.filterDefocus(1)*max(iDef-1,0.5))^emc.filterDefocus(2); fprintf('Using iDef %f\n',iDef); % Frequency is already squared so adjust to match iDef scale % factor. - iCCCweight = iCCCweight.*exp(iDef.*cccWeight.^(flgFilterDefocus(2)/2)); + iCCCweight = iCCCweight.*exp(iDef.*cccWeight.^(emc.filterDefocus(2)/2)); end end % Find range to extract, and check for domain error. - if (flgCutOutVolumes && ~doCut) + if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) [ indVAL, padVAL, shiftVAL ] = ... BH_isWindowValid(2*CUTPADDING+sizeWindow, ... sizeWindow, maskRadius, center); @@ -1114,7 +1076,7 @@ if ~ischar(indVAL) - if (flgCutOutVolumes && ~doCut) + if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) try particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); iParticle = gpuArray(getVolume(MRCImage(particleOUT_name),... @@ -1147,7 +1109,7 @@ - if (flgCutOutVolumes && doCut) + if (emc.flgCutOutVolumes && volumesNeedToBeExtracted) % Test with some generic padding , only to be used on bin 1 at % first!!! TODO add a flag to check this. @@ -1239,7 +1201,7 @@ iParticle = iParticle - mean(iParticle(interpMask_tmpBinary)); iParticle = iParticle ./ rms(iParticle(interpMask_tmpBinary)); iParticle = iParticle .* interpMask_tmp; - if (flgQualityWeight && numel(iCCCweight) > 1) + if (emc.flgQualityWeight && numel(iCCCweight) > 1) iParticle = real(ifftn(fftn(BH_padZeros3d(iParticle,... 'fwd',padCalc,'GPU','singleTaper')).*iCCCweight)); @@ -1865,9 +1827,9 @@ % Only send the lowest Bfactor if not flgFinalAvg if (flgFinalAvg) - bFactorSend = fscBfactor; + bFactorSend = emc.Fsc_bfactor; else - bFactorSend = fscBfactor(1); + bFactorSend = emc.Fsc_bfactor(1); end refTMP = gather(BH_multi_cRef_Vnorm(fscParams, aliParams, mskParams,... @@ -1903,9 +1865,9 @@ % Save the unweighted, weighted imgs, weightes, optionally filtered. if (flgFinalAvg) - for iBfactor = 1:length(fscBfactor) + for iBfactor = 1:length(emc.Fsc_bfactor) imout = sprintf('%s_class%d_%s_bFact-%d.mrc',outputPrefix, ... - className, 'final',fscBfactor(iBfactor)); + className, 'final',emc.Fsc_bfactor(iBfactor)); SAVE_IMG(refTMP{iBfactor}, imout, pixelSize); end @@ -1925,7 +1887,7 @@ subTomoMeta = masterTM; subTomoMeta.('CUTPADDING') = CUTPADDING; - if (flgCutOutVolumes && doCut) + if (emc.flgCutOutVolumes && volumesNeedToBeExtracted) subTomoMeta.('volumesAreCutOut') = 1; end save(emc.('subTomoMeta'), 'subTomoMeta'); From 82f5a13d94cc88cb8a5c283ad826da3b502f5cf8 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 11:38:58 -0500 Subject: [PATCH 012/151] Add parsing function to check and replace deprecated parameter names so old parameter files do not break. --- alignment/BH_alignRaw3d_v2.m | 46 ++--- alignment/BH_templateSearch3d_2.m | 13 +- ctf/BH_ctf_Correct3d.m | 7 - ctf/BH_ctf_Estimate.m | 46 +---- ctf/BH_ctf_Updatefft.m | 134 +++---------- logicals/EMC_assert_deprecated_substitution.m | 16 ++ metaData/BH_parseParameterFile.m | 35 ++++ metaData/BH_removeDuplicates.m | 4 +- metaData/BH_skipClassAlignment.m | 10 +- statistics/BH_clusterPub.m | 15 +- statistics/BH_fscGold_class.m | 17 +- statistics/BH_pcaPub.m | 23 +-- synthetic/BH_synthetic_mapBack.m | 3 - transformations/BH_average3d.m | 188 +++++------------- 14 files changed, 172 insertions(+), 385 deletions(-) create mode 100644 logicals/EMC_assert_deprecated_substitution.m diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index eb037ffd..43d1a04c 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -104,22 +104,14 @@ maxGoldStandard = subTomoMeta.('maxGoldStandard'); -nGPUs = emc.('nGPUs') - - -flgClassify= emc.('flgClassify'); -try - flgMultiRefAlignment=emc.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end +nGPUs = emc.('nGPUs'); try updateClassByBestReferenceScore = emc.('updateClassByBestReferenceScore'); catch updateClassByBestReferenceScore = false; end -if (~flgMultiRefAlignment) +if (~emc.multi_reference_alignment) updateClassByBestReferenceScore = false; end @@ -139,9 +131,7 @@ samplingRate = emc.('Ali_samplingRate'); pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; -if emc.('SuperResolution') - pixelSize = pixelSize * 2; -end + flgPrecision = 'single'; %emc.('flgPrecision'); angleSearch = emc.('Raw_angleSearch'); @@ -168,13 +158,9 @@ rotConvention = 'Bah'; % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = emc.('doHelical'); -catch - doHelical = 0; -end + if ( doHelical ) - rotConvention = 'Helical' + rotConvention = 'Helical'; end @@ -183,8 +169,8 @@ catch scaleCalcSize = 1.5; end -% % % % if (flgClassify || flgMultiRefAlignment) -if (flgClassify) +% % % % if (emc.classification || emc.multi_reference_alignment) +if (emc.classification) refName = emc.('Ref_className'); else refName = emc.('Raw_className'); @@ -200,12 +186,12 @@ classVector{2} = emc.('Raw_classes_eve')(1,:); -% % % % if (flgClassify || flgMultiRefAlignment) -if (flgClassify) +% % % % if (emc.classification || emc.multi_reference_alignment) +if (emc.classification) geometry = subTomoMeta.(cycleNumber).ClassAlignment; refVectorFull{1}= [emc.('Ref_references_odd');1] refVectorFull{2}= [emc.('Ref_references_eve');1] -elseif (flgMultiRefAlignment) +elseif (emc.multi_reference_alignment) geometry = subTomoMeta.(cycleNumber).ClusterRefGeom; refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ] refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ] @@ -489,9 +475,9 @@ for iWccc = 1:length(nReferences(1)); wCCC{iWccc} = 0; end -if (flgClassify || flgMultiRefAlignment) +if (emc.classification || emc.multi_reference_alignment) for iRef = 1:nReferences(1) - if (flgClassify) + if (emc.classification) fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); else fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Raw%d',iRef)); % % % % @@ -748,7 +734,7 @@ bandpassFilt_tmp = cell(nReferences(1),1); bandpassFiltREF_tmp = cell(nReferences(1),1); for iRef = 1:nReferences(1) - if flgMultiRefAlignment <= 2 + if emc.multi_reference_alignment <= 2 bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); else @@ -795,7 +781,7 @@ for iGold = 1:2 for iRef = 1:nReferences(iGold) - if flgMultiRefAlignment <= 2 + if emc.multi_reference_alignment <= 2 ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); @@ -1147,7 +1133,7 @@ end - switch flgMultiRefAlignment + switch emc.multi_reference_alignment case 0 refToAlign = 1; case 1 @@ -1155,7 +1141,7 @@ case 2 refToAlign = classIDX; otherwise - error('flgMultiRefAlignment is not 0,1,2') + error('emc.multi_reference_alignment is not 0,1,2') end for iRef = refToAlign diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index da037146..ac8deb60 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -138,9 +138,6 @@ end pixelSizeFULL = emc.('PIXEL_SIZE').*10^10; -if emc.('SuperResolution') - pixelSizeFULL = pixelSizeFULL * 2; -end pixelSize = pixelSizeFULL.*samplingRate; @@ -307,16 +304,12 @@ rotConvention = 'Bah'; % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search -try - doHelical = emc.('doHelical'); -catch - doHelical = 0; -end + if ( doHelical ) - rotConvention = 'Helical' + rotConvention = 'Helical'; end -rotConvention + if (use_new_grid_search) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 3d74cc25..af7d0d2d 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -311,13 +311,6 @@ pixelSize = emc.('PIXEL_SIZE').*10^10 .* samplingRate; -% if (recWithoutMat) -% reconstructionParameters(1) = ')(i) =(1) ./ pixelSize; -% end - -if emc.('SuperResolution') - pixelSize = pixelSize * 2; -end eraseRadius = ceil(1.5.*(emc.('beadDiameter')./emc.('PIXEL_SIZE').*0.5) / samplingRate); diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index c9a3aec0..e28b9a0c 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -96,33 +96,12 @@ Cs = emc.('Cs'); VOLTAGE = emc.('VOLTAGE'); AMPCONT = emc.('AMPCONT'); -SuperResolution = emc.('SuperResolution'); - -if (SuperResolution) - if SuperResolution == 1 - % Standard scenario crop to physical nyquist - scalePixelsBy = 2; - elseif SuperResolution > 10^10*PIXEL_SIZE - % Crop to the given pixels size - error('Scaling to arbitrary pixel size is not working\n'); - % Need to factor in the trunctation to integer pixel size. - % scalePixelsBy = SuperResolution/(10^10*PIXEL_SIZE); - else - error('SuperResolution must be 0 (off) 1 (crop to physical Nyquist) or a pixel Size larger than current\n'); - end - PIXEL_SIZE = scalePixelsBy.* PIXEL_SIZE; -else - scalePixelsBy = 1; -end -% if 10^10*PIXEL_SIZE < 1.2 -% fprintf('PixelSize is less than 1.2 Ang so we have to use the cpu\n'); -% useGPU = 0; -% METHOD = 'cpu'; -% else + +scalePixelsBy = 1; + useGPU = 1; METHOD = 'GPU'; -% end % Sanity check if (PIXEL_SIZE > 20e-10 || PIXEL_SIZE < 0) @@ -385,12 +364,7 @@ end end -if (SuperResolution) - % Forcing output to odd size. - sizeCropped = floor([d1,d2,d3]./2)-(1-mod(floor([d1,d2,d3]./2),2)); -else - sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); -end +sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); sizeCropped(3) = d3; STACK = zeros(sizeCropped,'single'); @@ -471,15 +445,9 @@ iProjection = iProjection - mean(iProjection(:)); - - if ( SuperResolution ) - iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... - padVal(1,:),padVal(2,:),shiftMETHOD,'singleTaper'); - else - iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - shiftMETHOD,'singleTaper'); - end - + + iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... + shiftMETHOD,'singleTaper'); if (i == 1 && bh_global_do_2d_fourier_interp) bhF = fourierTransformer(iProjection,'OddSizeOversampled'); diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 629a347e..5909dbb3 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -8,16 +8,7 @@ flgSkipUpdate = 0; % To avoid accidently masking any failures in subsequent update, clean out % all stacks and reconstructions from the local cache. -try - eucentric_minTilt = emc.('eucentric_minTilt'); -catch - eucentric_minTilt = 15; -end -try - flgShiftEucentric = emc.('eucentric_fit'); -catch - flgShiftEucentric = 0; -end + try % Should be negative, but to test. defShiftSign = emc.('testFlipSign'); @@ -56,7 +47,7 @@ end eucShiftsResults = 0; -if flgShiftEucentric +if emc.eucentric_fit eucShiftsResults = cell(size(ITER_LIST)); end % BH_geometryAnalysis(sprintf('%s',PARAMETER_FILE),sprintf('%d',subTomoMeta.currentCycle),'TiltAlignment','UpdateTilts',sprintf('[%d,0,0]',subTomoMeta.currentCycle),'STD') @@ -162,19 +153,7 @@ eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); % Converte bead diameter to pixels and add a little to be safe. PIXEL_SIZE = emc.('PIXEL_SIZE'); - SuperResolution = emc.('SuperResolution'); - % Don't apply any fourier cropping of super-res data if only updating, - % as it would already be done. - if strcmpi(applyFullorUpdate,'update') - SuperResolution = 0; - end - - if (SuperResolution) - % Transform the raw images at full sampling then crop the fft to physical - % nyquist - PIXEL_SIZE = 2.* PIXEL_SIZE; - end eraseSigma = 3;%emc.('beadSigma'); @@ -228,23 +207,19 @@ % created in IMod alignment. iHeader = getHeader(iMrcObj); - iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* (1+abs(SuperResolution)), ... - iHeader.cellDimensionY/iHeader.nY .* (1+abs(SuperResolution)), ... + iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... + iHeader.cellDimensionY/iHeader.nY, ... iHeader.cellDimensionZ/iHeader.nZ]; iOriginHeader= [iHeader.xOrigin , ... iHeader.yOrigin , ... - iHeader.zOrigin ] ./ (1+abs(SuperResolution)); + iHeader.zOrigin ]; d1 = iHeader.nX; d2 = iHeader.nY; d3 = size(INPUT_CELL{iStack,1},1);%iHeader.nZ; osX = 1-mod(d1,2); osY = 1-mod(d2,2); - - if (SuperResolution) - gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,-0.235,'GPU','nyquistHigh'); - else - gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); - end + + gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); TLT = INPUT_CELL{iStack,1}; pathName = INPUT_CELL{iStack,3} @@ -281,8 +256,8 @@ end - if ( flgShiftEucentric && mapBackIter ) - toFit = abs(mbTLT) > eucentric_minTilt; + if ( emc.eucentric_fit && mapBackIter ) + toFit = abs(mbTLT) > emc.eucentric_maxTilt; % For now take the mean, but it would probably be better to fit a line, % use the Y intercept, and use the deviation from 0 of the slope as a @@ -330,12 +305,7 @@ continue; end - if (SuperResolution) - % Forcing output to odd size. - sizeCropped = floor([d1,d2,d3]./2)-(1-mod(floor([d1,d2,d3]./2),2)); - else - sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); - end + sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); sizeCropped(3) = d3; STACK = zeros(sizeCropped,'single'); @@ -343,14 +313,7 @@ for i = 1:d3 - if (SuperResolution) - % The transform shifts need to be scaled by 2 since the stored values - % are relative to full sampling, while the tomoCPR are relative to - % physical pixel size. - updateScale = 2; - else - updateScale = 1; - end + updateScale = 1; if (mapBackIter) @@ -383,54 +346,22 @@ % Pad the projection prior to xforming in Fourier space. - if (SuperResolution) - - iProjection = single(getVolume(iMrcObj,[],[],tlt_tmp{i}(23),'keep')); - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); - - % Information beyond the physical nyquist should be removed to limit - % aliasing of noise prior tto interpolation. - iProjection = BH_padZeros3d(iProjection,[0,0],[0,0],'GPU','singleTaper',mean(iProjection(:))); - trimVal = BH_multi_padVal(1.*size(iProjection),sizeCropped(1:2)); - - largeOutliersMean= mean(iProjection(:)); - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single','gpuArray'); - - iProjection = real(ifftn(ifftshift(... - BH_padZeros3d(fftshift(... - fftn(iProjection)), ... - trimVal(1,:),trimVal(2,:),... - 'GPU','single')))); - - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),[0,0,0],[0,0],'Bah','GPU','forward',1/2,sizeCropped(1:2)); - sizeODD = size(iProjection)-[osX,osY]; - else - sizeODD = [d1,d2]-[osX,osY]; - - - - % If it is even sized, shift up one pixel so that the origin is in the middle - % of the odd output here we can just read it in this way, unlike super res. - - iProjection = ... - single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep')); - - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); - - largeOutliersMean= mean(iProjection(:)); - - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); - - - - end + sizeODD = [d1,d2]-[osX,osY]; + + % If it is even sized, shift up one pixel so that the origin is in the middle + % of the odd output here we can just read it in this way, unlike super res. + + iProjection = ... + single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep')); + + iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); + + largeOutliersMean= mean(iProjection(:)); + + largeOutliersSTD = std(iProjection(:)); + largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... + iProjection > largeOutliersMean + 6*largeOutliersSTD); + iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); % Because the rotation/scaling and translation are done separately, % we must use a square transform; otherwise, a rotation angle dependent @@ -445,13 +376,8 @@ iProjection = iProjection ./ std(iProjection(:)); - if ( SuperResolution ) - iProjection = BH_padZeros3d(iProjection(1+osX:end,1+osY:end), ... - padVal(1,:),padVal(2,:),'GPU','singleTaper'); - else - iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - 'GPU','singleTaper'); - end + iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... + 'GPU','singleTaper'); if (i == 1 && bh_global_do_2d_fourier_interp) bhF = fourierTransformer(iProjection,'OddSizeOversampled'); @@ -606,7 +532,7 @@ % -if (flgShiftEucentric && mapBackIter) +if (emc.eucentric_fit && mapBackIter) % Update the sub tomo z coords with an estimate of the shift cycle_to_update = subTomoMeta.('tomoCPR_run_in_cycle')(find(subTomoMeta.('tomoCPR_run_in_cycle')(:,1) == subTomoMeta.currentTomoCPR),2); for iGPU = 1:nGPUs diff --git a/logicals/EMC_assert_deprecated_substitution.m b/logicals/EMC_assert_deprecated_substitution.m new file mode 100644 index 00000000..3ffa635d --- /dev/null +++ b/logicals/EMC_assert_deprecated_substitution.m @@ -0,0 +1,16 @@ +function [parameter_struct] = EMC_assert_deprecated_substitution(parameter_struct, default_value, current_field, deprecated_field) + + % Copying this stuct around is probaby not the most efficient way to do this + % but accuracy is more important than speed here. + + % Handle type checks and default settings outside. + if isfield(parameter_struct, deprecated_field) + parameter_struct.(current_field) = parameter_struct.(deprecated_field); + parameter_struct = rmfield(parameter_struct, deprecated_field); + else + if ~isfield(parameter_struct, current_field) + parameter_struct.(current_field) = default_value; + end + end + +end \ No newline at end of file diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 24b5412c..e9dd4085 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -117,5 +117,40 @@ emc.doHelical = false; end +if isfield(emc,'eucentric_fit') + EMC_assert_boolean(emc.eucentric_fit) +else + emc.eucentric_fit = false; +end + +if isfield(emc,'eucentric_minTilt') + EMC_assert_numeric(emc.eucentric_maxTilt) +else + emc.eucentric_maxTilt = 50.0; +end + + +% TODO: these should maybe be two different orthogonal parameters + % if > 1 keep this many subtomos + % if < 1 keep this fraction +if isfield(emc,'flgCCCcutoff') + EMC_assert_numeric(emc.flgCCCcutoff,1) +else + emc.flgCCCcutoff = 0.0; +end + +% TOOD: DOC +emc = EMC_assert_deprecated_substitution(emc, false, 'projectVolumes', 'flgProjectVolumes'); +EMC_assert_boolean(emc.projectVolumes); + +% Whether the cycle is expected to be used for classification or alignment. +% Eventually, the distinction should not matter. +emc = EMC_assert_deprecated_substitution(emc, false, 'classification', 'flgClassify'); +EMC_assert_boolean(emc.classification); + + +emc = EMC_assert_deprecated_substitution(emc, false, 'multi_reference_alignment', 'flgMultiRefAlignment'); +EMC_assert_boolean(emc.multi_reference_alignment); + end diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index f93096d3..24014fe0 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -26,9 +26,7 @@ dupSampling = ceil(10e-10 / emc.('PIXEL_SIZE')); pixelSize = emc.('PIXEL_SIZE').*dupSampling.*10^10; -if emc.('SuperResolution') - pixelSize = pixelSize * 2; -end + latticeRadius = emc.('particleRadius'); dupRadius = max(1,floor(0.2*min(latticeRadius)/pixelSize)); diff --git a/metaData/BH_skipClassAlignment.m b/metaData/BH_skipClassAlignment.m index 00d5f6d6..21f71813 100755 --- a/metaData/BH_skipClassAlignment.m +++ b/metaData/BH_skipClassAlignment.m @@ -29,12 +29,6 @@ emc = BH_parseParameterFile(PARAMETER_FILE); -flgClassify = emc.('flgClassify'); -try - flgMultiRefAlignment = emc.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); @@ -42,11 +36,11 @@ if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - if (flgMultiRefAlignment && ~flgClassify) + if (emc.multi_reference_alignment && ~emc.classification) subTomoMeta.(cycleNumber).('RawAlign') = ... subTomoMeta.(cycleNumber).('Avg_geometry'); - elseif (flgMultiRefAlignment && flgClassify) + elseif (emc.multi_reference_alignment && emc.classification) subTomoMeta.(cycleNumber).('RawAlign') = ... subTomoMeta.(cycleNumber).('ClusterClsGeom'); else diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index f1dfa57e..d87ff8f1 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -51,12 +51,11 @@ test_multi_ref_diffmap=true; -flgClassify = emc.('flgClassify'); %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of %%% these features are re-introduced, this will need to be reverted. -if ( flgClassify ); flgClassify = -1 ; end -if flgClassify < 0 +if ( emc.classification ); emc.classification = -1 ; end +if emc.classification < 0 flgGold = 0; else flgGold = 1; @@ -132,14 +131,8 @@ masterTM = subTomoMeta; clear subTomoMeta -try - flgMultiRefAlignment = emc.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end - -geom_name='' -if (flgMultiRefAlignment ) +geom_name=''; +if (emc.multi_reference_alignment ) geom_name='ClusterClsGeom'; else geom_name='Avg_geometry'; diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index d4d57779..9aad7464 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -40,13 +40,7 @@ flgCones = emc.('flgCones'); -flgClassify= emc.('flgClassify') -try - flgMultiRefAlignment = emc.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end - +emc.classification = emc.('emc.classification'); try scaleCalcSize = emc.('scaleCalcSize'); catch @@ -94,11 +88,6 @@ samplingRate = emc.('Ali_samplingRate'); pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; -if emc.('SuperResolution') - pixelSize = pixelSize * 2; -end - - if ( flgCones ) coneInc = 30; @@ -173,7 +162,7 @@ switch STAGEofALIGNMENT case 'RawAlignment' savePrefix = 'Raw'; - if (flgClassify) + if (emc.classification) fieldPrefix = 'Raw'; className = 0; @@ -200,7 +189,7 @@ case 'NoAlignment' savePrefix = 'Raw'; - if (flgClassify) + if (emc.classification) fieldPrefix = 'NoA'; else diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index b411169a..2777e292 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -159,11 +159,6 @@ error('refSamplingRate ~= samplingRate') end -% FIXME: SuperResolution should be deprecated -if emc.('SuperResolution') - pixelSize = pixelSize * 2; - refPixelSize = refPixelSize * 2; -end nCores = BH_multi_parallelWorkers(emc.('nCpuCores')); pInfo = parcluster(); @@ -224,16 +219,15 @@ test_updated_bandpass = false; end -flgClassify = emc.('flgClassify'); % Removed flgGold everywhere else, but keep ability to classify full data set at % the end (after all alignment is finished.) %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of %%% these features are re-introduced, this will need to be reverted. -if ( flgClassify ); flgClassify = -1 ; end +if ( emc.classification ); emc.classification = -1 ; end -if flgClassify < 0 +if emc.classification < 0 flgGold = 0; else flgGold = 1; @@ -243,19 +237,14 @@ load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; -try - flgMultiRefAlignment = emc.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end -if (test_multi_ref_diffmap && ~flgMultiRefAlignment) +if (test_multi_ref_diffmap && ~emc.multi_reference_alignment) test_multi_ref_diffmap = false; - fprintf("WARNING: test_multi_ref_diffmap is incompatible with ~flgMultiRefAlignment, disabling\n"); + fprintf("WARNING: test_multi_ref_diffmap is incompatible with ~emc.multi_reference_alignment, disabling\n"); end geom_name='' -if (flgMultiRefAlignment ) +if (emc.multi_reference_alignment ) geom_name='ClusterClsGeom'; else @@ -310,7 +299,7 @@ refName = 0; end -% If flgClassify is negative combine the data for clustering, but don't set +% If emc.classification is negative combine the data for clustering, but don't set % any of the alignment changes to be persistant so that extracted class % averages are still independent half-sets. if (flgGold) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index c3d53aa9..ee889c7f 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -502,9 +502,6 @@ % The model is scaled to full sampling prior to passing to tiltalign, % make sure the header in the synthetic stack is set appropriately. fullPixelSize = emc.('PIXEL_SIZE').*10^10; - if emc.('SuperResolution') - fullPixelSize = fullPixelSize * 2; - end pixelSize = fullPixelSize.*samplingRate; try diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 7357f765..f68d9039 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -6,17 +6,14 @@ error('args = PARAMETER_FILE, CYCLE, STAGEofALIGNMENT') end - - % FIXME: hacking in a test test_fuzz=false; test_multi_ref_diffmap= true; -startTime = clock; +startTime = datetime("now"); CYCLE = EMC_str2double(CYCLE); - if strcmpi(STAGEofALIGNMENT, 'RawAlignment') % Ensure we don't have any duplicates: TODO: add an override flag % This modifies the RawAlign geometry, so should be cycle -1 @@ -25,11 +22,10 @@ end end -cycleNumber = sprintf('cycle%0.3u', CYCLE) +cycleNumber = sprintf('cycle%0.3u', CYCLE); emc = BH_parseParameterFile(PARAMETER_FILE); load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); -reconScaling = 1; mapBackIter = subTomoMeta.currentTomoCPR; @@ -39,16 +35,7 @@ emc.flgQualityWeight = 0; end - - - -try - projectVolumes = emc.('flgProjectVolumes'); -catch - projectVolumes = false; -end - -if (projectVolumes && ~emc.flgCutOutVolumes) +if (emc.projectVolumes && ~emc.flgCutOutVolumes) emc.flgCutOutVolumes = true; end @@ -64,21 +51,12 @@ end -% Note this will be set to false unless we are averging after an update -try - flgShiftEucentric = emc.('eucentric_fit'); -catch - flgShiftEucentric = 0; -end rotConvention = 'Bah'; - -if ( doHelical ) +if ( emc.doHelical ) rotConvention = 'Helical'; end -rotConvention - % The weights are only re-estimated for an out of plane search. Until this % happens, they are not valid. @@ -93,21 +71,16 @@ end fprintf('track stats is %d\n',emc.track_stats) -flgClassify= emc.('flgClassify'); %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of %%% these features are re-introduced, this will need to be reverted. -if ( flgClassify ); flgClassify = -1 ; end -try - flgMultiRefAlignment = emc.('flgMultiRefAlignment'); -catch - flgMultiRefAlignment = 0; -end +% FIXME: get rid of the -1 +if ( emc.classification ); emc.classification = -1 ; end + flgGold=1; pixelSize = emc.('PIXEL_SIZE').*10^10; -if emc.('SuperResolution') - pixelSize = pixelSize * 2; -end + + nGPUs = emc.('nGPUs'); % Optionally specify gpu idxs if numel(nGPUs) == 1 @@ -163,7 +136,7 @@ case 'RawAlignment' - if (flgClassify) + if (emc.classification) fieldPrefix = 'Raw' else @@ -177,10 +150,10 @@ className = emc.(sprintf('%s_className','Raw')); samplingRate = emc.('Ali_samplingRate'); - if (flgMultiRefAlignment && (test_multi_ref_diffmap || ~flgClassify)) + if (emc.multi_reference_alignment && (test_multi_ref_diffmap || ~emc.classification)) className = emc.(sprintf('Raw_className')) saveClassSum = emc.(sprintf('Raw_className')) - elseif (flgMultiRefAlignment && flgClassify) + elseif (emc.multi_reference_alignment && emc.classification) fprintf('\n\nMutliRef and Classify enabled.\n'); fprintf('Only creating the global class average for PCA\n\n.'); className = 0; @@ -202,7 +175,7 @@ className = emc.(sprintf('%s_className',fieldPrefix)); samplingRate = emc.('Ali_samplingRate'); - if (flgClassify) + if (emc.classification) %samplingRate = emc.('Pca_samplingRate'); else %samplingRate = emc.('Raw_samplingRate'); @@ -245,7 +218,7 @@ samplingRate = emc.(sprintf('Cls_samplingRate')); className = emc.(sprintf('%s_className',fieldPrefix)); - if flgClassify < 0 + if emc.classification < 0 flgGold = 0; end @@ -271,13 +244,7 @@ flgCones = emc.('flgCones'); -try - % if > 1 keep this many subtomos - % if < 1 keep this fraction - cccCutOff = emc.('flgCCCcutoff'); -catch - cccCutOff = 0.0; -end + cutPrecision = 'single'; %emc.('flgPrecision'); try interpOrder = emc.('interpOrder'); @@ -343,14 +310,14 @@ if strcmpi(STAGEofALIGNMENT, 'RawAlignment') if ( CYCLE ) - cycleRead = sprintf('cycle%0.3u', CYCLE - 1) + cycleRead = sprintf('cycle%0.3u', CYCLE - 1); else - flgShiftEucentric = false; % No possible updates on cycle 0 - cycleRead = sprintf('cycle%0.3u', CYCLE) + emc.eucentric_fit = false; % No possible updates on cycle 0 + cycleRead = sprintf('cycle%0.3u', CYCLE); end else - flgShiftEucentric = false; % No possible updates for other stages of alignments - cycleRead = sprintf('cycle%0.3u', CYCLE ) + emc.eucentric_fit = false; % No possible updates for other stages of alignments + cycleRead = sprintf('cycle%0.3u', CYCLE ); end % leave averages at size appropriate for interpolation when extracting to use @@ -366,13 +333,13 @@ geometry = subTomoMeta.(cycleRead).geometry; eachTomo = false;%true; end - if ~(flgClassify) + if ~(emc.classification) doNotTrim = true; end case 'FinalAlignment' geometry = subTomoMeta.(cycleRead).Avg_geometry; - if ~(flgClassify) + if ~(emc.classification) doNotTrim = true; end @@ -440,23 +407,23 @@ if isfield(masterTM,('tomoCPR_run_in_cycle')) - if (flgShiftEucentric && ~isfield(masterTM.(sprintf('%s',cycleRead)), 'eucentric_shifts')) + if (emc.eucentric_fit && ~isfield(masterTM.(sprintf('%s',cycleRead)), 'eucentric_shifts')) cycle_to_update = masterTM.('tomoCPR_run_in_cycle')(find(masterTM.('tomoCPR_run_in_cycle')(:,1) == masterTM.currentTomoCPR),2); if (cycle_to_update == cycleRead) error('You specified eucentric_fit=1, and you are averaging cycle %d and no shifts are found from cycle %d\n',cycleNumber,cycleRead); else - flgShiftEucentric = false; + emc.eucentric_fit = false; end end else - flgShiftEucentric = false; + emc.eucentric_fit = false; end % Get the number of tomograms to process. tomoList = fieldnames(geometry); nTomograms = length(tomoList); -if (flgClassify) +if (emc.classification) [ ~, maskSize, maskRadius, maskCenter ] = ... BH_multi_maskCheck(emc, 'Ali', pixelSize); @@ -736,16 +703,16 @@ end - if (cccCutOff > 1.0) + if (emc.flgCCCcutoff > 1.0) sorted_ccc = sort(cccVect); reqVol = int32(round(cccCutOff)) length(sorted_ccc) - reqVol - cccCutOff = sorted_ccc(length(sorted_ccc) - reqVol); + emc.flgCCCcutoff = sorted_ccc(length(sorted_ccc) - reqVol); fprintf('Removing all volumes with score < %2.2f to return the requested %d volumes\n\n',cccCutOff,reqVol); - elseif (cccCutOff > 0.0) + elseif (emc.flgCCCcutoff > 0.0) sorted_ccc = sort(cccVect); reqVol = cccCutoff; - cccCutOff = sorted_ccc(floor(length(cccVect).*(1 - reqVol))); + emc.flgCCCcutoff = sorted_ccc(floor(length(cccVect).*(1 - reqVol))); fprintf('Removing all volumes with score < %2.2f to return the requested percent %2.2f of possible volumes\n\n',cccCutOff,reqVol); end @@ -839,7 +806,7 @@ nTomos = 1; for iTomo = iterList{iParProc} - if (flgShiftEucentric) + if (emc.eucentric_fit) try geometry_tmp.(tomoList{iTomo})(:,13) = geometry_tmp.(tomoList{iTomo})(:,13) + ... masterTM.(sprintf('%s',cycleRead)).('eucentric_shifts').(tomoList{iTomo}) ; @@ -900,8 +867,10 @@ if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) volumeData = []; else - - [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... + + reconScaling = 1; + [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( ... + tomoList{iTomo}, ... reconCoords, mapBackIter, ... samplingRate,iGPUidx,reconScaling,loadTomo); @@ -1117,7 +1086,7 @@ particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); positionList(iSubTomo,[11:13]+26*(iPeak-1)) = shiftVAL+CUTPADDING+ceil((sizeWindow+1)./2); - if (projectVolumes) + if (emc.projectVolumes) SAVE_IMG(sum(iParticle,3),particleOUT_name,pixelSize); else particleOUT = BH_padZeros3d(gather(iParticle), CUTPADDING.*[1,1,1], ... @@ -1411,44 +1380,19 @@ % get the total class average by combining eve/odd classStorage = cell(maxClasses,2); -if (doNotTrim) && (flgClassify) +if (doNotTrim) && (emc.classification) filteredClass= cell(maxClasses,2); % low-pass to see class averages more clearly. % % [ bandpassFilt ] = BH_bandpass3d( sizeMask, 0.2, 300, 30, 'GPU',pixelSize); end - - -% % % if (doNotTrim) && (flgClassify) -% % % % reduce z dimension to area focused on in classification -% % % % taken from PCA mask values. -% % % zCenter = (sizeMask(3)+1)./2 + pcaMaskCenter(3); -% % % zLow = floor(zCenter - pcaMaskRadius(3)); -% % % zTop = zLow + 2.* pcaMaskRadius(3); -% % % if zLow < 1 -% % % fprintf('setting Z-low from %d to 1\n', zLow); -% % % zLow = 1; -% % % end -% % % if zTop > sizeMask(3) %%%size(filteredClass{iClassPos,iGold},3) -% % % fprintf('setting Z-top from %d to sizeFiltAvg,3\n', zTop); -% % % zTop = sizeMask(3); %%%size(filteredClass{iClassPos,iGold},3); -% % % end -% % % zLow = zLow; -% % % zTop = zTop ; % note not 2nd indx, just adding a shift -% % % sizeFilteredClass = [sizeMask(1),sizeMask(2),( zTop - zLow +1)] -% % % end - for iClass = 1:maxClasses classStorage{iClass,1} = zeros(sizeMask, 'single'); classStorage{iClass,2} = zeros(sizeMask, 'single'); - % % % if (doNotTrim) && (flgClassify) - % % % filteredClass{iClass,1} = zeros(sizeFilteredClass, 'single'); - % % % filteredClass{iClass,2} = zeros(sizeFilteredClass, 'single'); - % % % end end if (eachTomo) - [o,c] = system('mkdir -p initialTomoAvgs'); + system('mkdir -p initialTomoAvgs'); % sizeWeight mask is sizeMask or 128^3 whichever is larger % bandpassFiltTomo = BH_bandpass3d( sizeCalc, lpTomo(1), lpTomo(2), lpTomo(3), 'GPU',pixelSize); @@ -1478,13 +1422,9 @@ for iClassPos = 1:maxClasses - - - if (doNotTrim) && (flgClassify) + if (doNotTrim) && (emc.classification) % % % % % % % m = BH_mask3d('sphere',sizeMask,floor(sizeMask./2-6),pcaMaskCenter); [ m ] = EMC_maskShape('sphere', sizeMask,floor(sizeMask./2-6), 'gpu', {'shift', pcaMaskCenter}); - - else m = 1; end @@ -1495,47 +1435,22 @@ avgVolume{iClassPos,iGold} = avgVolume{iClassPos,iGold} ./ ... sum(nExtracted(iClassPos,iGold)); - - end fprintf('flgGold = %d\n',flgGold); for iGold = 1:2-flgFinalAvg - - % % % if( flgGold ) if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; - end - % % % else - % % % halfSet = 'STD'; - % % % end - + end classStorage{iClassPos,iGold} = gather(avgVolume{iClassPos,iGold} ); - - - if isnan( mean(classStorage{iClassPos,iGold}(:)) ) + if ~isfinite( mean(classStorage{iClassPos,iGold}(:)) ) clear classAVG fprintf('zeroing out classavg because of NaN values detected.\n') else - % % % if (doNotTrim) && (flgClassify) - % % % % lowpass according to Kms bandpass - % % % bandpassFilt = BH_bandpass3d( sizeMask, 0.1,300,30, 'GPU',pixelSize); - % % % - % % % tmpFilt = BH_bandLimitCenterNormalize( ... - % % % classStorage{iClassPos,iGold}.* ... - % % % m, bandpassFilt, (m>0.95), ... - % % % [0,0,0;0,0,0],'single'); - % % % tmpFilt = m.*real(ifftn(tmpFilt)); - % % % tmpFilt = tmpFilt(:,:,zLow:zTop ); - % % % tmpFilt = tmpFilt - mean(tmpFilt(:)); - % % % tmpFilt = tmpFilt ./ rms(tmpFilt(:)); - % % % - % % % filteredClass{iClassPos,iGold} = gather(tmpFilt); - % % % end % Normalize the regular averages classStorage{iClassPos,iGold} = classStorage{iClassPos,iGold} - ... @@ -1552,21 +1467,16 @@ end end - - - end end - - % Using the filtered class average, calc real space CCC to reorder the even % class to [most likely] match the corresponding odd class. classListOut = 0; % % % if (flgGold) && strcmpi(STAGEofALIGNMENT, 'Cluster') -if strcmpi(STAGEofALIGNMENT, 'Cluster') && (flgClassify ~= -1) +if strcmpi(STAGEofALIGNMENT, 'Cluster') && (emc.classification ~= -1) % % % [classListOut, geometry] = reorder_classes(filteredClass(:,1),filteredClass(:,2),maxClasses, geometry); % % % filteredClass(:,2) = filteredClass(classListOut(:,2), 2); @@ -1590,8 +1500,8 @@ % multi-reference alignment. if strcmpi(STAGEofALIGNMENT, 'Cluster') masterTM.(cycleNumber).(ClusterGeomNAME) = geometry; -elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') && flgMultiRefAlignment - if (flgClassify) +elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') && emc.multi_reference_alignment + if (emc.classification) masterTM.(cycleNumber).('ClusterClsGeom') = geometry; else masterTM.(cycleNumber).('ClusterRefGeom') = geometry; @@ -1644,7 +1554,7 @@ end SAVE_IMG(montOUT, imout); %%%%%%%% - % % % if (doNotTrim) && (flgClassify) + % % % if (doNotTrim) && (emc.classification) % % % [montOUT, ~] = BH_montage4d(filteredClass(:,iGold), ''); % % % imout = sprintf('%s_filtered%d_%s_%s.mrc',outputPrefix, ... % % % className, fieldPrefix, halfSet); @@ -1761,7 +1671,7 @@ % This is slow ass when using cones and class averages and wouldn't be too % hard to put into parallel. Do that once the next manuscript is finished. - if (~flgMultiRefAlignment && ~flgClassify ) + if (~emc.multi_reference_alignment && ~emc.classification ) nClassesReWgt = 1; else nClassesReWgt = maxClasses; @@ -1783,7 +1693,7 @@ iRefPrev = iRef; end - if (flgGold) || (flgClassify < 0) + if (flgGold) || (emc.classification < 0) flgCombine = 0; flgRefCutOff = 1; else @@ -1839,7 +1749,7 @@ if ~(flgFinalAvg) refIMG{1}{iOdd} = refTMP{1,1}; - if (flgGold) || (flgClassify < 0) + if (flgGold) || (emc.classification < 0) refIMG{2}{iEve} = refTMP{1,2}; end clear refTMP @@ -1849,7 +1759,7 @@ for iGold = 1:2-flgFinalAvg - if( flgGold ) || (flgClassify < 0) + if( flgGold ) || (emc.classification < 0) if iGold == 1 halfSet = 'ODD'; else From 76c219148d446c2ae2541989dc53d0deda51120a Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 12:20:24 -0500 Subject: [PATCH 013/151] Added range checks for asserts and change microscope parameters. --- alignment/BH_alignRaw3d_v2.m | 23 ++++---- alignment/BH_runAutoAlign.m | 23 ++++---- alignment/BH_templateSearch3d_2.m | 4 +- ctf/BH_ctf_Correct.m | 13 ++--- ctf/BH_ctf_Correct3d.m | 51 ++++++++--------- ctf/BH_ctf_Estimate.m | 93 ++++++++++++------------------- ctf/BH_ctf_Refine2.m | 27 ++------- ctf/BH_ctf_Updatefft.m | 13 ++--- logicals/EMC_assert_integer.m | 27 +++++++-- logicals/EMC_assert_numeric.m | 27 +++++++-- metaData/BH_parseParameterFile.m | 45 +++++++++++++++ metaData/BH_removeDuplicates.m | 4 +- statistics/BH_fscGold_class.m | 2 +- statistics/BH_pcaPub.m | 4 +- synthetic/BH_synthetic_mapBack.m | 4 +- synthetic/BH_to_cisTEM_mapBack.m | 13 ++--- transformations/BH_average3d.m | 78 +++++++++++--------------- 17 files changed, 233 insertions(+), 218 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 43d1a04c..ffde1350 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -130,12 +130,12 @@ flgRaw_shapeMask = 0;%= emc.('experimentalOpts')(3) samplingRate = emc.('Ali_samplingRate'); -pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; +emc.pixel_size_angstroms = emc.pixel_size_angstroms.*samplingRate; flgPrecision = 'single'; %emc.('flgPrecision'); angleSearch = emc.('Raw_angleSearch'); -peakSearch = (emc.('particleRadius')./pixelSize); +peakSearch = (emc.('particleRadius')./emc.pixel_size_angstroms); peakCOM = [1,1,1].*3; className = emc.('Raw_className'); @@ -146,7 +146,7 @@ end try eraseMaskType = emc.('Peak_mType'); - eraseMaskRadius = emc.('Peak_mRadius')./pixelSize; + eraseMaskRadius = emc.('Peak_mRadius')./emc.pixel_size_angstroms; fprintf('Further restricting peak search to radius %f %f %f\n',... eraseMaskRadius); eraseMask = 1; @@ -253,7 +253,7 @@ [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', pixelSize) + BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) @@ -383,11 +383,8 @@ if ~isempty(refTMP{iP}) tIMG{n} = refTMP{iP}; refTMP{iP} = []; if (flgCenterRefCOM) - % Not sure if this is always the best approach, but it may be - % useful in some cases. - % % % % % % % [~,iCOM] = BH_mask3d(gpuArray(tIMG{n}).*comMask,pixelSize,'','',1); - [~, ~, ~,iCOM] = EMC_maskReference(gpuArray(tIMG{n}).*comMask, pixelSize, {'fsc',true; 'com', true}); + [~, ~, ~,iCOM] = EMC_maskReference(gpuArray(tIMG{n}).*comMask, emc.pixel_size_angstroms, {'fsc',true; 'com', true}); fprintf('centering ref %d on COM %3.3f %3.3f %3.3f \n',n,iCOM); tIMG{n} = BH_resample3d(tIMG{n},[0,0,0],gather(iCOM), ... @@ -423,7 +420,7 @@ end -[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, pixelSize, maxGoldStandard ); +[ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, emc.pixel_size_angstroms, maxGoldStandard ); @@ -459,7 +456,7 @@ if ( flgRaw_shapeMask ) [ volMask ] = gather(sqrt(volMask .* ... - EMC_maskReference(refIMG{1}{iRef}+refIMG{2}{iRef}, pixelSize, {'fsc', true}))); + EMC_maskReference(refIMG{1}{iRef}+refIMG{2}{iRef}, emc.pixel_size_angstroms, {'fsc', true}))); else % % % % % % % [ volMask ] = gather(BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter)); @@ -485,7 +482,7 @@ [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); + radialGrid = single(radialGrid./emc.pixel_size_angstroms); % returns a cpu array if (flgWeightCCC) [ bandpassFilt{iRef}, ~,wCCC] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1, 1); @@ -505,7 +502,7 @@ fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... 'GPU', {'none'}, 1, 0, 1 ); - radialGrid = single(radialGrid./pixelSize); + radialGrid = single(radialGrid./emc.pixel_size_angstroms); % returns a cpu array if (flgWeightCCC) [ bandpassFilt{iRef},~,wCCC{iRef} ] = BH_multi_cRef( fscINFO, radialGrid, emc.Fsc_bfactor(1), 1, 1 ); @@ -1634,7 +1631,7 @@ end % Print out in Angstrom - printShifts = printShifts .* pixelSize; + printShifts = printShifts .* emc.pixel_size_angstroms; deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index af7fbfb1..f47bba91 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -10,7 +10,6 @@ skip_tilts = EMC_str2double(varargin{1}); end -pixelSize = emc.('PIXEL_SIZE').*10^10; imgRotation = EMC_str2double(imgRotation); try @@ -72,9 +71,9 @@ % FIXME this should probably be specified in Ang try - FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(emc.('autoAli_max_shift_in_angstroms')./pixelSize); + FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(emc.('autoAli_max_shift_in_angstroms')./emc.pixel_size_angstroms); catch - FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(40 ./ pixelSize); + FIRST_ITER_SHIFT_LIMIT_PIXELS = ceil(40 ./ emc.pixel_size_angstroms); end try @@ -143,11 +142,11 @@ cd('../'); -binHigh=ceil(MIN_SAMPLING_RATE ./ pixelSize); +binHigh=ceil(MIN_SAMPLING_RATE ./ emc.pixel_size_angstroms); if MAX_SAMPLING_RATE > 4 binLow = MAX_SAMPLING_RATE; else - binLow = ceil(MAX_SAMPLING_RATE ./ pixelSize); + binLow = ceil(MAX_SAMPLING_RATE ./ emc.pixel_size_angstroms); end binInc = -1*ceil((binHigh- binLow)./3); @@ -222,10 +221,10 @@ fprintf('Preprocessing tilt-series\n'); -%gradientAliasFilter = BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',pixelSize); +%gradientAliasFilter = BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',emc.pixel_size_angstroms); gradientAliasFilter = {BH_bandpass3d(1.*[nX,nY,1],0,0,0,'GPU','nyquistHigh'),... - BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',pixelSize)}; -if pixelSize < 2 + BH_bandpass3d([nX,nY,1],1e-6,LOW_RES_CUTOFF,RESOLUTION_CUTOFF,'GPU',emc.pixel_size_angstroms)}; +if emc.pixel_size_angstroms < 2 medianFilter = 5; else medianFilter = 3; @@ -240,7 +239,7 @@ inputStack(:,:,iPrj) = gather(tmpPrj); end -SAVE_IMG(inputStack,fixedName,pixelSize); +SAVE_IMG(inputStack,fixedName,emc.pixel_size_angstroms); fprintf('finished preprocessing tilt-series\n'); clear tmpPrj inputStack @@ -260,7 +259,7 @@ system(sprintf('%s %s %f %f %d %d %d %d %d %d %s %d %d %d %f %f %f %d %d %d > ./emC_autoAliLog_%s.txt',... runPath, ... baseName, ... - pixelSize, ... + emc.pixel_size_angstroms, ... imgRotation, ... binHigh, ... binLow, ... @@ -310,7 +309,7 @@ % Stopping for now at a bin5, this should be dynamic along with a handful % of other options. min_sampling_rate = 5; - [ to_few_beads ] = BH_refine_on_beads(baseName,nX,nY,3000,pixelSize,1.05.*100, min_sampling_rate); + [ to_few_beads ] = BH_refine_on_beads(baseName,nX,nY,3000,emc.pixel_size_angstroms,1.05.*100, min_sampling_rate); if (to_few_beads) fprintf('\nWARNING: to few beads found. Using iterative patch tracking results\n'); @@ -336,7 +335,7 @@ cd fixedStacks system(sprintf('%s %s %d %d %d %d', findBeadsPath, baseName,... nX,nY,3000,... - ceil(1.05*100/pixelSize))); + ceil(1.05*100/emc.pixel_size_angstroms))); cd .. end diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index ac8deb60..61619ee4 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -137,9 +137,9 @@ peakThreshold = over_ride; end -pixelSizeFULL = emc.('PIXEL_SIZE').*10^10; +pixelSizeFULL = emc.pixel_size_angstroms; -pixelSize = pixelSizeFULL.*samplingRate; +pixelSize = emc.pixel_size_angstroms .* samplingRate; % For testing print_warning=false; diff --git a/ctf/BH_ctf_Correct.m b/ctf/BH_ctf_Correct.m index 291d3dba..bb23f598 100755 --- a/ctf/BH_ctf_Correct.m +++ b/ctf/BH_ctf_Correct.m @@ -45,7 +45,6 @@ end end -pixelSize = emc.('PIXEL_SIZE'); !mkdir -p ctfStacks try @@ -111,7 +110,7 @@ initImg = randn(fastFTSize,'single','gpuArray'); f = FFT(initImg); - ctf = CTF(fastFTSize,pixelSize*10^10,'GPU'); + ctf = CTF(fastFTSize,emc.pixel_size_angstroms,'GPU'); maxZ = 500; maxEval = cosd(TLT(iPrj,4)).*(d1/2) + maxZ./2*abs(sind(TLT(iPrj,4))); oX = ceil((d1+1)./2); @@ -121,7 +120,7 @@ STRIPWIDTH = 512; STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); % take at least 1200 Ang & include the taper if equal to STRIPWIDTH - tileSize = floor(max(600./pixelSize, STRIPWIDTH + 28)); + tileSize = floor(max(600./emc.pixel_size_si, STRIPWIDTH + 28)); tileSize = tileSize + mod(tileSize,2); %fprintf('stripwidth tilesize %d %d\n',STRIPWIDTH,tileSize); incLow = ceil(tileSize./2); @@ -151,16 +150,14 @@ % The eval mask condition can be replaced once the per tomo condition % is trusted. if any(ismember(i:endIDX,iEvalMask)) - - - DF = D0 +(i + stripDefocusOffset - oX)*pixelSize*-1.*tand(TLT(iPrj,4)); - + + DF = D0 + ( i + stripDefocusOffset - oX)*emc.pixel_size_si*-1.*tand(TLT(iPrj,4) ); if ~( isempty(DF) ) iDefocus = [DF - ddF, DF + ddF, dPhi]; - if pixelSize < 2.0e-10 + if emc.pixel_size_si < 2.0e-10 % use double precision - this is not enabled, but needs to be - % requires changes to radial grid as well. ctf.new_img(iDefocus,CS,WL,AMPCONT,-1,-1); diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index af7d0d2d..b407e52e 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -309,10 +309,10 @@ nGPUs = length(gpuList); end -pixelSize = emc.('PIXEL_SIZE').*10^10 .* samplingRate; +emc.pixel_size_angstroms = emc.pixel_size_angstroms .* samplingRate; -eraseRadius = ceil(1.5.*(emc.('beadDiameter')./emc.('PIXEL_SIZE').*0.5) / samplingRate); +eraseRadius = ceil(1.5.*(emc.('beadDiameter')./emc.pixel_size_si.*0.5) / samplingRate); nTomosPerTilt = 0; recGeom = 0; @@ -549,13 +549,13 @@ tomoNumber = 1; else [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... + iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... 0,1); end else [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... + iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... 0,1); end @@ -588,7 +588,7 @@ % For each tomo create a list of slices that are to be reconstructed % for every section section. - [ sectionList ] = calcTomoSections(iCoords, tomoNumber,pixelSize, ... + [ sectionList ] = calcTomoSections(iCoords, tomoNumber,emc.pixel_size_angstroms, ... nSections,tiltList{iTilt}, ctf3dDepth); @@ -599,7 +599,7 @@ else [ avgZ, maxZ, tomoNumber, surfaceFit ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, pixelSize, ... + iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... sectionList,0); @@ -672,8 +672,8 @@ [ correctedStack ] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,pixelSize,maskedStack,... - maxZ*10/pixelSize,flgDampenAliasedFrequencies,... + avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... + maxZ*10/emc.pixel_size_angstroms,flgDampenAliasedFrequencies,... preCombDefocus,samplingRate,... applyExposureFilter,surfaceFit,... useSurfaceFit,invertDose,... @@ -691,7 +691,7 @@ outputStack = sprintf('%s/%s_ali%d_%d.fixed', ... tmpCache,tiltList{iTilt},mapBackIter+1,iSection) - SAVE_IMG(correctedStack,outputStack,pixelSize); + SAVE_IMG(correctedStack,outputStack,emc.pixel_size_angstroms); correctedStack = []; % Loop over tomos reconstructing section and appending a file to @@ -926,7 +926,7 @@ end -function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, pixelSize,... +function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, emc.pixel_size_angstroms,... nSections,tiltName, ctf3Depth) nTomos = length(tomoNumber); @@ -938,7 +938,7 @@ % With rounding this could end up a bit short except the top and bottom are both % half a section larger than minimally needed. -nSec = floor(ctf3Depth*10^10/pixelSize) ; +nSec = floor(ctf3Depth*10^10/emc.pixel_size_angstroms) ; nSec = nSec + ~mod(nSec,2); halfSec = (nSec-1)/2; @@ -1031,7 +1031,7 @@ function [correctedStack] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,pixelSize,maskedStack,... + avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... maxZ,flgDampenAliasedFrequencies,... preCombDefocus,samplingRate,... applyExposureFilter,surfaceFit,... @@ -1057,9 +1057,6 @@ [d1,d2,nPrjs] = size(maskedStack); - - -PIXEL_SIZE = pixelSize*10^-10; % This is just going to be written out to disk so keep in main memory. correctedStack = zeros(d1,d2,nPrjs,'single'); @@ -1107,12 +1104,12 @@ 'Cylindrical','GPU', ... {'none'},1,0,0); end -radialGrid = {radialGrid./PIXEL_SIZE,0,phi}; +radialGrid = {radialGrid./emc.pixel_size_si,0,phi}; phi = []; -fprintf('%f %f\n',filterProjectionsForTomoCPRBackground,pixelSize); +fprintf('%f %f\n',filterProjectionsForTomoCPRBackground,emc.pixel_size_angstroms); if (filterProjectionsForTomoCPRBackground ~= 0) - bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',pixelSize); + bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',emc.pixel_size_angstroms); else bpFilter = 1; end @@ -1135,10 +1132,10 @@ - STRIPWIDTH = min(floor((0.5*ctf3dDepth/PIXEL_SIZE)/abs(tand(TLT(iPrj,4)))),512); + STRIPWIDTH = min(floor((0.5*ctf3dDepth/emc.pixel_size_si)/abs(tand(TLT(iPrj,4)))),512); STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); % take at least 1200 Ang & include the taper if equal to STRIPWIDTH - tileSize = floor(max(600./pixelSize, STRIPWIDTH + 28)); + tileSize = floor(max(600./emc.pixel_size_angstroms, STRIPWIDTH + 28)); tileSize = tileSize + mod(tileSize,2); %fprintf('stripwidth tilesize %d %d\n',STRIPWIDTH,tileSize); incLow = ceil(tileSize./2); @@ -1188,7 +1185,7 @@ % Transform the specimen plane tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ +oX); tY = round(rA(2).*rX + rA(5).*rY + rA(8).*rZ +oY); - tZ = PIXEL_SIZE.*(rA(3).*rX + rA(6).*rY + rA(9).*rZ) + full_defocusOffset; + tZ = emc.pixel_size_si.*(rA(3).*rX + rA(6).*rY + rA(9).*rZ) + full_defocusOffset; % Some edge pixels can be out of bounds depending on the orientation of % the plan fit. Setting to zero will will ignore them (assuming defocus @@ -1225,7 +1222,7 @@ modHqz = []; else - if PIXEL_SIZE < 2.0e-10 + if emc.pixel_size_si < 2.0e-10 % use double precision - this is not enabled, but needs to be - % requires changes to radial grid as well. Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,-1); @@ -1265,7 +1262,7 @@ samplingMask(samplingMask == 0) = 1; if (flgWhitenPS(1)) - correctedStack(:,:,TLT(iPrj,1)) =gather(BH_whitenNoiseSpectrum(correctedPrj./samplingMask,'',pixelSize,1)); + correctedStack(:,:,TLT(iPrj,1)) =gather(BH_whitenNoiseSpectrum(correctedPrj./samplingMask,'',emc.pixel_size_angstroms,1)); else correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj./samplingMask); @@ -1279,7 +1276,7 @@ function [avgZ, maxZ, tomoNumber,surfaceFit] = calcAvgZ(masterTM,iCoords, ... tiltName,tomoList,... - nTomos, pixelSize,... + nTomos, emc.pixel_size_angstroms,... samplingRate,cycleNumber,... sectionList,calcMaxZ) @@ -1311,7 +1308,7 @@ maxZ = maxZ + (samplingRate*2); -maxZ = maxZ.*pixelSize./10; +maxZ = maxZ.*emc.pixel_size_angstroms./10; fprintf('combining thickness and shift on tilt %s, found a maxZ %3.3f nm\n',tiltName,maxZ); if (calcMaxZ) @@ -1374,7 +1371,7 @@ zList = zList - tomoOrigin(3) + micOrigin(3); totalZ = totalZ + sum(zList); fprintf('%s tomo has %d subTomos with mean Z %3.3f nm\n', ... - iTomoName, length(zList), mean(zList)*pixelSize./10); + iTomoName, length(zList), mean(zList)*emc.pixel_size_angstroms./10); nSubTomos = nSubTomos + length(zList); for iSection = 1:nSections @@ -1404,7 +1401,7 @@ end % loop over tomos -avgZ = totalZ/nSubTomos*pixelSize/10*10^-9; +avgZ = totalZ/nSubTomos*emc.pixel_size_angstroms/10*10^-9; % sf(x,y) = p00 + p10*x + p01*y; % surfaceFit = fit([xFull, yFull],zFull,'poly11'); diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index e28b9a0c..5a36b95e 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -91,34 +91,13 @@ end - -PIXEL_SIZE = emc.('PIXEL_SIZE'); -Cs = emc.('Cs'); -VOLTAGE = emc.('VOLTAGE'); -AMPCONT = emc.('AMPCONT'); - - scalePixelsBy = 1; useGPU = 1; METHOD = 'GPU'; -% Sanity check -if (PIXEL_SIZE > 20e-10 || PIXEL_SIZE < 0) - error('pixel size should be [0,20e-10]'); -elseif (Cs > 10e-3 || Cs < 0) - fprintf('\nWARNING Cs should be[10e-3,0]\n'); -elseif(VOLTAGE > 1000e3 || VOLTAGE < 20e3) - error ('VOLTAGE should be [20e3,1000e3]'); -elseif (AMPCONT < 0.025 || AMPCONT > 0.25) - fprintf('\nWARNING: AMPCONT probably should be [0.025,0.25]\n'); -end -WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; - -if Cs == 0 - Cs = 1e-6; -end +WAVELENGTH = 10^-12*1226.39/sqrt(emc.VOLTAGE + 0.97845*10^-6*emc.VOLTAGE^2) ; CUM_e_DOSE = emc.('CUM_e_DOSE'); % test astigmatism vals @@ -136,14 +115,14 @@ eraseSigma = 3; -eraseRadius = ceil(1.2.*(emc.('beadDiameter')./PIXEL_SIZE.*0.5)); +eraseRadius = ceil(1.2.*(emc.('beadDiameter')./emc.pixel_size_si.*0.5)); flgImodErase = 0 % Assuming that the first CTF zero is always less than this value -FIXED_FIRSTZERO = PIXEL_SIZE / (70*10^-10) ; -highCutoff = PIXEL_SIZE/emc.('defCutOff'); +FIXED_FIRSTZERO = emc.pixel_size_si / (70*10^-10) ; +highCutoff = emc.pixel_size_si/emc.('defCutOff'); % I still use the def for underfocus < 0 as this places the origin at the % focal plan in the microscope rather than on the specimen. Which makes % more sense to me. @@ -178,15 +157,15 @@ end % Starting at +/- 100nm -deltaZTolerance = deltaZTolerance / PIXEL_SIZE; +deltaZTolerance = deltaZTolerance / emc.pixel_size_si; % Use to check for proper gradient. -zShift = zShift / PIXEL_SIZE; +zShift = zShift / emc.pixel_size_si; % Tile size & overlap try tileSize = emc.('ctfTileSize'); catch - tileSize = floor(680e-10 / PIXEL_SIZE); + tileSize = floor(680e-10 / emc.pixel_size_si); end tileOverlap = 2; @@ -290,8 +269,8 @@ TLT(:,2:3) = repmat([0.00,0.00],size(TLT,1),1); TLT(:,5:10) = repmat([0,90.0,1.0,0.0,0.0,1.0],size(TLT,1),1); % Defocus will go at 15 - 12 and 13 currently unused. -TLT(:,16:18) = repmat([PIXEL_SIZE,Cs,WAVELENGTH],size(TLT,1),1); -TLT(:,19) = TLT(:,19) + AMPCONT; +TLT(:,16:18) = repmat([emc.pixel_size_si,emc.Cs,WAVELENGTH],size(TLT,1),1); +TLT(:,19) = TLT(:,19) + emc.AMPCONT; oddSize = [d1,d2,d3] - (1-mod([d1,d2,d3],2)); TLT(:,20:22) = repmat(oddSize,size(TLT,1),1); @@ -492,7 +471,7 @@ STACK = BH_eraseBeads(STACK,eraseRadius, fileName, scalePixelsBy,0,sortrows(TLT,1)); end -[ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',100,PIXEL_SIZE*10^10,samplingMaskStack); +[ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',100,emc.pixel_size_si*10^10,samplingMaskStack); SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); @@ -503,10 +482,10 @@ gpuDevice(gpuIDX) [d1,d2,d3] = size(STACK); - if (PIXEL_SIZE*10^10 < 0) + if (emc.pixel_size_si*10^10 < 0) flgCrop = 1; - [croppedIMG,pixelOUT] = cropIMG(STACK(:,:,1),PIXEL_SIZE*10^10); + [croppedIMG,pixelOUT] = cropIMG(STACK(:,:,1),emc.pixel_size_si*10^10); [d1C,d2C] = size(croppedIMG); clear croppedIMG tltForExp = TLT; tltForExp(:,16) = pixelOUT*10^-10; @@ -519,7 +498,7 @@ else flgCrop = 0; - pixelOUT = PIXEL_SIZE*10^10; + pixelOUT = emc.pixel_size_si*10^10; d1C = d1; d2C = d2; tltForExp = TLT; @@ -622,7 +601,7 @@ % % % % tmpTile = zeros([paddedSize.*[1,1],3],'single','gpuArray'); if flgCrop - [iProjection,~] = cropIMG(gpuArray(STACK(:,:,TLT(k,1))),PIXEL_SIZE*10^10); + [iProjection,~] = cropIMG(gpuArray(STACK(:,:,TLT(k,1))),emc.pixel_size_si*10^10); else iProjection = (gpuArray(STACK(:,:,TLT(k,1)))); end @@ -748,7 +727,7 @@ TLT = TLT(idx,:); % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast + % scaleFactor, defocus, emc.pixel_size_si, CS, Wavelength, Amplitude contrast fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... @@ -775,10 +754,10 @@ % TODO add a global switch for the damping - if (PIXEL_SIZE < 1*10^-10) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0,-1); + if (emc.pixel_size_si < 1*10^-10) + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH,DF,paddedSize,-emc.AMPCONT,-1.0,-1); else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH,DF,paddedSize,-emc.AMPCONT,-1.0); end try @@ -810,10 +789,10 @@ end DF = maxDef*10^-6; - if (PIXEL_SIZE < 1*10^-10) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0,-1); + if (emc.pixel_size_si < 1*10^-10) + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH,DF,paddedSize,-emc.AMPCONT,-1.0,-1); else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH,DF,paddedSize,-AMPCONT,-1.0); + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH,DF,paddedSize,-emc.AMPCONT,-1.0); end [ bg, bandpass, rV ] = prepare_spectrum( Hqz, highCutoff, freqVector, radialAvg, 0); @@ -885,13 +864,13 @@ df1 = maxDef*10^-6 - iDelDF*astigStep; df2 = maxDef*10^-6 + iDelDF*astigStep; - if (PIXEL_SIZE < 1*10^-10) + if (emc.pixel_size_si < 1*10^-10) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0,-1); + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH, ... + [df1,df2,iAng],size(radialForCTF{1}),-emc.AMPCONT,-1.0,-1); else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... - [df1,df2,iAng],size(radialForCTF{1}),-AMPCONT,-1.0); + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH, ... + [df1,df2,iAng],size(radialForCTF{1}),-emc.AMPCONT,-1.0); end @@ -929,15 +908,15 @@ % values |df1| < |df2| which is against convention. if abs(df1) >= abs(df2) - if (PIXEL_SIZE < 1*10^-10) + if (emc.pixel_size_si < 1*10^-10) - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH, ... [df1,df2,iAng+mAng], ... - size(radialForCTF{1}), -AMPCONT,-1.0,-1); + size(radialForCTF{1}), -emc.AMPCONT,-1.0,-1); else - [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... + [ Hqz ] = BH_ctfCalc(radialForCTF,emc.Cs,WAVELENGTH, ... [df1,df2,iAng+mAng], ... - size(radialForCTF{1}), -AMPCONT,-1.0); + size(radialForCTF{1}), -emc.AMPCONT,-1.0); end @@ -971,7 +950,7 @@ radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; currentDefocusEst = maxDef; currentDefocusWin = (defWIN*.25); - measuredVsExpected(1,:) = [maxDef + zShift*PIXEL_SIZE*10^6, maxDef, maxDef - zShift*PIXEL_SIZE*10^6]; + measuredVsExpected(1,:) = [maxDef + zShift*emc.pixel_size_si*10^6, maxDef, maxDef - zShift*emc.pixel_size_si*10^6]; measuredVsExpected(2,2) = maxDef; % Add the determined defocus, and write out with mic paramters as well. TLT(:,15) = repmat(maxDef*10^-6,size(TLT,1),1); @@ -990,7 +969,7 @@ TLT = TLT(idx,:); % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast + % scaleFactor, defocus, emc.pixel_size_si, CS, Wavelength, Amplitude contrast fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... @@ -1027,7 +1006,7 @@ TLT = TLT(idx,:); % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, pixelSize, CS, Wavelength, Amplitude contrast + % scaleFactor, defocus, emc.pixel_size_si, CS, Wavelength, Amplitude contrast fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... @@ -1230,7 +1209,7 @@ end end -function [ diagnosticIMG ] = make_diagnosticIMG( Hqz, pixelSize, bandpass, bg, IMG) +function [ diagnosticIMG ] = make_diagnosticIMG( Hqz, emc.pixel_size_si, bandpass, bg, IMG) iImg = 1; @@ -1240,7 +1219,7 @@ BH_multi_gridCoordinates(size(Hqz),'Cartesian',... 'GPU',{'none'},1,0,1); -radialGrid = radialGrid ./ pixelSize; +radialGrid = radialGrid ./ emc.pixel_size_si; lowCut = radialGrid(1, find(bandpass , 1,'first')); highCut= radialGrid(1, find(bandpass , 1,'last')); diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index 00fb1119..548a0b9e 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -50,36 +50,19 @@ pathName = '.'; end -PIXEL_SIZE = emc.('PIXEL_SIZE'); - Cs = emc.('Cs'); VOLTAGE = emc.('VOLTAGE'); AMPCONT = emc.('AMPCONT'); -ctfParams = [PIXEL_SIZE*10^10,VOLTAGE./1000,Cs.*1000,AMPCONT]; +ctfParams = [emc.pixel_size_si*10^10,VOLTAGE./1000,Cs.*1000,AMPCONT]; -% Sanity check -if (PIXEL_SIZE > 20e-10 || PIXEL_SIZE < 0) - error('pixel size should be [0,20e-10]'); -elseif (Cs > 5*10^-3 || Cs < 0) - error('Cs should be[1e-3,10e-3]'); -elseif(VOLTAGE > 1000e3 || VOLTAGE < 20e3) - error ('VOLTAGE should be [20e3,1000e3]'); -elseif (AMPCONT < 0.025 || AMPCONT > 0.25) - error('AMPCONT should be [0.025,0.25]'); -else - WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; -end +WAVELENGTH = 10^-12*1226.39/sqrt(VOLTAGE + 0.97845*10^-6*VOLTAGE^2) ; -if (Cs == 0) - fprintf('You set Cs to zero, over-riding to 5 micron\n'); - Cs = 5e-6; -end % Assuming that the first CTF zero is always less than this value -FIXED_FIRSTZERO = PIXEL_SIZE / 40*10^-10 ; +FIXED_FIRSTZERO = emc.pixel_size_si / 40*10^-10 ; % Size to padTile to should be even, large, and preferably a power of 2 try @@ -90,7 +73,7 @@ % Tile size & overlap tileOverlap = 4; -tileSize = floor(680e-10 / PIXEL_SIZE); +tileSize = floor(680e-10 / emc.pixel_size_si); tileSize = tileSize + mod(tileSize,2); fprintf('Using a tile size of %d',tileSize); overlap = floor(tileSize ./ tileOverlap); @@ -152,7 +135,7 @@ BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU',{'none'},1,1,0); - radialForCTF = {radialForCTF./PIXEL_SIZE,1,phi} ; + radialForCTF = {radialForCTF./emc.pixel_size_si,1,phi} ; clear phi diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 5909dbb3..cecad6cd 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -151,19 +151,16 @@ eraseStack = sprintf('rm cache/%s_*.fixed',STACK_PRFX); eraseRec = sprintf('rm cache/%s_*.rec',STACK_PRFX); - % Converte bead diameter to pixels and add a little to be safe. - PIXEL_SIZE = emc.('PIXEL_SIZE'); - eraseSigma = 3;%emc.('beadSigma'); - eraseRadius = ceil(1.2.*(emc.('beadDiameter')./PIXEL_SIZE.*0.5)); + eraseRadius = ceil(1.2.*(emc.('beadDiameter')./emc.pixel_size_si.*0.5)); flgImodErase = 0 % FIXME, this should be stored from previous mask calc and accessed there. % For now just take based on tomogram (which will be larger than the true specimen thickness) %THICKNESS = recGeomForThickness.(sprintf('%s_1',STACK_PRFX)); - %THICKNESS = min(10,abs(THICKNESS(1,3)-THICKNESS(2,3)).*PIXEL_SIZE.*10^9); + %THICKNESS = min(10,abs(THICKNESS(1,3)-THICKNESS(2,3)).*emc.pixel_size_si.*10^9); THICKNESS = 100; % Assuming all extreme pixels have already been removed from the stack. %PRJ_STACK = {sprintf('%s_local04_18.mrc',mjIDX)};%,sprintf('%s_local14_18.mrc',mjIDX),sprintf('%s_local24_18.mrc',mjIDX),sprintf('%s_local34_18.mrc',mjIDX)}; @@ -468,7 +465,7 @@ TLT(:,4) = mbTLT; if (defShifts) TLT(:,15) = TLT(:,15) + defShifts; - TLT(:,16) = PIXEL_SIZE; + TLT(:,16) = emc.pixel_size_si; end @@ -497,7 +494,7 @@ fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',... THICKNESS, STACK_PRFX); - [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,PIXEL_SIZE*10^10,gpuArray(samplingMaskStack)); + [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,emc.pixel_size_angstroms,gpuArray(samplingMaskStack)); SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName)); @@ -514,7 +511,7 @@ fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',... THICKNESS, STACK_PRFX); - [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,PIXEL_SIZE*10^10,gpuArray(samplingMaskStack)); + [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,emc.pixel_size_angstroms,gpuArray(samplingMaskStack)); SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName),iPixelHeader,iOriginHeader); diff --git a/logicals/EMC_assert_integer.m b/logicals/EMC_assert_integer.m index e5733a25..2b5246a3 100644 --- a/logicals/EMC_assert_integer.m +++ b/logicals/EMC_assert_integer.m @@ -3,22 +3,41 @@ function EMC_assert_integer(input_val, varargin) % Use the default for any length assert_length = false; assert_passed = false; - if ( nargin == 2 ) + assert_range = false; + if ( nargin > 1 ) assert_length = true; wanted_numel = varargin{1}; - elseif ( nargin > 2 ) - error('EMC_assert_numeric: too many input arguments'); + if ~isa(wanted_numel, "integer") + error('EMC_assert_numeric: second argument must be an integer'); + end + if (nargin == 3) + assert_range = true; + range = varargin{2}; + if ( ~isa(range, 'numeric') || numel(range) ~= 2 ) + error('EMC_assert_integer: third argument must be numeric with two elements'); + end + else + error('EMC_assert_numeric: too many input arguments'); + end end + % If it is not numeric, throw an error if ( isa(input_val, 'integer') ) + % Does it have enough values? if ( assert_length ) if ( numel(input_val) == wanted_numel ) assert_passed = true; end + if (assert_range) + for val_in_range = 1:wanted_numel + if ( input_val(val_in_range) < range(1) || input_val(val_in_range) > range(2) ) + assert_passed = false; + end + end + end else assert_passed = true; end - return; end if ( assert_passed == false ) diff --git a/logicals/EMC_assert_numeric.m b/logicals/EMC_assert_numeric.m index dbde3505..cf0ebd9a 100644 --- a/logicals/EMC_assert_numeric.m +++ b/logicals/EMC_assert_numeric.m @@ -3,22 +3,41 @@ function EMC_assert_numeric(input_val, varargin) % Use the default for any length assert_length = false; assert_passed = false; - if ( nargin == 2 ) + assert_range = false; + if ( nargin > 1 ) assert_length = true; wanted_numel = varargin{1}; - elseif ( nargin > 2 ) - error('EMC_assert_numeric: too many input arguments'); + if ~isa(wanted_numel, "integer") + error('EMC_assert_numeric: second argument must be an integer'); + end + if (nargin == 3) + assert_range = true; + range = varargin{2}; + if ( ~isa(range, 'numeric') || numel(range) ~= 2 ) + error('EMC_assert_numeric: third argument must be numeric with two elements'); + end + else + error('EMC_assert_numeric: too many input arguments'); + end end + % If it is not numeric, throw an error if ( isa(input_val, 'numeric') ) + % Does it have enough values? if ( assert_length ) if ( numel(input_val) == wanted_numel ) assert_passed = true; end + if (assert_range) + for val_in_range = 1:wanted_numel + if ( input_val(val_in_range) < range(1) || input_val(val_in_range) > range(2) ) + assert_passed = false; + end + end + end else assert_passed = true; end - return; end if ( assert_passed == false ) diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index e9dd4085..fe557531 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -40,7 +40,52 @@ end end +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Asserts on required parameters +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +if isfield(emc, 'nGPUs') + EMC_assert_integer(emc.nGPUs, 1); +else + error('nGPUs is a required parameter'); +end + +if isfield(emc, 'PIXEL_SIZE') + EMC_assert_numeric(emc.PIXEL_SIZE, 1, [0, 100e-10]); + emc.pixel_size_si = emc.PIXEL_SIZE; + emc.pixel_size_angstroms = emc.PIXEL_SIZE.*10^10; +else + error('PIXEL_SIZE is a required parameter'); +end + +if isfield(emc, 'Cs') + EMC_assert_numeric(emc.Cs, 1, [0, 10e-3]); +else + error('Cs is a required parameter'); +end + +if isfield(emc, 'VOLTAGE') + EMC_assert_numeric(emc.VOLTAGE, 1, [20e3, 1000e3]); +else + error('VOLTAGE is a required parameter'); +end + +if isfield(emc, 'AMPCONT') + EMC_assert_numeric(emc.AMPCONT, 1, [0.0, 1.0]); + if emc.Cs == 0 + emc.Cs = 1e-10; + end + +else + error('AMPCONT is a required parameter'); +end + + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Now check for optional parameters +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Early development parameter, used to store more than one orientation during template matching % and use for further refinement. diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index 24014fe0..b7de1de1 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -23,9 +23,9 @@ cycleNumber = sprintf('cycle%0.3u', CYCLE); emc = BH_parseParameterFile(PARAMETER_FILE); -dupSampling = ceil(10e-10 / emc.('PIXEL_SIZE')); +dupSampling = ceil(10e-10 / emc.pixel_size_si); -pixelSize = emc.('PIXEL_SIZE').*dupSampling.*10^10; +pixelSize = emc.pixel_size_angstroms.*dupSampling; latticeRadius = emc.('particleRadius'); diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 9aad7464..76881be7 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -87,7 +87,7 @@ outputPrefix = sprintf('./FSC/%s_%s', cycleNumber, emc.('subTomoMeta')); samplingRate = emc.('Ali_samplingRate'); -pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; +emc.pixel_size_si = emc.pixel_size_angstroms .* samplingRate; if ( flgCones ) coneInc = 30; diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 2777e292..f2a2df91 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -151,8 +151,8 @@ refSamplingRate= emc.('Ali_samplingRate'); randomSubset = emc.('Pca_randSubset'); maxEigs = emc.('Pca_maxEigs'); -pixelSize = emc.('PIXEL_SIZE').*10^10.*samplingRate; -refPixelSize = emc.('PIXEL_SIZE').*10^10.*refSamplingRate; +pixelSize = emc.pixel_size_angstroms .* samplingRate; +refPixelSize = emc.pixel_size_angstroms .* refSamplingRate; % FIMXE: Probably remove this incomplete idea if (refSamplingRate ~= samplingRate) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index ee889c7f..3d116fd4 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -501,8 +501,8 @@ end % The model is scaled to full sampling prior to passing to tiltalign, % make sure the header in the synthetic stack is set appropriately. - fullPixelSize = emc.('PIXEL_SIZE').*10^10; - pixelSize = fullPixelSize.*samplingRate; + fullPixelSize = emc.pixel_size_angstroms; + pixelSize = fullPixelSize .* samplingRate; try eraseMaskType = emc.('Peak_mType'); diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 3802e41f..b9c76461 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -172,14 +172,11 @@ % The model is scaled to full sampling prior to passing to tiltalign, % make sure the header in the synthetic stack is set appropriately. - fullPixelSize = emc.('PIXEL_SIZE').*10^10; - + fullPixelSize = emc.pixel_size_angstroms; pixelSize = fullPixelSize.*samplingRate; - PARTICLE_RADIUS = floor(max(emc.('particleRadius')./pixelSize)); - [~,tiltBaseName,~] = fileparts(tiltList{1}); mbOUT = {[tmpCache],[mapBackIter+1],[tiltBaseName]}; fprintf('\nmBOUT name is %smapBack%d/%s\n',mbOUT{1:3}); @@ -646,7 +643,7 @@ sigma = 10.0; score = 10.0; % TODO test with scaled CCC score? scoreChange = 0.0; - pixelSize = emc.('PIXEL_SIZE') * 10^10; + pixelSize = emc.pixel_size_angstroms; micVoltage = emc.('VOLTAGE') * 10^-3; micCS = emc.('Cs') * 10^3; ampContrast = emc.('AMPCONT') * 10^0; @@ -745,7 +742,7 @@ 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : '%2.2d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,emc.('PIXEL_SIZE')*10^10, ... + symmetry,emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); fprintf(recScript, '\neof\n'); @@ -814,7 +811,7 @@ 'no\n',...Threshold input reconstruction [No] : '%2.2d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_REFINE3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,emc.('PIXEL_SIZE')*10^10, ... + symmetry,emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... resForFitting,resForFitting,maxThreads); @@ -867,7 +864,7 @@ 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : '%2.2d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'), baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... - symmetry,emc.('PIXEL_SIZE')*10^10, ... + symmetry,emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); fprintf(recScript, '\neof\n'); diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index f68d9039..c79af58e 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -50,14 +50,11 @@ end end - - rotConvention = 'Bah'; if ( emc.doHelical ) rotConvention = 'Helical'; end - % The weights are only re-estimated for an out of plane search. Until this % happens, they are not valid. if (emc.track_stats) @@ -78,20 +75,18 @@ if ( emc.classification ); emc.classification = -1 ; end flgGold=1; -pixelSize = emc.('PIXEL_SIZE').*10^10; +emc.pixel_size_angstroms = emc.('PIXEL_SIZE').*10^10; -nGPUs = emc.('nGPUs'); % Optionally specify gpu idxs -if numel(nGPUs) == 1 - gpuList = 1:nGPUs; +if numel(emc.nGPUs) == 1 + gpuList = 1:emc.nGPUs; else - gpuList = nGPUs; - nGPUs = length(gpuList); + gpuList = emc.nGPUs; + emc.nGPUs = length(gpuList); end - try loadTomo = emc.('loadTomo') catch @@ -283,8 +278,8 @@ end outputPrefix = sprintf('%s_%s',cycleNumber, emc.('subTomoMeta')); -pixelSize = pixelSize .* samplingRate; -peakSearch = floor(0.85.*emc.('particleRadius')./pixelSize) +emc.pixel_size_angstroms = emc.pixel_size_angstroms .* samplingRate; +peakSearch = floor(0.85.*emc.('particleRadius')./emc.pixel_size_angstroms); peakCOM = [1,1,1].*3; @@ -426,7 +421,7 @@ if (emc.classification) [ ~, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', pixelSize); + BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms); % These are used when 'Cluster' is called, to take the masking parameters % from focused PCA/Classification, to produce a montage with reduced % Z-dimension & low pass filtering to be used in decision making but not @@ -435,11 +430,11 @@ [~, ~, ~, pcaMaskCenter ] = ... - BH_multi_maskCheck(emc, 'Cls', pixelSize); + BH_multi_maskCheck(emc, 'Cls', emc.pixel_size_angstroms); else [ ~, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', pixelSize); + BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms); end @@ -447,7 +442,7 @@ BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) padREF = [0,0,0;0,0,0]; -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); +[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,emc.nGPUs, sizeCalc(1),limitToOne); origMaskSize = sizeMask; %%%%% Considering removing doNotTrim and making this the default. Temporarily @@ -593,7 +588,7 @@ radial_shrink_factor = 2; - [ normal_vect, chi2 ] = BH_fit_ellipsoidal_prior(pixelSize .* particle_coords(positions_to_analyze,3:5), ... + [ normal_vect, chi2 ] = BH_fit_ellipsoidal_prior(emc.pixel_size_angstroms .* particle_coords(positions_to_analyze,3:5), ... emc.('particleRadius')(3), ... radial_shrink_factor, ... display_fit); @@ -750,7 +745,7 @@ end % % Clear all of the GPUs prior to entering the main processing loop -for iGPU = 1:nGPUs +for iGPU = 1:emc.nGPUs gpuDevice(iGPU); end @@ -759,7 +754,7 @@ % for iParProc = parVect % Get the gpuIDX assigned to this process - gpuIDXList = mod(parVect+nGPUs,nGPUs)+1; + gpuIDXList = mod(parVect+emc.nGPUs,emc.nGPUs)+1; iGPUidx = gpuIDXList(iParProc); gpuDevice(iGPUidx); fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); @@ -788,7 +783,7 @@ [cccWeight,~,~,~,~,~] = BH_multi_gridCoordinates(sizeCalc, ... 'Cartesian','GPU',... {'none'},1,0,1); - cccWeight = (cccWeight ./ pixelSize).^2; + cccWeight = (cccWeight ./ emc.pixel_size_angstroms).^2; @@ -1087,11 +1082,11 @@ particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); positionList(iSubTomo,[11:13]+26*(iPeak-1)) = shiftVAL+CUTPADDING+ceil((sizeWindow+1)./2); if (emc.projectVolumes) - SAVE_IMG(sum(iParticle,3),particleOUT_name,pixelSize); + SAVE_IMG(sum(iParticle,3),particleOUT_name,emc.pixel_size_angstroms); else particleOUT = BH_padZeros3d(gather(iParticle), CUTPADDING.*[1,1,1], ... CUTPADDING.*[1,1,1], 'cpu', 'single'); - SAVE_IMG(particleOUT,particleOUT_name,pixelSize); + SAVE_IMG(particleOUT,particleOUT_name,emc.pixel_size_angstroms); end @@ -1383,7 +1378,7 @@ if (doNotTrim) && (emc.classification) filteredClass= cell(maxClasses,2); % low-pass to see class averages more clearly. - % % [ bandpassFilt ] = BH_bandpass3d( sizeMask, 0.2, 300, 30, 'GPU',pixelSize); + % % [ bandpassFilt ] = BH_bandpass3d( sizeMask, 0.2, 300, 30, 'GPU',emc.pixel_size_angstroms); end for iClass = 1:maxClasses @@ -1394,7 +1389,7 @@ if (eachTomo) system('mkdir -p initialTomoAvgs'); % sizeWeight mask is sizeMask or 128^3 whichever is larger - % bandpassFiltTomo = BH_bandpass3d( sizeCalc, lpTomo(1), lpTomo(2), lpTomo(3), 'GPU',pixelSize); + % bandpassFiltTomo = BH_bandpass3d( sizeCalc, lpTomo(1), lpTomo(2), lpTomo(3), 'GPU',emc.pixel_size_angstroms); for iParProc = 1:nParProcesses nTomos = 1; @@ -1404,7 +1399,7 @@ classTmp = avgTomoResults{iParProc}{nTomos}; avgTomoResults{iParProc}{nTomos} = []; - SAVE_IMG(classTmp,tomoName,pixelSize); + SAVE_IMG(classTmp,tomoName,emc.pixel_size_angstroms); clear classTmp nTomos = nTomos + 1; end @@ -1538,7 +1533,7 @@ if (flgFinalAvg) system(sprintf('mv %s preHalfSetAli_%s',imout,imout)); end - SAVE_IMG(montOUT, imout, pixelSize); + SAVE_IMG(montOUT, imout, emc.pixel_size_angstroms); %%%%%%%% [montOUT, imgLocations] = BH_montage4d(avgWedge(:,iGold), ''); @@ -1553,15 +1548,7 @@ system(sprintf('mv %s preHalfSetAli_%s',imout,imout)); end SAVE_IMG(montOUT, imout); - %%%%%%%% - % % % if (doNotTrim) && (emc.classification) - % % % [montOUT, ~] = BH_montage4d(filteredClass(:,iGold), ''); - % % % imout = sprintf('%s_filtered%d_%s_%s.mrc',outputPrefix, ... - % % % className, fieldPrefix, halfSet); - % % % SAVE_IMG(MRCImage(gather(montOUT)), imout,pixelSize); - % % % - % % % end - + if (saveClassSum > -1) imgCounts = gather([classVector{iGold}(1,:) ; nExtracted(:,iGold)']); @@ -1571,7 +1558,7 @@ imout = sprintf('%s_class%d_%s_%s_NoWgt.mrc',outputPrefix, ... saveClassSum, 'Raw', halfSet); classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'Raw', halfSet); - SAVE_IMG(montOUT, imout,pixelSize); + SAVE_IMG(montOUT, imout,emc.pixel_size_angstroms); masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; [montOUT, imgLocations] = BH_montage4d(classWgtSum(iGold), ''); @@ -1579,7 +1566,7 @@ imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... saveClassSum, 'Raw', halfSet); classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'Raw', halfSet); - SAVE_IMG(montOUT, imout,pixelSize); + SAVE_IMG(montOUT, imout,emc.pixel_size_angstroms); masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; end @@ -1610,7 +1597,7 @@ % clean everything up, since this function is called from other functions. delete(gcp('nocreate')); -for iGPU = 1:nGPUs +for iGPU = 1:emc.nGPUs gpuDevice(gpuList(iGPU)); end @@ -1623,13 +1610,13 @@ try - EMC_parpool(nGPUs) + EMC_parpool(emc.nGPUs) catch delete(gcp('nocreate')); - EMC_parpool(nGPUs) + EMC_parpool(emc.nGPUs) end -for iGPU = 1:nGPUs +for iGPU = 1:emc.nGPUs gpuDevice(gpuList(iGPU)); end @@ -1745,7 +1732,7 @@ refTMP = gather(BH_multi_cRef_Vnorm(fscParams, aliParams, mskParams,... {refIMG{1}{iOdd},refIMG{2}{iEve}}, ... {refWGT{1}{iOdd},refWGT{2}{iEve}}, ... - flgCombine,flgRefCutOff, pixelSize, bFactorSend)); + flgCombine,flgRefCutOff, emc.pixel_size_angstroms, bFactorSend)); if ~(flgFinalAvg) refIMG{1}{iOdd} = refTMP{1,1}; @@ -1768,10 +1755,9 @@ else halfSet = 'STD'; end - iRef imgIN = sprintf('class_%d_Locations_%s_%s_NoWgt', ... className, fieldPrefix, halfSet); - imgCounts = masterTM.(cycleNumber).(imgIN){3}; + imgCounts = masterTM.(cycleNumber).(imgIN){3}; % Save the unweighted, weighted imgs, weightes, optionally filtered. if (flgFinalAvg) @@ -1779,7 +1765,7 @@ imout = sprintf('%s_class%d_%s_bFact-%d.mrc',outputPrefix, ... className, 'final',emc.Fsc_bfactor(iBfactor)); - SAVE_IMG(refTMP{iBfactor}, imout, pixelSize); + SAVE_IMG(refTMP{iBfactor}, imout, emc.pixel_size_angstroms); end else @@ -1789,7 +1775,7 @@ classOut = sprintf('class_%d_Locations_%s_%s', className,fieldPrefix, halfSet); masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; - SAVE_IMG(montOUT, imout, pixelSize); + SAVE_IMG(montOUT, imout, emc.pixel_size_angstroms); end %%%%%%% end From 8f7984a1c8dcd19e21f7746b4bc3f2321cfd7c1f Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 12:24:01 -0500 Subject: [PATCH 014/151] Remove load tomo parameter --- alignment/BH_alignRaw3d_v2.m | 38 ++++++------------------------ transformations/BH_average3d.m | 43 ++++++---------------------------- 2 files changed, 14 insertions(+), 67 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index ffde1350..e8ea159e 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -139,11 +139,7 @@ peakCOM = [1,1,1].*3; className = emc.('Raw_className'); -try - loadTomo = emc.('loadTomo') -catch - loadTomo = 0; -end + try eraseMaskType = emc.('Peak_mType'); eraseMaskRadius = emc.('Peak_mRadius')./emc.pixel_size_angstroms; @@ -265,12 +261,7 @@ flgLimitToOneProcess = 0; end -if ( loadTomo ) - limitToOne = loadTomo; - if (flgLimitToOneProcess) - limitToOne = min(limitToOne, flgLimitToOneProcess); - end -elseif (flgLimitToOneProcess) +if (flgLimitToOneProcess) limitToOne = flgLimitToOneProcess; else limitToOne = emc.('nCpuCores'); @@ -832,15 +823,8 @@ else [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,loadTomo); - if ( loadTomo ) - volHeader = struct(); - volHeader.('nX') = size(volumeData,1); - volHeader.('nY') = size(volumeData,2); - volHeader.('nZ') = size(volumeData,3); - else + samplingRate,iGPUidx,reconScaling,0); volHeader = getHeader(volumeData); - end end @@ -949,18 +933,10 @@ continue; end else - - if ( loadTomo ) - iparticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3))); - - else - iparticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - end - + + iparticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); end [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... padVAL(2,1:3), 'GPU', 'singleTaper'); diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index c79af58e..8f9b5364 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -75,8 +75,6 @@ if ( emc.classification ); emc.classification = -1 ; end flgGold=1; -emc.pixel_size_angstroms = emc.('PIXEL_SIZE').*10^10; - % Optionally specify gpu idxs if numel(emc.nGPUs) == 1 @@ -86,13 +84,6 @@ emc.nGPUs = length(gpuList); end - -try - loadTomo = emc.('loadTomo') -catch - loadTomo = 0 -end - try scaleCalcSize = emc.('scaleCalcSize'); catch @@ -253,12 +244,7 @@ flgLimitToOneProcess = 0; end -if ( loadTomo ) - limitToOne = loadTomo; - if (flgLimitToOneProcess) - limitToOne = min(limitToOne, flgLimitToOneProcess); - end -elseif interpOrder == 4 +if (interpOrder == 4) limitToOne = 1; elseif (flgLimitToOneProcess) limitToOne = flgLimitToOneProcess; @@ -867,16 +853,9 @@ [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( ... tomoList{iTomo}, ... reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,loadTomo); + samplingRate,iGPUidx,reconScaling,0); - if ( loadTomo ) - volHeader = struct(); - volHeader.('nX') = size(volumeData,1); - volHeader.('nY') = size(volumeData,2); - volHeader.('nZ') = size(volumeData,3); - else - volHeader = getHeader(volumeData); - end + volHeader = getHeader(volumeData); end @@ -1052,18 +1031,10 @@ continue; end else - - if ( loadTomo ) - iParticle = gpuArray(volumeData(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3))); - - else - iParticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - end - + + iParticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); end From 6cd5840eb1608af025602c80e4b1b95bd35b377e Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 13:44:44 -0500 Subject: [PATCH 015/151] Whole pile of parameteres moved --- alignment/BH_alignRaw3d_v2.m | 87 +------- alignment/BH_templateSearch3d_2.m | 14 +- logicals/BH_multi_parallelJobs.m | 7 +- metaData/BH_geometryAnalysis.m | 2 +- metaData/BH_parseParameterFile.m | 97 ++++++++- statistics/BH_clusterPub.m | 9 +- statistics/BH_fscGold_class.m | 207 +++--------------- statistics/BH_pcaPub.m | 177 ++++++---------- synthetic/BH_synthetic_mapBack.m | 5 - testScripts/emClarity.m | 5 +- transformations/BH_average3d.m | 323 +++++++++-------------------- transformations/BH_axialSymmetry.m | 72 ------- 12 files changed, 310 insertions(+), 695 deletions(-) delete mode 100755 transformations/BH_axialSymmetry.m diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index e8ea159e..4e33b0f7 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -41,7 +41,7 @@ cpuVar = struct(); GPUVar = struct(); -startTime = clock; +startTime = datetime("now"); CYCLE = EMC_str2double(CYCLE); cycle_numerator = ''; cycle_denominator =''; @@ -61,8 +61,6 @@ end - - emc = BH_parseParameterFile(PARAMETER_FILE); cycleNumber = sprintf('cycle%0.3u', CYCLE); load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); @@ -70,36 +68,12 @@ reconScaling = 1; -% TODO decide on a "reasonable" padding based on expected shifts. -try - CUTPADDING = subTomoMeta.('CUTPADDING') -catch - CUTPADDING=20 -end - - - -try - symmetry_op = emc.('symmetry'); -catch - error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); -end - try use_new_grid_search = emc.('use_new_grid_search'); catch use_new_grid_search = true; end -try - force_no_symmetry = emc.('force_no_symmetry'); -catch - force_no_symmetry = false; -end -if (force_no_symmetry) - symmetry_op='C1' - fprintf('\nWarning, overriding symmetry in the alignment. THis is just for benchmarking\n'); -end maxGoldStandard = subTomoMeta.('maxGoldStandard'); @@ -159,13 +133,6 @@ rotConvention = 'Helical'; end - -try - scaleCalcSize = emc.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end -% % % % if (emc.classification || emc.multi_reference_alignment) if (emc.classification) refName = emc.('Ref_className'); else @@ -252,28 +219,12 @@ BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) -try - flgLimitToOneProcess = emc.('flgLimitToOneProcess'); -catch - flgLimitToOneProcess = 0; -end - -if (flgLimitToOneProcess) - limitToOne = flgLimitToOneProcess; -else - limitToOne = emc.('nCpuCores'); -end -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); +[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, sizeCalc(1), emc.nCpuCores); if ( flgReverseOrder ) - % fprintf('nCpuCores is %d\n', limitToOne); - % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); - % for iParProc = 1:nParProcesses - % iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' - % end % Flip the order for reverse processing on a second machine. This will also disable saving of % of the metadata so there aren't conflicts. for iParProc = 1:nParProcesses @@ -281,25 +232,7 @@ end elseif ( flgStartThird ) - % fprintf('nCpuCores is %d\n', limitToOne); - % [ nParProcesses, iterList_full] = BH_multi_parallelJobs(nTomograms,nGPUs*cycle_denominator, sizeCalc(1),limitToOne*cycle_denominator); - - % for iParProc = 1:nParProcesses - % iterList_full{iParProc} = sortedTomoIDX(iterList_full{iParProc})'; - % end - - % % Need to scale this back down - % nParProcesses = limitToOne; - - % Shift to start at one third through to process on a third machine. This will also disable saving of - % of the metadata so there aren't conflicts. - % iterList = {}; - % for iParProc = 1:nParProcesses - % idx = cycle_numerator + (iParProc-1)*cycle_denominator; - % if (idx <= length(iterList_full)) - % iterList{iParProc} = iterList_full{idx}; - % end - % end + for iParProc = 1:nParProcesses % Note the use of floor is more like ceiling here (rounds away from % zero) @@ -311,14 +244,7 @@ else - % error('not supported run config'); - % fprintf('nCpuCores is %d\n', limitToOne); - % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,nGPUs, sizeCalc(1),limitToOne); - % for iParProc = 1:nParProcesses - % iterList{iParProc} = sortedTomoIDX(iterList{iParProc})' - % end - - + end if any(peakSearch > maskRadius) @@ -608,7 +534,7 @@ updateWeights = false; gridSearch = ''; if (use_new_grid_search) - gridSearch = eulerSearch(symmetry_op, angleSearch(1),... + gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, true); nAngles = sum(gridSearch.number_of_angles_at_each_theta); inPlaneSearch = gridSearch.parameter_map.psi @@ -689,7 +615,6 @@ parVect = 1:nParProcesses; fprintf('Starting main loopwith N references %d\n', nReferences(1)); parfor iParProc = parVect - symmetry = symmetry_op; % Why TF would this be necessary? bestAngles_tmp = struct(); geometry_tmp = geometry; diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 61619ee4..ad7ddaf1 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -33,7 +33,7 @@ % SYMMETRY = EMC_str2double(SYMMETRY); SYMMETRY=1; -startTime = clock ; +startTime = datetime("now") ; emc = BH_parseParameterFile(PARAMETER_FILE); @@ -100,12 +100,6 @@ use_new_grid_search = true; end -try - symmetry = emc.('symmetry'); -catch - error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); -end - try eraseMaskType = emc.('Peak_mType'); catch @@ -313,7 +307,7 @@ if (use_new_grid_search) - gridSearch = eulerSearch(symmetry, angleSearch(1),... + gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, false); nAngles = sum(gridSearch.number_of_angles_at_each_theta); inPlaneSearch = gridSearch.parameter_map.psi; @@ -1183,7 +1177,7 @@ fprintf('rmDim %f szK %f\n', rmDim,szK); removalMask = BH_mask3d(eraseMaskType,[2,2,2].*rmDim+1,eraseMaskRadius,[0,0,0]); rmInt = interpolator(gpuArray(removalMask),[0,0,0],[0,0,0],rotConvention ,'forward','C1'); -symOps = interpolator(gpuArray(removalMask),[0,0,0],[0,0,0],rotConvention ,'forward',symmetry); +symOps = interpolator(gpuArray(removalMask),[0,0,0],[0,0,0],rotConvention ,'forward',emc.symmetry); maskCutOff = 0.98; nIncluded = gather(sum(sum(sum(removalMask > maskCutOff)))); @@ -1424,7 +1418,7 @@ -fprintf('Total execution time : %f seconds\n', etime(clock, startTime)); +fprintf('Total execution time : %f seconds\n', datetime("now") - startTime); diff --git a/logicals/BH_multi_parallelJobs.m b/logicals/BH_multi_parallelJobs.m index 2f4783ea..5ad4a1d6 100644 --- a/logicals/BH_multi_parallelJobs.m +++ b/logicals/BH_multi_parallelJobs.m @@ -1,8 +1,5 @@ -function [nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, ... - nGPUs, ... - calcSize,flgAvg) -%UNTITLED Summary of this function goes here -% Detailed explanation goes here +function [nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, calcSize, flgAvg) + % Rough Scaling of processes by mem available. Assuming all gpus are % equivalent, which is often true, but this should be improved. The current diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 99337c5b..6038ccf6 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -50,7 +50,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... end end -startTime = clock; +startTime = datetime("now"); cycleNumber = sprintf('cycle%0.3u', CYCLE); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index fe557531..02e08051 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -45,11 +45,24 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% if isfield(emc, 'nGPUs') - EMC_assert_integer(emc.nGPUs, 1); + EMC_assert_integer(emc.nGPUs, 1, [1, 1000]); else error('nGPUs is a required parameter'); end +if isfield(emc, 'nCpuCores') + EMC_assert_integer(emc.nCpuCores, 1, [1, 1000]); +else + error('nCpuCores is a required parameter'); +end + +symmetry_has_been_checked = false; +if ~isfield(emc, 'symmetry') + %TODO asserts on allowed values for symmetry paraemeter + error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); +end +symmetry_has_been_checked = true; + if isfield(emc, 'PIXEL_SIZE') EMC_assert_numeric(emc.PIXEL_SIZE, 1, [0, 100e-10]); emc.pixel_size_si = emc.PIXEL_SIZE; @@ -121,6 +134,13 @@ emc.('Fsc_bfactor') = 40.0; end +if isfield(emc, 'flgCones') + EMC_assert_boolean(emc.flgCones) +else + emc.('Fsc_bfactor') = false; +end + + % Used to downweight higher frequencies based on relative CCC scores. % Based on one of Niko's papers, but catching some edge cases for tomo. % Overwritten if cycle == 0 as the scores from template matching do not work for this metric as they are SNR not CCC. @@ -197,5 +217,80 @@ emc = EMC_assert_deprecated_substitution(emc, false, 'multi_reference_alignment', 'flgMultiRefAlignment'); EMC_assert_boolean(emc.multi_reference_alignment); +% Zero padding of the volumes before alignment/other FFT ops +emc = EMC_assert_deprecated_substitution(emc, 1.5, 'scale_calc_size', 'scaleCalcSize'); +EMC_assert_numeric(emc.scale_calc_size, 1, [1.0, 2.0]); + +emc = EMC_assert_deprecated_substitution(emc, false, 'limit_to_one_core', 'flgLimitToOneProcess'); +EMC_assert_boolean(emc.limit_to_one_core); + +if (emc.limit_to_one_core) + emc.nCpuCores = 1; +end + +if isfield(emc, 'force_no_symmetry') + EMC_assert_boolean(emc.force_no_symmetry) + if (~symmetry_has_been_checked) + error('force_no_symmetry must be after symmetry check'); + end + % Warning must be after symmetry check + if (force_no_symmetry) + emc.symmetry='C1'; + end +else + force_no_symmetry = false; +end + +if isfield(emc, 'Pca_constrain_symmetry') + EMC_assert_boolean(emc.Pca_constrain_symmetry) +else + emc.Pca_constrain_symmetry = false; +end + +emc = EMC_assert_deprecated_substitution(emc, false, 'fsc_with_chimera', 'fscWithChimera'); +EMC_assert_boolean(emc.fsc_with_chimera); + +emc = EMC_assert_deprecated_substitution(emc, 0.1, 'minimum_particle_for_fsc_weighting', 'minimumparticleVolume'); +EMC_assert_numeric(emc.minimum_particle_for_fsc_weighting, 0, [0.01, 1.0]); + +emc = EMC_assert_deprecated_substitution(emc, 1.0, 'fsc_shape_mask', 'flgFscShapeMask'); +EMC_assert_numeric(emc.fsc_shape_mask, 1, [0.0, 2.0]); + +if isfield(emc, 'shape_mask_lowpass') + EMC_assert_numeric(emc.shape_mask_lowpass, 1, [10, 100]); +else + emc.shape_mask_lowpass = 14; +end + +if isfield(emc, 'shape_mask_threshold') + EMC_assert_numeric(emc.shape_mask_threshold, 1, [0.1, 10.0]); +else + emc.shape_mask_threshold = 2.4; +end + +if isfield(emc, 'shape_mask_test') + EMC_assert_boolean(emc.shape_mask_test); +else + emc.shape_mask_test = false; +end + +emc = EMC_assert_deprecated_substitution(emc, 22.0, 'pca_scale_spaces', 'pcaScaleSpace'); +EMC_assert_numeric(emc.pca_scale_spaces); +emc.('n_scale_spaces') = numel(emc.pca_scale_spaces); + +if isfield(emc, 'Pca_maxEigs') + EMC_assert_integer(emc.Pca_maxEigs, 1, [1, 1000]); +else + emc.Pca_maxEigs = 36; +end + +if isfield(emc, 'Pca_randSubset') + EMC_assert_integer(emc.Pca_randSubset, 1); +else + emc.Pca_randSubset = 0; +end + + + end diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index d87ff8f1..86d13183 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -43,7 +43,7 @@ error('PARAMETER_FILE, CYCLE') end -startTime = clock; +startTime = datetime("now"); CYCLE = EMC_str2double(CYCLE); cycleNumber = sprintf('cycle%0.3u', CYCLE); @@ -62,7 +62,7 @@ end -nRows = length(emc.('pcaScaleSpace')); +nRows = emc.n_scale_spaces; featureVector = cell(2,1); if flgGold featureVector{1,1} = emc.('Pca_coeffs_odd'); @@ -74,7 +74,7 @@ nFeatures = size(featureVector{1,1}); if (nFeatures(1) ~= nRows) - error('There should be a set of indices for each pcaScaleSpace, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') + error('There should be a set of indices for each pca_scale_spaces, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') end clusterVector= emc.('Pca_clusters'); @@ -462,8 +462,9 @@ save(emc.('subTomoMeta'), 'subTomoMeta'); %save(sprintf('%s_pca.mat',OUTPUT_PREFIX), 'nTOTAL','U', 'S', 'V', 'coeffs') - fprintf('Total execution time on set %s: %f seconds\n', halfSet,etime(clock, startTime)); + fprintf('Total execution time on set %s: %f seconds\n', halfSet, datetime("now") - startTime); delete(gcp('nocreate')); end % end of Gold loop + end % end of cluster function diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 76881be7..2dcd2550 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -38,58 +38,13 @@ cycleNumber = sprintf('cycle%0.3u', CYCLE); prevCycleNumber = sprintf('cycle%0.3u',CYCLE-1); - -flgCones = emc.('flgCones'); -emc.classification = emc.('emc.classification'); -try - scaleCalcSize = emc.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end - -try - flgFscShapeMask = emc.('flgFscShapeMask'); -catch - flgFscShapeMask = 1; -end - -try - shape_mask_lowpass = emc.('shape_mask_lowpass'); -catch - shape_mask_lowpass = 14; -end - -try - shape_mask_threshold = emc.('shape_mask_threshold'); -catch - shape_mask_threshold = 2.4; -end - -try - % Apply the mask with the given parameters, save and exit. - shape_mask_test = emc.('shape_mask_test'); -catch - shape_mask_test = false; -end - % Estimating the particle volume still occasionaly goes awry. Place a cap and return a cautionary message. - -try - minimumParticleVolume = emc.('minimumparticleVolume'); -catch - minimumParticleVolume = 0.1; -end - -try fscWithChimera = emc.('fscWithChimera'); -catch fscWithChimera = 0; -end - outputPrefix = sprintf('./FSC/%s_%s', cycleNumber, emc.('subTomoMeta')); samplingRate = emc.('Ali_samplingRate'); emc.pixel_size_si = emc.pixel_size_angstroms .* samplingRate; -if ( flgCones ) +if ( emc.flgCones ) coneInc = 30; nCones = 37; calcCones = 1; @@ -116,9 +71,6 @@ end end - - - % Note this is taken from the class section, not Fsc refName = emc.('Cls_className');% emc.('Ref_className'); @@ -130,10 +82,6 @@ bh_global_MTF = 2; end - - - - % The default is fsc-Gold Standard so the two images should need some degree of % alignment prior to calculating the fsc. flgAlignImages = 1; @@ -173,8 +121,6 @@ fieldPrefix = 'REF'; end - - nReferences = length(classVector(1,:)) imageName{1} = sprintf('class_%d_Locations_%s_ODD_NoWgt', className,fieldPrefix); @@ -230,15 +176,11 @@ refSym{iGold} = ones(length(refVector{iGold})); end - -% [ maskType, maskSize, maskRadius, maskCenter ] = ... -% BH_multi_maskCheck(emc, 'Ali', pixelSize,'FSC') - [ maskType, maskSize, maskRadius, maskCenter ] = ... BH_multi_maskCheck(emc, 'Ali', pixelSize) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) padDIM = max(max(sizeWindow),384); padREF = [0,0,0;0,0,0]; @@ -286,7 +228,6 @@ end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -332,17 +273,12 @@ for iGold = 1:2 for iRef = 1:nReferences - % iHalf = refVector{iGold}(iRef); if iGold == 1 - refRotAvg{iRef} = refIMG{iGold}{iRef};%BH_axialSymmetry(refIMG{iGold}{iRef}, 120,... - %0, 'GPU',[0,0,0]); + refRotAvg{iRef} = refIMG{iGold}{iRef}; end - - end end - bestAnglesTotal = zeros(nReferences,12); nCount = 1; @@ -352,19 +288,19 @@ fprintf('working on %d/ %d references FscGold\n', iRef, nReferences); - [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(refIMG{1}{iRef}), pixelSize, {'fsc', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); - [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(refIMG{2}{iRef}), pixelSize, {'fsc', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); + [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(refIMG{1}{iRef}), pixelSize, {'fsc', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); + [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(refIMG{2}{iRef}), pixelSize, {'fsc', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); - if (shape_mask_test) + if (emc.shape_mask_test) fprintf('\nSaving your masks and exiting!\n'); SAVE_IMG(shapeMask_1,sprintf('%s-shape_mask_%2.2f_lowpass_%2.2f_threshold.mrc', ... - outputPrefix, shape_mask_lowpass,shape_mask_threshold),pixelSize); + outputPrefix, emc.shape_mask_lowpass,emc.shape_mask_threshold),pixelSize); return; end - if (flgFscShapeMask) - shapeMask_1 = gather((shapeMask_1.*volMask{1}).^flgFscShapeMask); - shapeMask_2 = gather((shapeMask_2.*volMask{2}).^flgFscShapeMask); + if (emc.fsc_shape_mask) + shapeMask_1 = gather((shapeMask_1.*volMask{1}).^emc.fsc_shape_mask); + shapeMask_2 = gather((shapeMask_2.*volMask{2}).^emc.fsc_shape_mask); else shapeMask_1 = 1; shapeMask_2 = 1; @@ -384,10 +320,10 @@ SAVE_IMG(MRCImage(gather(refIMG{1}{iRef}.*pV1)), ... oddName, 1.0,1); - if (fscWithChimera) + if (emc.fsc_with_chimera) [whereIsChimera, ~] = system('which chimera'); if (whereIsChimera) - error('fscWithChimera is called, but "chimera" not in system PATH.') + error('fsc_with_chimera is called, but "chimera" not in system PATH.') end writeOutPyAli() system(sprintf('chimera --nogui --script "FSC/fitInMap.py %s %s %s-fitInMap.txt" ',... @@ -413,7 +349,6 @@ end - gpuDevice(1); if ( flgEstSNR ) @@ -457,7 +392,6 @@ if (flgAlignImages) && ~(flgJustFSC) - img2 = refIMG{2}{iRef}; if (flgEstSNR) @@ -487,7 +421,6 @@ end end - halfSet = 'GLD'; fprintf('resampling ref %d.\n', iRef); img1 = BH_padZeros3d(img1,[0,0,0],[0,0,0],'cpu','singleTaper'); @@ -496,41 +429,33 @@ img1=IMG1; img2=IMG2; - [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(img1), pixelSize, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(img2), pixelSize, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); - if (shape_mask_test) + if (emc.shape_mask_test) fprintf('\nSaving your masks and exiting!\n'); SAVE_IMG(shapeMask_1,sprintf('%s-shape_mask_%2.2f_lowpass_%2.2f_threshold.mrc', ... - outputPrefix, shape_mask_lowpass,shape_mask_threshold),pixelSize); + outputPrefix, emc.shape_mask_lowpass,emc.shape_mask_threshold),pixelSize); return; end - - if (flgFscShapeMask) - shapeMask_1 = gather((shapeMask_1.*volMask{1}).^flgFscShapeMask); - shapeMask_2 = gather((shapeMask_2.*volMask{2}).^flgFscShapeMask); + if (emc.fsc_shape_mask) + shapeMask_1 = gather((shapeMask_1.*volMask{1}).^emc.fsc_shape_mask); + shapeMask_2 = gather((shapeMask_2.*volMask{2}).^emc.fsc_shape_mask); else shapeMask_1 = 1; shapeMask_2 = 1; end - particleVolume(iRef) = gather(mean([particleFraction1,particleFraction2])); pV1 = gather(pV1.*volMask{1}); pV2 = gather(pV2.*volMask{2}); - - halfSet = 'OUT'; else halfSet = 'STD'; end - - - imgFilt1 = BH_bandLimitCenterNormalize(img1.*volMask{1}.*shapeMask_1,bandpassFilt{iGold}, ... (volMask{1} > 0.01),padCalc, 'single'); imgFilt1 = real(ifftn(imgFilt1)); @@ -563,7 +488,7 @@ {'none'}, 1, 0, 1 ); rad = single(rad)./pixelSize; - if (flgFscShapeMask) + if (emc.fsc_shape_mask) %!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! % TODO add a flag since the phase randomized is not used in practice @@ -614,9 +539,6 @@ end - - - clear randLowRES randHighRES fou1 = fftn(fou1.*BH_padZeros3d(pV1, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); fou2 = fftn(fou2.*BH_padZeros3d(pV2 , fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); @@ -649,7 +571,7 @@ [shellsFreq, shellsFSC, shellsNUM,shellsPOWER] = ... calc_shells(fou1, fou2, rad, pixelSize,coneList, halfAngle); clear fou1 fou2 - if (flgFscShapeMask) + if (emc.fsc_shape_mask) fou1 = fftn(img1.*BH_padZeros3d(pV1, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); clear pv1 fou2 = fftn(img2.*BH_padZeros3d(pV2, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); @@ -690,7 +612,7 @@ end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% test save masking curve - if (flgFscShapeMask) + if (emc.fsc_shape_mask) transitionFreq = find(osX > fscTcutoff,1,'first'); @@ -705,7 +627,7 @@ SAVE_IMG(MRCImage(single(gather(shapeMask_2))), ... sprintf('%s-%d-shapeMask_%d.mrc', outputPrefix, iRef, 2)); - if (minimumParticleVolume < 1) + if (emc.minimum_particle_for_fsc_weighting < 1) % Only save if used. SAVE_IMG(MRCImage(single(gather(pV1))), ... sprintf('%s-%d-particleVolEst_%d.mrc', outputPrefix, iRef, 1)); @@ -794,7 +716,7 @@ lowestRes = 0; highestRes = 0; - if (flgCones) + if (emc.flgCones) lowestRes = 1./osX(lowCut1); highestRes = 1./osX(lowCut1); for iCone = 1:nCones @@ -847,7 +769,7 @@ cRefCurve = sqrt( abs(2.*fnval(fitFSC{1},osX)./(1+fnval(fitFSC{1},osX))) ) ... .* forceMaskAlign{1} .* whiteningFilter; cRefAli{1} = fit(osX,cRefCurve./max(cRefCurve(:)),'cubicSpline'); - if (flgCones) + if (emc.flgCones) for iCone = 1:nCones cRefCurve = sqrt( abs(2.*fnval(fitFSC{iCone+1},osX) ./ ... @@ -888,7 +810,7 @@ masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRef)) = ... {maskType, sizeMask, ... maskRadius, maskCenter, ... - flgFscShapeMask, shape_mask_lowpass, shape_mask_threshold}; + emc.fsc_shape_mask, emc.shape_mask_lowpass, emc.shape_mask_threshold}; masterTM.('currentResForDefocusError') = osX(oneBitCut).^-1; end @@ -929,7 +851,7 @@ fclose(fscOUT); - if (flgCones) + if (emc.flgCones) figure('Visible','off'), plot(osX,fnval(fitFSC{1},osX),'kd','MarkerSize',2.5); hold on; plot(osX, oneBIT,'c'); @@ -940,7 +862,6 @@ plot(osX,fnval(fitFSC{iCone},osX),'k--'); end - % title({'FSC',sprintf('0.5 %3.2f\n0.143 %3.2f',1./fmid,1./fgold)}); title({'FSC',sprintf('0.5 - %3.2f\n0.143 - %3.2f (%3.2f-%3.2f)\noneBit %3.2f\n halfBit %3.2f\n',1./fmid,1./fgold,lowestRes,highestRes,osX(oneBitCut(1)).^-1,osX(halfBitCut(1)).^-1)}); xlabel('Spatial Freq'); ylabel('fsc'); @@ -980,13 +901,9 @@ xlabel('Spatial Freq'); ylabel('fsc'); ylim([-.05 1.025]) file_out = sprintf('%s-%d-fscFull_%s', outputPrefix, iRef, halfSet); - % saveas(gcf, file_out,'pdf') - % fscRandOUT = fopen(sprintf('%s-%d-fscFull_%s.txt', outputPrefix, iRef, halfSet),'w'); - % fprintf(fscRandOUT,'%4.4f\t%4.4f\t%4.4f\t%4.4f\n',[osX,fnval(fitTightFSC,osX),fnval(fscTrue,osX),fnval(fitFSC{1},osX)]'); - % fclose(fscRandOUT); figure('Visible','off'), plot(osX,cRef{1}(osX),'kd','MarkerSize',3); hold on; - if (flgCones) + if (emc.flgCones) plot(osX,cRef{2}(osX),'k--'); for iCone = 3:length(cRef) @@ -1003,7 +920,7 @@ file_out = sprintf('%s-%d-cRef_%s', outputPrefix, iRef, halfSet); saveas(gcf, file_out,'pdf') figure('Visible','off'), plot(osX,cRefAli{1}(osX),'kd','MarkerSize',3); hold on; - if (flgCones) + if (emc.flgCones) plot(osX,cRefAli{2}(osX),'k--'); for iCone = 3:length(cRefAli) @@ -1018,75 +935,7 @@ file_out = sprintf('%s-%d-cRefAli_%s', outputPrefix, iRef, halfSet); saveas(gcf, file_out,'pdf') - % % try - % fitPower = fit(shellsFreq(:,1).^2,log(shellsPOWER(:,1)),'cubicSpline'); - % LR = 10; - % MR = 7; - % HR = min((1.05*fgold).^2, shellsFreq(end-1,1).^2); - % if isempty(HR) - % HR = lowCut1; - % end - % lowRes = find(shellsFreq(:,1).^2 > (1/LR)^2, 1,'first'); - % midRes = find(shellsFreq(:,1).^2 > (1/MR)^2, 1,'first'); - % endRes = find(shellsFreq(:,1).^2 > HR, 1,'first'); - % plot1 = false; - % plot2 = false; - % if (midRes - lowRes > 2) - % bFactorFIT1 = fit(shellsFreq(lowRes:midRes,1).^2 , ... - % log(shellsPOWER(lowRes:midRes,1)),'poly1'); - % plot1= true; - % end - % if (endRes-midRes > 2) - % bFactorFIT2 = fit(shellsFreq(midRes:endRes,1).^2 , ... - % log(shellsPOWER(midRes:endRes,1)),'poly1'); - % plot2 = true; - % end - - % figure('Visible','off'), plot(osX.^2,fitPower(osX.^2),'k'); hold on; - % if (plot1) - % plot(osX.^2,bFactorFIT1(osX.^2),'b--'); - % else - % bFactorFIT1 = struct() - % bFactorFIT1.('p1') = 0; - % end - % if (plot2) - % plot(osX.^2,bFactorFIT2(osX.^2),'b--'); - % else - % bFactorFIT2 = struct() - % bFactorFIT2.('p1') = 0; - % end - % line([(1/LR)^2,(1/LR)^2], ... - % [min(log(shellsPOWER(:,1))), ... - % max(log(shellsPOWER(:,1)))], ... - % 'Color','k','LineStyle','--'); - % line([(1/MR)^2,(1/MR)^2], ... - % [min(log(shellsPOWER(:,1))), ... - % max(log(shellsPOWER(:,1)))], ... - % 'Color','k','LineStyle','--'); - % line([HR,HR], ... - % [min(log(shellsPOWER(:,1))), ... - % max(log(shellsPOWER(:,1)))], ... - % 'Color','k','LineStyle','--'); - % % plot(osX.^2,bFactorFIT2(osX.^2),'b--'); - % % outCurve(:,1).^2,outCurve(:,8),'g'); - - % title({'Guinier Plot',sprintf('\nbFactor(%2.1f-%2.1f-%2.1f)\n %d,%d', ... - % LR,MR,sqrt(1./HR),round(bFactorFIT1.p1*-4),... - % round(bFactorFIT2.p1*-4))}); ... - % xlabel('1/Ang^2'),... - % ylabel('log(F)'); - % ylim([0.95*min(log(shellsPOWER(:,1))),... - % 1.05*max(log(shellsPOWER(:,1)))]) - - % legend('uncorrected','corrected','Location','northeast',... - % 'Orientation', 'vertical'); - % file_out = sprintf('%s-%d-guinier_%s', outputPrefix, iRef, halfSet); - % savefig(gcf,file_out); - % saveas(gcf, file_out,'pdf') - % catch - % fprintf('\nRan into some error in the guinier analysis.\n'); - % fprintf('\nSince this is not critical, skipping and continue.\n'); - % end + end subTomoMeta = masterTM; diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index f2a2df91..c0778a53 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -10,7 +10,7 @@ % samplingRate = Binning factor, assumed to be integer value. Image is first % smoothed by an appropriate low-pass filter to reduce aliasing. % -% randomSubset = -1, count all non-ignored particles (class -9999). +% emc.Pca_randSubset = -1, count all non-ignored particles (class -9999). % % float, randomly select this many paparticleBandpassrticles for the % decomposition, denote by updating the flag in column 8 to be 1 @@ -18,7 +18,7 @@ % % string - indicates a mat file with prior decomposition. % -% maxEigs = Maximum number of principle components to save, general 50 +% emc.Pca_maxEigs = Maximum number of principle components to save, general 50 % has been plenty. This is a big memory saver. % % bandpass = [HIGH_THRESH, HIGH_CUT, LOW_CUT, PIXEL_SIZE] @@ -92,7 +92,7 @@ test_multi_ref_diffmap = true; test_scale_space_bug_fix = false; -startTime = clock; +startTime = datetime("now"); CYCLE = EMC_str2double(CYCLE); PREVIOUS_PCA = EMC_str2double(PREVIOUS_PCA); @@ -145,12 +145,10 @@ %%% Put this in the param file later - the input values should be in angstrom %%% and are the relevant scale spaces for classification. -pcaScaleSpace = emc.('pcaScaleSpace'); -nScaleSpace = numel(pcaScaleSpace); + samplingRate = emc.('Cls_samplingRate'); refSamplingRate= emc.('Ali_samplingRate'); -randomSubset = emc.('Pca_randSubset'); -maxEigs = emc.('Pca_maxEigs'); + pixelSize = emc.pixel_size_angstroms .* samplingRate; refPixelSize = emc.pixel_size_angstroms .* refSamplingRate; @@ -163,11 +161,7 @@ pInfo = parcluster(); nTempParticles = emc.('PcaGpuPull'); -try - scaleCalcSize = emc.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end + outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); %%%flgGold = emc.('flgGoldStandard'); @@ -183,17 +177,9 @@ % The defaults used in fscGold are modified here to make a more permissive % mask since we are concerned with densities that are likely damped during % averaging due to low occupancy. -try - shape_mask_lowpass = emc.('shape_mask_lowpass'); -catch - shape_mask_lowpass = 14 + 10; -end +emc.shape_mask_lowpass = emc.shape_mask_lowpass + 10; +emc.shape_mask_threshold = emc.shape_mask_threshold - 0.4; -try - shape_mask_threshold = emc.('shape_mask_threshold'); -catch - shape_mask_threshold = 2.4 - 0.4; -end try tmpVal = emc.('whitenPS'); @@ -206,28 +192,13 @@ wiener_constant = 0.0; end -try - % Apply the mask with the given parameters, save and exit. - shape_mask_test = emc.('shape_mask_test'); -catch - shape_mask_test = false; -end - -try - test_updated_bandpass = emc.('test_updated_bandpass'); -catch - test_updated_bandpass = false; -end % Removed flgGold everywhere else, but keep ability to classify full data set at % the end (after all alignment is finished.) -%%% For general release, I've disabled class average alignment and -%%% multi-reference alignment, so set the default to OFF. If either of -%%% these features are re-introduced, this will need to be reverted. -if ( emc.classification ); emc.classification = -1 ; end -if emc.classification < 0 + +if (emc.classification) flgGold = 0; else flgGold = 1; @@ -286,11 +257,11 @@ cpuVols = struct; [ preSizeWindow, preSizeCalc, preSizeMask, prePadWindow, prePadCalc ] = ... - BH_multi_validArea(preMaskSize,preMaskRadius, scaleCalcSize ) + BH_multi_validArea(preMaskSize,preMaskRadius, emc.scale_calc_size ) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) if (test_multi_ref_diffmap) @@ -347,9 +318,9 @@ averageMotif = cell(2,1); if (test_multi_ref_diffmap) - fprintf('nScaleSpace = %d\n',nScaleSpace); + fprintf('emc.n_scale_spaces = %d\n',emc.n_scale_spaces); fprintf('nReferences = %d\n',nReferences); - nScaleSpace = nReferences(1); + emc.n_scale_spaces = nReferences(1); pause(3); else nReferences = [1,1]; @@ -446,7 +417,7 @@ % In most cases, this is the number of "features" specified in the % parameter file, but in some data not even this may non-zero singluar % values are found, so the number could be different (lower) - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces eigsFound = size(coeffs{iScale},1); fname = sprintf('%s_varianceMap%d-%s-%d.mrc', ... outputPrefix, eigsFound, halfSet, iScale); @@ -458,26 +429,14 @@ end end -try - symmetry = emc.('symmetry'); - fprintf('\n\tWarning: As of emClarity 1.7.0.12 the symmetry parameter is applied to the volume and mask in PCA!\n') -catch - error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); -end - -try - constrain_symmetry = emc.('Pca_constrain_symmetry'); -catch - constrain_symmetry = false; -end if (PREVIOUS_PCA) volumeMask = gpuArray(getVolume(MRCImage( ... sprintf('%s_pcaVolMask.mrc',outputPrefix)))); else - if (constrain_symmetry) - gridSearch = eulerSearch(symmetry,180,5,360,5,0.0,1,true); + if (emc.Pca_constrain_symmetry) + gridSearch = eulerSearch(emc.symmetry,180,5,360,5,0.0,1,true); [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter, ... '3d', gridSearch.number_of_asymmetric_units); else @@ -493,7 +452,7 @@ % include when sets are left 100% separate. % volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, ... - {'pca', true; 'lowpass', shape_mask_lowpass; 'threshold', shape_mask_threshold}); + {'pca', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', shape_mask_threshold}); end @@ -509,8 +468,8 @@ end volMask = struct(); -nPixels = zeros(2,nScaleSpace); -for iScale = 1:nScaleSpace +nPixels = zeros(2,emc.n_scale_spaces); +for iScale = 1:emc.n_scale_spaces for iGold = 1:1+flgGold stHALF = sprintf('h%d',iGold); stSCALE = sprintf('s%d',iScale); @@ -539,9 +498,9 @@ % radius, convert Ang to pix , denom = equiv stdv from normal to include, e.g. % for 95% use 1/sig = 1/2 -%stdDev = 1/2 .* (pcaScaleSpace ./ pixelSize - 1) .* 3.0./log(pcaScaleSpace) -threeSigma = 1/3 .* (pcaScaleSpace ./ pixelSize) -for iScale = 1:nScaleSpace +%stdDev = 1/2 .* (emc.pca_scale_spaces ./ pixelSize - 1) .* 3.0./log(emc.pca_scale_spaces) +threeSigma = 1/3 .* (emc.pca_scale_spaces ./ pixelSize) +for iScale = 1:emc.n_scale_spaces kernelSize = ceil(threeSigma(iScale)) + 3; kernelSize = kernelSize + (1-mod(kernelSize,2)); @@ -551,11 +510,11 @@ end -avgMotif_FT = cell(1+flgGold,nScaleSpace); -avgFiltered = cell(1+flgGold,nScaleSpace); +avgMotif_FT = cell(1+flgGold,emc.n_scale_spaces); +avgFiltered = cell(1+flgGold,emc.n_scale_spaces); % Here always read in both, combine if flgGold = 0 for iGold = 1:1+flgGold - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces if (test_multi_ref_diffmap) tmp_avg = averageMotif{iGold}{iScale}; @@ -600,7 +559,7 @@ -% If randomSubset is string with a previous matfile use this, without any +% If emc.Pca_randSubset is string with a previous matfile use this, without any % decomposition. for iGold = 1:1+flgGold @@ -623,15 +582,15 @@ if (PREVIOUS_PCA) previousPCA = sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet); - randomSubset = -1; - [ geometry, nTOTAL, nSUBSET ] = BH_randomSubset( geometry,'pca', -1 , randSet); + emc.Pca_randSubset = -1; + [ geometry, nTOTAL, nSUBSET ] = BH_emc.Pca_randSubset( geometry,'pca', -1 , randSet); else - if (randomSubset) + if (emc.Pca_randSubset) previousPCA = false; - [ geometry, nTOTAL, nSUBSET ] = BH_randomSubset( geometry,'pca', randomSubset, randSet ); + [ geometry, nTOTAL, nSUBSET ] = BH_emc.Pca_randSubset( geometry,'pca', emc.Pca_randSubset, randSet ); else previousPCA = false; - [ geometry, nTOTAL, nSUBSET ] = BH_randomSubset( geometry,'pca', -1 , randSet); + [ geometry, nTOTAL, nSUBSET ] = BH_emc.Pca_randSubset( geometry,'pca', -1 , randSet); end end @@ -650,7 +609,7 @@ clear dataMatrix tempDataMatrix dataMatrix = cell(3,1); tempDataMatrix = cell(3,1); - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces dataMatrix{iScale} = zeros(nPixels(iGold,iScale), nSUBSET, 'single'); tempDataMatrix{iScale} = zeros(nPixels(iGold,iScale), nTempParticles, 'single', 'gpuArray'); end @@ -659,7 +618,7 @@ % the device is reset at the end of each loop.) gpuMasks = struct(); - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces stSCALE = sprintf('s%d',iScale); gpuMasks.('volMask').(stSCALE) = ... @@ -675,7 +634,7 @@ end % % % for iGold_inner = 1:1+flgGold - % % % for iScale = 1:nScaleSpace + % % % for iScale = 1:emc.n_scale_spaces % % % avgMotif_FT{iGold_inner, iScale} = ... % % % gpuArray(cpuVols.('avgMotif_FT').(sprintf('g%d_%d',iGold_inner,iScale))); % % % end @@ -851,9 +810,9 @@ % being used many times, over the angle loop. It may be more efficient to do this outside the for subtomo loop here, but % to start, just do it the same way. use_only_once = true; - [ ~, iParticle ] = interpolator(gpuArray(iParticle),angles, shiftVAL, 'Bah', 'inv', symmetry, use_only_once); + [ ~, iParticle ] = interpolator(gpuArray(iParticle),angles, shiftVAL, 'Bah', 'inv', emc.symmetry, use_only_once); - [ ~, iWedge ] = interpolator(gpuArray(wedgeMask),angles,[0,0,0], 'Bah', 'inv', symmetry, use_only_once); + [ ~, iWedge ] = interpolator(gpuArray(wedgeMask),angles,[0,0,0], 'Bah', 'inv', emc.symmetry, use_only_once); else % Transform the particle, and then trim to motif size @@ -874,7 +833,7 @@ - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces iPrt = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); @@ -915,7 +874,7 @@ % pull data of the gpu every 1000 particls (adjust this to max mem) if nTemp == nTempParticles - 1 - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces dataMatrix{iScale}(:,1+nTempPrev:nTemp+nTempPrev-1) = ... gather(tempDataMatrix{iScale}(:,1:nTemp-1)); end @@ -954,7 +913,7 @@ end % end of the loop over Tomograms, % % % volBinaryMask = reshape(gather(volBinaryMask),sizeMask); - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces masks.('binary').(stHALF).(sprintf('s%d',iScale)) = ... reshape(masks.('binary').(stHALF).(sprintf('s%d',iScale)),sizeMask); end @@ -965,7 +924,7 @@ subTomoMeta = masterTM; save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... gather(tempDataMatrix{iScale}(:,1:nTemp-1)); end @@ -978,7 +937,7 @@ idxList = idxList(cleanIDX); peakList = peakList(cleanIDX); - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces dataMatrix{iScale} = dataMatrix{iScale}(:,1:size(idxList,2)); % Center the rows for row = 1:size(dataMatrix{iScale},1) @@ -1002,10 +961,10 @@ oldPca = load(previousPCA); U = oldPca.U; clear oldPca; - sDiag = cell(nScaleSpace,1); - coeffs = cell(nScaleSpace,1); + sDiag = cell(emc.n_scale_spaces,1); + coeffs = cell(emc.n_scale_spaces,1); - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces % Sanity checks on the dimensionality numEigs = size(U{iScale}, 2); if nPixels(iGold,iScale) ~= size(U{iScale}, 1) @@ -1016,20 +975,20 @@ end else - U = cell(nScaleSpace,1); - V = cell(nScaleSpace,1); - S = cell(nScaleSpace,1); - sDiag = cell(nScaleSpace,1); - coeffs = cell(nScaleSpace,1); - varianceMap = cell(nScaleSpace,1); - for iScale = 1:nScaleSpace + U = cell(emc.n_scale_spaces,1); + V = cell(emc.n_scale_spaces,1); + S = cell(emc.n_scale_spaces,1); + sDiag = cell(emc.n_scale_spaces,1); + coeffs = cell(emc.n_scale_spaces,1); + varianceMap = cell(emc.n_scale_spaces,1); + for iScale = 1:emc.n_scale_spaces krylovScalar = 5; % Calculate the decomposition [ U{iScale},S{iScale},V{iScale}, convergenceFlag ] = svds(double(dataMatrix{iScale}), ... - maxEigs, 'largest', ... + emc.Pca_maxEigs, 'largest', ... 'MaxIterations',1000, ... % default 300 - 'SubspaceDimension',max(krylovScalar*maxEigs,30),... % default max(3*maxEigs,15) + 'SubspaceDimension',max(krylovScalar*emc.Pca_maxEigs,30),... % default max(3*emc.Pca_maxEigs,15) 'Display',true); % Diagnostics default false (will this work in compiled?) U{iScale} = single(U{iScale}); S{iScale} = single(S{iScale}); @@ -1040,7 +999,7 @@ sDiag{iScale} = diag(S{iScale}); numNonZero = find(( sDiag{iScale} ~= 0 ), 1, 'last'); % For Method 1, save eigenvectors 1-4 (or user-specified max) as images - eigsFound = min(maxEigs, numNonZero); + eigsFound = min(emc.Pca_maxEigs, numNonZero); fprintf('Found %d / %d non-zero eigenvalues sum = %4.4f, in set %s.\n All singular values converged is t/f ( %d ) ', ... numNonZero, size(S{iScale}, 1), sum(sDiag{iScale}), halfSet, convergenceFlag); @@ -1052,7 +1011,7 @@ fprintf('Size S, %d %d Size U %d %d \n', size(S{iScale},1),size(S{iScale},2), size(U{iScale},1),size(U{iScale},2)); % We want the diagnol of US^2U'/ n-1 - % This will be maxEigs * Nvoxels matrix (U is Nvoxels * maxEigs) + % This will be emc.Pca_maxEigs * Nvoxels matrix (U is Nvoxels * emc.Pca_maxEigs) rightSide = S{iScale}(1:numNonZero,1:numNonZero).^2*U{iScale}'; varianceMap = zeros(nPixels(iGold,iScale),1); for k = 1:nPixels(iGold,iScale) @@ -1094,17 +1053,17 @@ % If requested, limit the number of principal components and coeffs saved - if maxEigs < size(S{iScale}, 1) + if emc.Pca_maxEigs < size(S{iScale}, 1) fprintf('Saving only the first %d principal components.\n', ... - maxEigs); + emc.Pca_maxEigs); if ~isempty(U{iScale}) % U will not exist for pcaMethods 2 or 3 - U{iScale} = U{iScale}(:, 1:maxEigs); + U{iScale} = U{iScale}(:, 1:emc.Pca_maxEigs); end if ~(previousPCA) - S{iScale} = S{iScale}(1:maxEigs, 1:maxEigs); - V{iScale} = V{iScale}(:, 1:maxEigs); + S{iScale} = S{iScale}(1:emc.Pca_maxEigs, 1:emc.Pca_maxEigs); + V{iScale} = V{iScale}(:, 1:emc.Pca_maxEigs); end - coeffs{iScale} = coeffs{iScale}(1:maxEigs, :); % + coeffs{iScale} = coeffs{iScale}(1:emc.Pca_maxEigs, :); % end end end @@ -1113,27 +1072,27 @@ % Only U is needed for further analysis, so save only this, unless % troubleshooting. if (previousPCA) - for iScale = 1:nScaleSpace + for iScale = 1:emc.n_scale_spaces % If requested, limit the number of principal components and coeffs saved. - if maxEigs < size(U{iScale}, 2) + if emc.Pca_maxEigs < size(U{iScale}, 2) fprintf('Saving only the first %d principal components.\n', ... p.pcaMaxNumComponents); - U{iScale} = U{iScale}(:, 1:maxEigs); - coeffs{iScale} = coeffs{iScale}(1:maxEigs, :); + U{iScale} = U{iScale}(:, 1:emc.Pca_maxEigs); + coeffs{iScale} = coeffs{iScale}(1:emc.Pca_maxEigs, :); end end save(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet), 'nTOTAL', 'coeffs','idxList','peakList', 'sDiag'); else - if (randomSubset) + if (emc.Pca_randSubset) save(sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet),'U', 'idxList','peakList'); else save(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet), 'nTOTAL', 'coeffs','idxList','peakList', 'sDiag'); end end - fprintf('Total execution time on %s set: %f seconds\n', halfSet, etime(clock, startTime)); + fprintf('Total execution time on %s set: %f seconds\n', halfSet, datetime("now") - startTime); close all force; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 3d116fd4..6a6a4ca3 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -355,11 +355,6 @@ refName = emc.('Raw_className'); -try - symmetry = emc.('symmetry'); -catch - error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); -end classVector{1} = emc.('Raw_classes_odd')(1,:); classSymmetry{1}= emc.('Raw_classes_odd')(2,:); diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index d468f6c7..9a877408 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -940,15 +940,14 @@ fprintf('(\t\tOr better yet, check with ben!\t\t)\n'); fprintf('\nIf you do use/change any of these, please mention in your methods and EMDB entry!\n'); fprintf('\n\n----------------------------------\n\n'); -fprintf('\nscaleCalcSize\toversampling of vol for xcorr. Def:\t1.5\n'); +fprintf('\nscale_calc_size\toversampling of vol for xcorr. Def:\t1.5\n'); fprintf('\npaddedSize\tpadded size of tiles in ctf estimateion\n'); fprintf('\nflgFscShapeMask\t default 1\n'); fprintf('\nflgPcaShapeMask\t default 1\n'); fprintf('\nflgQualityWeight\t Downweight high-freq of low scoring sub-tomos. Def:\t4\n'); -fprintf('\ninterpOrder\t Linear interpolation (1) Spline/Fourier (4 - not working, do not use)\n'); -fprintf('\nflgLimitToOneProcess\t For OOM issues in averaging. Boolean Def:\t0\n'); +fprintf('\nlimit_to_one_core\t For OOM issues in averaging. Boolean Def:\t0\n'); fprintf('\nflgCenterRefCOM\tShift reference to center of mass. Boolean Def:\t1\n'); fprintf('\nconserveDiskSpace\n'); fprintf('\nPca_distMeasure\tMeasure for difference. euclidean, cityblock, correlation, cosine Def:\t sqeuclidean\n'); diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 8f9b5364..53b5309b 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -2,15 +2,25 @@ function [ ] = BH_average3d(PARAMETER_FILE, CYCLE, STAGEofALIGNMENT) -if (nargin ~= 3) - error('args = PARAMETER_FILE, CYCLE, STAGEofALIGNMENT') -end % FIXME: hacking in a test test_fuzz=false; test_multi_ref_diffmap= true; +if (nargin ~= 3) + error('args = PARAMETER_FILE, CYCLE, STAGEofALIGNMENT') +end + +global bh_global_ML_compressByFactor; +global bh_global_ML_angleTolerance; +if isempty(bh_global_ML_compressByFactor) + bh_global_ML_compressByFactor = 2.0; +end +if isempty(bh_global_ML_angleTolerance) + bh_global_ML_angleTolerance = 5; +end + startTime = datetime("now"); CYCLE = EMC_str2double(CYCLE); @@ -72,38 +82,22 @@ %%% multi-reference alignment, so set the default to OFF. If either of %%% these features are re-introduced, this will need to be reverted. % FIXME: get rid of the -1 -if ( emc.classification ); emc.classification = -1 ; end flgGold=1; % Optionally specify gpu idxs -if numel(emc.nGPUs) == 1 +if (numel(emc.nGPUs) == 1) gpuList = 1:emc.nGPUs; else gpuList = emc.nGPUs; emc.nGPUs = length(gpuList); end -try - scaleCalcSize = emc.('scaleCalcSize'); -catch - scaleCalcSize = 1.5; -end - - -global bh_global_ML_compressByFactor; -global bh_global_ML_angleTolerance; -if isempty(bh_global_ML_compressByFactor) - bh_global_ML_compressByFactor = 2.0; -end -if isempty(bh_global_ML_angleTolerance) - bh_global_ML_angleTolerance = 5; -end - if (emc.nPeaks > 1) fprintf('For ML approach:\nUsing a compression factor %3.3f\nUsing an angulare tolerance of %3.3f degrees\n', ... bh_global_ML_compressByFactor, bh_global_ML_angleTolerance); end + % for now only turn on (optionally) in reference generation. classVector = cell(2,1); @@ -118,17 +112,16 @@ peakCOM = []; peakSearch = []; saveClassSum = -1; + +% The prefix of the variable read in from the parameter file. +parameterPrefix = ''; +% The prefix used in file names and subTomoMeta fields +savePrefix = ''; switch STAGEofALIGNMENT case 'RawAlignment' - - - if (emc.classification) - fieldPrefix = 'Raw' - - else - fieldPrefix = 'REF' - end - + + fieldPrefix = 'Raw'; + savePrefix = 'REF'; classVector{1} = emc.(sprintf('%s_classes_odd','Raw')); classVector{2} = emc.(sprintf('%s_classes_eve','Raw')); @@ -149,24 +142,17 @@ end - case 'FinalAlignment' % Special case for the final cycle. % Assuming RawAlignment already run for this cycle and FSC is calculated % Goal is to re-extract odd-half, applying the xform found in fscGold - fieldPrefix = 'Raw' - + fieldPrefix = 'Raw'; + savePrefix = 'REF'; classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); className = emc.(sprintf('%s_className',fieldPrefix)); samplingRate = emc.('Ali_samplingRate'); - if (emc.classification) - %samplingRate = emc.('Pca_samplingRate'); - else - %samplingRate = emc.('Raw_samplingRate'); - fieldPrefix = 'REF' - end flgFinalAvg = 1; % Update at some point to handle multiple classes, for now just test on the @@ -195,6 +181,7 @@ STAGEofALIGNMENT = 'Cluster'; ClusterGeomNAME = 'ClusterClsGeom'; fieldPrefix = 'Cls'; + savePrefix = 'Cls'; classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); @@ -204,17 +191,15 @@ samplingRate = emc.(sprintf('Cls_samplingRate')); className = emc.(sprintf('%s_className',fieldPrefix)); - if emc.classification < 0 + if (emc.classification) flgGold = 0; end case 'SnrEstimate' - if CYCLE - fieldPrefix = 'Raw'; - else - fieldPrefix = 'NoA'; - end + fieldPrefix = 'Raw'; + savePrefix = 'REF'; + classVector{1} = [1:25;ones(1,25)]; classVector{2} = [1:25;ones(1,25)]; @@ -222,46 +207,10 @@ samplingRate = emc.(sprintf('%s_samplingRate','Ali')); otherwise - error('STAGEofALIGNMENT incorrect') + error('STAGEofALIGNMENT incorrect'); end -fprintf('StOAlign = %s, fieldPrefix = %s\n', STAGEofALIGNMENT, fieldPrefix); - -flgCones = emc.('flgCones'); - - -cutPrecision = 'single'; %emc.('flgPrecision'); -try - interpOrder = emc.('interpOrder'); -catch - interpOrder = 1; -end - -try - flgLimitToOneProcess = emc.('flgLimitToOneProcess'); -catch - flgLimitToOneProcess = 0; -end - -if (interpOrder == 4) - limitToOne = 1; -elseif (flgLimitToOneProcess) - limitToOne = flgLimitToOneProcess; -else - limitToOne = emc.('nCpuCores'); - interpOrder = 1; -end - - - - - -fprintf('Interporder %d, limitToOneProcess %d\n',interpOrder,limitToOne); - -if ~(ismember(interpOrder,[1,4])) - error('interpolationOrder must be 1,,4 - linear,sinc'); -end outputPrefix = sprintf('%s_%s',cycleNumber, emc.('subTomoMeta')); emc.pixel_size_angstroms = emc.pixel_size_angstroms .* samplingRate; @@ -328,22 +277,18 @@ cN = cell(2,1); if (flgGold) - cN{1} = sprintf('%s_%d_%d_nClass_%d_ODD',outputPrefix,classCoeffs{1}(1,1), ... - classCoeffs{1}(1,end), className) + cN{1} = sprintf('%s_%d_%d_nClass_%d_ODD',outputPrefix,classCoeffs{1}(1,1), classCoeffs{1}(1,end), className); geometry{1} = subTomoMeta.(cycleRead).ClusterResults.(cN{1}); - cN{2} = sprintf('%s_%d_%d_nClass_%d_EVE',outputPrefix,classCoeffs{2}(1,1), ... - classCoeffs{2}(1,end), className) + cN{2} = sprintf('%s_%d_%d_nClass_%d_EVE',outputPrefix,classCoeffs{2}(1,1), classCoeffs{2}(1,end), className); geometry{2} = subTomoMeta.(cycleRead).ClusterResults.(cN{2}); geometry = BH_mergeClassGeometry(geometry{1}, geometry{2}); else if (test_fuzz) - cN{1} = sprintf('%s_%d_%d_nClass_%d_STD','cycle002_full_2',classCoeffs{1}(1,1), ... - classCoeffs{1}(1,end), className) + cN{1} = sprintf('%s_%d_%d_nClass_%d_STD','cycle002_full_2',classCoeffs{1}(1,1), classCoeffs{1}(1,end), className); else - cN{1} = sprintf('%s_%d_%d_nClass_%d_STD',outputPrefix,classCoeffs{1}(1,1), ... - classCoeffs{1}(1,end), className) + cN{1} = sprintf('%s_%d_%d_nClass_%d_STD',outputPrefix,classCoeffs{1}(1,1), classCoeffs{1}(1,end), className); end geometry = subTomoMeta.(cycleRead).ClusterResults.(cN{1}); @@ -370,26 +315,21 @@ end -masterTM = subTomoMeta; clear subTomoMeta -try - symmetry = emc.('symmetry'); -catch - error('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)'); -end + class_idx=''; class_weights=''; if (test_fuzz) % FIXME: testing fuzzy classification, this is hardcoded - class_idx = masterTM.(cycleRead).ClusterResults.cycle002_full_2_0_64_nClass_36_STD_idxList; - class_weights = masterTM.(cycleRead).ClusterResults.cycle002_full_2_0_64_nClass_36_STD_p; + class_idx = subTomoMeta.(cycleRead).ClusterResults.cycle002_full_2_0_64_nClass_36_STD_idxList; + class_weights = subTomoMeta.(cycleRead).ClusterResults.cycle002_full_2_0_64_nClass_36_STD_p; end -if isfield(masterTM,('tomoCPR_run_in_cycle')) +if isfield(subTomoMeta,('tomoCPR_run_in_cycle')) - if (emc.eucentric_fit && ~isfield(masterTM.(sprintf('%s',cycleRead)), 'eucentric_shifts')) - cycle_to_update = masterTM.('tomoCPR_run_in_cycle')(find(masterTM.('tomoCPR_run_in_cycle')(:,1) == masterTM.currentTomoCPR),2); + if (emc.eucentric_fit && ~isfield(subTomoMeta.(sprintf('%s',cycleRead)), 'eucentric_shifts')) + cycle_to_update = subTomoMeta.('tomoCPR_run_in_cycle')(find(subTomoMeta.('tomoCPR_run_in_cycle')(:,1) == subTomoMeta.currentTomoCPR),2); if (cycle_to_update == cycleRead) error('You specified eucentric_fit=1, and you are averaging cycle %d and no shifts are found from cycle %d\n',cycleNumber,cycleRead); else @@ -425,27 +365,14 @@ end [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc] = ... - BH_multi_validArea( maskSize, maskRadius, scaleCalcSize ) -padREF = [0,0,0;0,0,0]; - -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms,emc.nGPUs, sizeCalc(1),limitToOne); - -origMaskSize = sizeMask; -%%%%% Considering removing doNotTrim and making this the default. Temporarily -%%%%% override here. -% % % %doNotTrim = true; -% This should be moved into BH_multi_validAra -% if (doNotTrim) -% + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) + +[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, emc.nGPUs, sizeCalc(1), emc.nCpuCores); + if any( (sizeCalc - sizeWindow) < 0 ) sizeCalc = BH_multi_iterator( sizeWindow, 'fourier' ) end sizeMask = sizeWindow; -% sizeCALC = sizeWindow; -% Find the next largest size for fft -%[ sizeCalc ] = BH_multi_iterator( sizeCalc, 'fourier' ) - - padCalc = BH_multi_padVal(sizeWindow, sizeCalc); @@ -459,13 +386,13 @@ weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', className, 'EVE'); [ refIMG ] = BH_unStackMontage4d(1, ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... + subTomoMeta.(cycleNumber).(imgNAME){1}, ... + subTomoMeta.(cycleNumber).(imgNAME){2},... sizeWindow); [ refWDG ] = BH_unStackMontage4d(1, ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... + subTomoMeta.(cycleNumber).(weightNAME){1},... + subTomoMeta.(cycleNumber).(weightNAME){2},... sizeCalc); % % % % % % % [ peakMask] = gather(BH_mask3d('sphere', sizeMask, peakSearch, maskCenter)); @@ -527,12 +454,12 @@ if (spike_prior) - tiltList_tmp = fieldnames(masterTM.mapBackGeometry); + tiltList_tmp = fieldnames(subTomoMeta.mapBackGeometry); tiltList_tmp = tiltList_tmp(~ismember(tiltList_tmp,{'viewGroups','tomoName'})); nST = 1; tiltList = {}; % First make sure this tilt actualy has tomos. Why is this here/ for iStack = 1:length(tiltList_tmp) - if masterTM.mapBackGeometry.(tiltList_tmp{iStack}).nTomos + if subTomoMeta.mapBackGeometry.(tiltList_tmp{iStack}).nTomos tiltList{nST} = tiltList_tmp{iStack}; nST = nST +1; end @@ -544,14 +471,10 @@ % word?) i.e. make sure no principle axes are way to big, due to % points from adjacent virions that were not removed in % cleanTemplateSearch. - f = fieldnames(masterTM.mapBackGeometry.tomoName); + f = fieldnames(subTomoMeta.mapBackGeometry.tomoName); for iTomo = 1:length(f) - % tiltName = masterTM.mapBackGeometry.tomoName.(f{iTomo}).tiltName; - % tomoNumber = masterTM.mapBackGeometry.tomoName.(f{iTomo}).tomoNumber; - % iCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - tmpTomo = []; spike_info.(f{iTomo}).('angular_diff') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); spike_info.(f{iTomo}).('normal_distance') = zeros(size(geometry.(f{iTomo}) , 1),emc.nPeaks,'single'); @@ -697,7 +620,7 @@ fprintf('Removing all volumes with score < %2.2f to return the requested percent %2.2f of possible volumes\n\n',cccCutOff,reqVol); end - masterTM.(cycleNumber).('score_sigma') = std(cccVect); + subTomoMeta.(cycleNumber).('score_sigma') = std(cccVect); if (spike_prior) % spike_info.('normalization_factor') = 1;%nVolumes ./ (emc.nPeaks * addedWeight); % fprintf('From %d possible volumes the total weight is %3.3e\n',nVolumes,addedWeight); @@ -727,7 +650,7 @@ else maxCCC = []; avgCCC = []; - masterTM.(cycleNumber).('score_sigma') = 1; + subTomoMeta.(cycleNumber).('score_sigma') = 1; end % % Clear all of the GPUs prior to entering the main processing loop @@ -790,7 +713,7 @@ if (emc.eucentric_fit) try geometry_tmp.(tomoList{iTomo})(:,13) = geometry_tmp.(tomoList{iTomo})(:,13) + ... - masterTM.(sprintf('%s',cycleRead)).('eucentric_shifts').(tomoList{iTomo}) ; + subTomoMeta.(sprintf('%s',cycleRead)).('eucentric_shifts').(tomoList{iTomo}) ; catch fprintf('WARNING, did not find the eucentric shift for tomo %s\n', tomoList{iTomo}); end @@ -799,18 +722,10 @@ peakMask_tmp = gpuArray(peakMask); peakBinary_tmp = gpuArray(peakBinary); - interpPad = 0; interpTrim = 0 ; - if interpOrder == 4 - interpPad = BH_multi_padVal(sizeWindow,2.*sizeWindow); - interpTrim = BH_multi_padVal(2.*sizeWindow,sizeWindow); - interpMask_tmpBinary = single(find(interpMask > 0.01)); - interpMask_tmp = interpMask; - interpMaskWdg_tmp = interpMaskWdg; - else - interpMask_tmpBinary = gpuArray(single(find(interpMask > 0.01 ))); - interpMask_tmp = gpuArray(interpMask); - interpMaskWdg_tmp = (gpuArray(interpMaskWdg)); - end + + interpMask_tmpBinary = gpuArray(single(find(interpMask > 0.01 ))); + interpMask_tmp = gpuArray(interpMask); + interpMaskWdg_tmp = (gpuArray(interpMaskWdg)); if (eachTomo) tomoAvg = zeros(sizeMask, 'single', 'gpuArray'); @@ -825,10 +740,10 @@ - tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); + tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); [ binShift ] = [0,0,0];%BH_multi_calcBinShift( coords, samplingRate); % Load in the geometry for the tomogram, and get number of subTomos. @@ -840,10 +755,10 @@ volumeData = []; %fprintf('loading tomo %d\n',iTomo); - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; fprintf('Loading tomo %d from tilt %s \n',tomoNumber,tiltName); - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); + reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) volumeData = []; @@ -860,7 +775,7 @@ - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + iTiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; wgtName = sprintf('cache/%s_bin%d.wgt',iTiltName,samplingRate); @@ -916,19 +831,16 @@ iParticle = []; iCCCweight = []; iWedgeMask = []; - % symmetry = classVector{iGold}(2, iClassPos); - if ( emc.nPeaks > 1 ) % Calculate a relative weighting, normalize max score to one % and then raise to compressBy factor to downweight lower % scores. - [ peakWgt, sortedList ] = BH_weightAngCheckPeaks( ... positionList(iSubTomo,:),... emc.nPeaks, ... - masterTM.(cycleNumber).('score_sigma') ,... + subTomoMeta.(cycleNumber).('score_sigma') ,... iSubTomo, tomoList{iTomo},... emc.track_stats); % Update any re-ordering or elimination @@ -966,7 +878,7 @@ % angles = reshape(angles,3,3)*oddRot; % end - TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); + TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); if (make_sf3d) [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); @@ -1111,27 +1023,11 @@ %iParticle is already on GPU if it should be. - [~, iParticle] = interpolator(gpuArray(iParticle), angles, iShift, rotConvention , 'inv', symmetry, true); - - % [ iParticle ] = gpuArray( ... - % BH_resample3d(iParticle, ... - % angles, iShift, ... - % {rotConvention ,symmetry,interpM,1,interpMask_tmpBinary}, ... - % interpU,'inv')); - % - % [ iParticle ] = BH_padZeros3d(iParticle, ... - % -1.*padWindow(1,:),-1.*padWindow(2,:),... - % 'GPU','single'); + [~, iParticle] = interpolator(gpuArray(iParticle), angles, iShift, rotConvention , 'inv', emc.symmetry, true); - [~, iWedgeMask] = interpolator(gpuArray(iSF3D), angles, [0,0,0], rotConvention , 'inv', symmetry, true); - - % For now just leave linear interp, but test with spline - % [ iWedgeMask ] = BH_resample3d(iSF3D, ... - % angles, [0,0,0], ... - % {rotConvention ,symmetry,'linear', ... - % 1,interpMaskWdg_tmp}, ... - % 'GPU','inv'); + [~, iWedgeMask] = interpolator(gpuArray(iSF3D), angles, [0,0,0], rotConvention , 'inv', emc.symmetry, true); + iParticle = iParticle - mean(iParticle(interpMask_tmpBinary)); iParticle = iParticle ./ rms(iParticle(interpMask_tmpBinary)); @@ -1339,8 +1235,8 @@ -masterTM.(cycleNumber).('nSubTomoAveraged') = gather(sum(nExtracted(:))); -masterTM.(cycleNumber).(sprintf('newIgnored_Avg%s',fieldPrefix)) = ... +subTomoMeta.(cycleNumber).('nSubTomoAveraged') = gather(sum(nExtracted(:))); +subTomoMeta.(cycleNumber).(sprintf('newIgnored_Avg%s',fieldPrefix)) = ... gather(nIgnored); @@ -1442,7 +1338,7 @@ % class to [most likely] match the corresponding odd class. classListOut = 0; % % % if (flgGold) && strcmpi(STAGEofALIGNMENT, 'Cluster') -if strcmpi(STAGEofALIGNMENT, 'Cluster') && (emc.classification ~= -1) +if strcmpi(STAGEofALIGNMENT, 'Cluster') && ~(emc.classification) % % % [classListOut, geometry] = reorder_classes(filteredClass(:,1),filteredClass(:,2),maxClasses, geometry); % % % filteredClass(:,2) = filteredClass(classListOut(:,2), 2); @@ -1457,7 +1353,7 @@ fprintf(classMatches,'%d\t%d\t%2.6f\n', classListOut'); fclose(classMatches); - masterTM.(cycleNumber).(sprintf('class_%d_%s_EveOddIdx',className,fieldPrefix)) = classListOut; + subTomoMeta.(cycleNumber).(sprintf('class_%d_%s_EveOddIdx',className,fieldPrefix)) = classListOut; end @@ -1465,15 +1361,15 @@ % Second option allows re-use of class designations to generate % multi-reference alignment. if strcmpi(STAGEofALIGNMENT, 'Cluster') - masterTM.(cycleNumber).(ClusterGeomNAME) = geometry; + subTomoMeta.(cycleNumber).(ClusterGeomNAME) = geometry; elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') && emc.multi_reference_alignment if (emc.classification) - masterTM.(cycleNumber).('ClusterClsGeom') = geometry; + subTomoMeta.(cycleNumber).('ClusterClsGeom') = geometry; else - masterTM.(cycleNumber).('ClusterRefGeom') = geometry; + subTomoMeta.(cycleNumber).('ClusterRefGeom') = geometry; end else - masterTM.(cycleNumber).('Avg_geometry') = geometry; + subTomoMeta.(cycleNumber).('Avg_geometry') = geometry; end % Should this save differently depending on the stage of alignment?? I % think so but leave alone for nw. @@ -1499,7 +1395,7 @@ className, fieldPrefix, halfSet); classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', className,fieldPrefix, halfSet); - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; if (flgFinalAvg) system(sprintf('mv %s preHalfSetAli_%s',imout,imout)); @@ -1513,7 +1409,7 @@ classOut = sprintf('class_%d_Locations_%s_%s_Wgt', className,fieldPrefix, halfSet); % For the weight, instead of imgCounts save the padValues - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,fscPAD}; + subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,fscPAD}; if (flgFinalAvg) system(sprintf('mv %s preHalfSetAli_%s',imout,imout)); @@ -1530,7 +1426,7 @@ saveClassSum, 'Raw', halfSet); classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'Raw', halfSet); SAVE_IMG(montOUT, imout,emc.pixel_size_angstroms); - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; [montOUT, imgLocations] = BH_montage4d(classWgtSum(iGold), ''); @@ -1538,32 +1434,30 @@ saveClassSum, 'Raw', halfSet); classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'Raw', halfSet); SAVE_IMG(montOUT, imout,emc.pixel_size_angstroms); - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; end end -masterTM = gather(masterTM); +subTomoMeta = gather(subTomoMeta); classVector = gather(classVector); nExtracted = gather(nExtracted); -subTomoMeta = gather(masterTM); -subTomoMeta.(cycleNumber).('SymmetryApplied').(STAGEofALIGNMENT) = symmetry; +subTomoMeta.(cycleNumber).('SymmetryApplied').(STAGEofALIGNMENT) = emc.symmetry; subTomoMeta.(cycleNumber).('ClassVector').(STAGEofALIGNMENT) = classVector; cycleNumber = gather(cycleNumber); subTomoMeta.('currentCycle') = gather(CYCLE); -save(emc.('subTomoMeta'), 'subTomoMeta'); - +save(emc.subTomoMeta, 'subTomoMeta'); -fprintf('Total execution time : %f seconds\n', etime(clock, startTime)); +fprintf('Total execution time : %f seconds\n', datetime("now")-startTime); % clean everything up, since this function is called from other functions. @@ -1629,10 +1523,10 @@ % This is slow ass when using cones and class averages and wouldn't be too % hard to put into parallel. Do that once the next manuscript is finished. - if (~emc.multi_reference_alignment && ~emc.classification ) - nClassesReWgt = 1; - else + if (emc.multi_reference_alignment || emc.classification ) nClassesReWgt = maxClasses; + else + nClassesReWgt = 1; end for iRef = 1:nClassesReWgt @@ -1651,7 +1545,8 @@ iRefPrev = iRef; end - if (flgGold) || (emc.classification < 0) + % FIXME: logic? + if (flgGold || emc.classification) flgCombine = 0; flgRefCutOff = 1; else @@ -1659,38 +1554,16 @@ flgRefCutOff = 0; end + fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRefPrev)); + aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); + mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); - - - - % When switching from no Classification to classification, there will be no - % previous savePrefix ref, it will be Raw. Another reason to get rid of the - % (artificial) distinction. For now, use a try catch . - - try - fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRefPrev)); - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); - mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); - catch - fprintf('\nReverting from %s to Raw in loading fitFSC\n',savePrefix); - fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d','Raw',iRefPrev)); - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)); - mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d','Raw',iRefPrev)); - end - - % if (flgGold) && strcmpi(STAGEofALIGNMENT, 'Cluster') - % iOdd = cccFinal(iRef,2); - % iEve = cccFinal(iRef,3); - % else iOdd = iRef; iEve = iRef; - % end if (flgFinalAvg) % negative to combine but NOT apply the xform to the odd set flgCombine = -1; - flgRefCutoff = 0; - end % Only send the lowest Bfactor if not flgFinalAvg @@ -1707,7 +1580,7 @@ if ~(flgFinalAvg) refIMG{1}{iOdd} = refTMP{1,1}; - if (flgGold) || (emc.classification < 0) + if (flgGold || emc.classification) refIMG{2}{iEve} = refTMP{1,2}; end clear refTMP @@ -1717,7 +1590,7 @@ for iGold = 1:2-flgFinalAvg - if( flgGold ) || (emc.classification < 0) + if( flgGold || emc.classification ) if iGold == 1 halfSet = 'ODD'; else diff --git a/transformations/BH_axialSymmetry.m b/transformations/BH_axialSymmetry.m deleted file mode 100755 index 9a4b18d8..00000000 --- a/transformations/BH_axialSymmetry.m +++ /dev/null @@ -1,72 +0,0 @@ -function [ symIMG ] = BH_axialSymmetry( IMG, SYMMETRY, ANG_SHIFT, ... - METHOD, XYZ_SHIFT ) -%Apply axial (z) symmetry to an image. -% Goal is to generate either a symmetric reference that is more planar, or a -% rotationally smeared out reference to use to speed up a grid search by first -% searching over the polar and azimuthal angles with the rotationally averaged -% ref, then searching the in-plane angles at the best position only. - -overrideRadial = 0; -if SYMMETRY == -720 - overrideRadial = true; - SYMMETRY = 720; -end -if SYMMETRY > 0 - if SYMMETRY > 60 - if ~(overrideRadial) - flgRadial = true; - else - flgRadial = true; - end - % Generating a rotationally smeared out reference, not just applying axial symmetry - % Create a radial mask similar to ramp weight for WBP - [ radialCylinder ,~,~,~,~,~ ] = BH_multi_gridCoordinates( size(IMG), ... - 'Cylindrical',... - METHOD, ... - {'none'}, ... - 0, 0, 0 ); - else - flgRadial = false; - end - - symInc = 360/SYMMETRY; - % An additional axial rotation to orient the reference, use positive sense - % for indexing - if ANG_SHIFT < 0 - ANG_SHIFT = 360 + ANG_SHIFT; - end - - if SYMMETRY > 6 - % Old slower way - symIMG = zeros(size(IMG), 'single','gpuArray'); - for iSym = 0:SYMMETRY-1 - symIMG = symIMG + BH_resample3d(IMG,[ANG_SHIFT+(iSym.*symInc),0,0],XYZ_SHIFT, 'Bah', 'GPU', 'forward'); - end - else - % More memory but faster - symIMG = BH_resample3d(IMG,[ANG_SHIFT,0,0],XYZ_SHIFT, {'Bah',SYMMETRY,'linear'}, 'GPU', 'forward'); - end - -else - - if SYMMETRY == -1 || SYMMETRY == -2 || SYMMETRY == -3 - symIMG = flip(img,abs(SYMMETRY)); - else - error('mirror symmetry must be -1,-2,-3') - end - -end - -if (flgRadial) - symIMG = real(ifftn(fftn(symIMG).*radialCylinder)); -end - - - - - - - - - - From 3e0e727c65bab995c871e42d13204a95de037737 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 14:20:49 -0500 Subject: [PATCH 016/151] basic parsing passes --- alignment/BH_alignRaw3d_v2.m | 42 +++-------- alignment/BH_templateSearch3d_2.m | 25 ++----- logicals/EMC_assert_numeric.m | 8 ++- metaData/BH_parseParameterFile.m | 78 ++++++++++++++++++-- statistics/BH_clusterPub.m | 114 +++--------------------------- testScripts/emClarity.m | 12 ++-- transformations/BH_average3d.m | 14 ++-- 7 files changed, 115 insertions(+), 178 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 4e33b0f7..fe729bd7 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -68,11 +68,7 @@ reconScaling = 1; -try - use_new_grid_search = emc.('use_new_grid_search'); -catch - use_new_grid_search = true; -end + maxGoldStandard = subTomoMeta.('maxGoldStandard'); @@ -80,27 +76,9 @@ nGPUs = emc.('nGPUs'); -try - updateClassByBestReferenceScore = emc.('updateClassByBestReferenceScore'); -catch - updateClassByBestReferenceScore = false; -end -if (~emc.multi_reference_alignment) - updateClassByBestReferenceScore = false; -end -try - flgCenterRefCOM = emc.('flgCenterRefCOM'); -catch - flgCenterRefCOM = 1; -end -% FIXME: unused, fix experimental options option -try - flgSymmetrizeSubTomos = emc.('flgSymmetrizeSubTomos'); -catch - flgSymmetrizeSubTomos = 0; -end + flgRaw_shapeMask = 0;%= emc.('experimentalOpts')(3) samplingRate = emc.('Ali_samplingRate'); @@ -289,7 +267,7 @@ sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}(2:2:6)'; - if (flgCenterRefCOM) + if (emc.move_reference_by_com) % % % % % % % [ comMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); [ comMask ] = EMC_maskShape(maskType, sizeMask, maskRadius, 'gpu', {'shift', maskCenter}); end @@ -299,7 +277,7 @@ for iP = 1:numel(refTMP) if ~isempty(refTMP{iP}) tIMG{n} = refTMP{iP}; refTMP{iP} = []; - if (flgCenterRefCOM) + if (emc.move_reference_by_com) [~, ~, ~,iCOM] = EMC_maskReference(gpuArray(tIMG{n}).*comMask, emc.pixel_size_angstroms, {'fsc',true; 'com', true}); fprintf('centering ref %d on COM %3.3f %3.3f %3.3f \n',n,iCOM); @@ -533,7 +511,7 @@ updateWeights = false; gridSearch = ''; -if (use_new_grid_search) +if (emc.use_new_grid_search) gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, true); nAngles = sum(gridSearch.number_of_angles_at_each_theta); @@ -887,7 +865,7 @@ particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); end - if (use_new_grid_search) + if (emc.use_new_grid_search) theta_search = 1:gridSearch.number_of_out_of_plane_angles; else theta_search = 1:size(angleStep,1); @@ -895,7 +873,7 @@ for iAngle = theta_search - if (use_new_grid_search) + if (emc.use_new_grid_search) theta = gridSearch.parameter_map.theta(iAngle); if length(gridSearch.parameter_map.phi{iAngle}) > 1 phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); @@ -924,7 +902,7 @@ - if (use_new_grid_search) + if (emc.use_new_grid_search) % FIXME randomizer passed as bool to eulerSearch phi_search = gridSearch.parameter_map.phi{iAngle}; else @@ -934,7 +912,7 @@ for iAzimuth = phi_search - if (use_new_grid_search) + if (emc.use_new_grid_search) phi = rem(iAzimuth + azimuthalRandomizer,360); psiInc = gridSearch.psi_step; else @@ -1679,7 +1657,7 @@ % save('bestAnglesTemp.mat', 'bestAngles'); save('bestAngles.mat', 'bestAngles'); - [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, emc.nPeaks, rotConvention, updateWeights, updateClassByBestReferenceScore); + [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, emc.nPeaks, rotConvention, updateWeights, emc.update_class_by_ccc); masterTM.(cycleNumber).('RawAlign') = rawAlign; masterTM.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); masterTM.('updatedWeights') = true; diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index ad7ddaf1..de3e355e 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -94,11 +94,6 @@ convTMPNAME = sprintf('convmap_wedgeType_%d_bin%d',wedgeType,samplingRate) -try - use_new_grid_search = emc.('use_new_grid_search'); -catch - use_new_grid_search = true; -end try eraseMaskType = emc.('Peak_mType'); @@ -303,28 +298,20 @@ rotConvention = 'Helical'; end - - -if (use_new_grid_search) - +if (emc.use_new_grid_search) gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, false); nAngles = sum(gridSearch.number_of_angles_at_each_theta); inPlaneSearch = gridSearch.parameter_map.psi; - - else [ nInPlane, inPlaneSearch, angleStep, nAngles] ... = BH_multi_gridSearchAngles(angleSearch) end - - highThr=sqrt(2).*erfcinv(ceil(peakThreshold.*0.10).*2./(prod(size(tomogram)).*nAngles(1))) - [ OUTPUT ] = BH_multi_iterator( [targetSize; ... size(tomogram);... sizeTempBIN; ... @@ -666,7 +653,7 @@ % swapQuadrants = swapQuadrants(1:floor(size(swapQuadrants,1)/2)+1,:,:); -if (use_new_grid_search) +if (emc.use_new_grid_search) theta_search = 1:gridSearch.number_of_out_of_plane_angles; else theta_search = 1:size(angleStep,1); @@ -675,7 +662,7 @@ for iAngle = theta_search - if (use_new_grid_search) + if (emc.use_new_grid_search) theta = gridSearch.parameter_map.theta(iAngle); numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); else @@ -723,7 +710,7 @@ % chunk. firstLoopOverChunk = true; - if (use_new_grid_search) + if (emc.use_new_grid_search) fprintf('Working on tilt(%d/%d) tomoChunk(%d/%d)\t' ... ,iAngle,gridSearch.number_of_out_of_plane_angles, tomoIDX,nTomograms); else @@ -756,7 +743,7 @@ - if (use_new_grid_search) + if (emc.use_new_grid_search) phi_search = gridSearch.parameter_map.phi{iAngle}; else phi_search = 0:angleStep(iAngle,2); @@ -765,7 +752,7 @@ for iAzimuth = phi_search - if (use_new_grid_search) + if (emc.use_new_grid_search) phi = iAzimuth; else phi = phiStep * iAzimuth; diff --git a/logicals/EMC_assert_numeric.m b/logicals/EMC_assert_numeric.m index cf0ebd9a..2c0d72e1 100644 --- a/logicals/EMC_assert_numeric.m +++ b/logicals/EMC_assert_numeric.m @@ -4,19 +4,21 @@ function EMC_assert_numeric(input_val, varargin) assert_length = false; assert_passed = false; assert_range = false; + if ( nargin > 1 ) assert_length = true; wanted_numel = varargin{1}; - if ~isa(wanted_numel, "integer") + if ~isa(wanted_numel, "numeric") error('EMC_assert_numeric: second argument must be an integer'); end - if (nargin == 3) + if (nargin > 2) assert_range = true; range = varargin{2}; if ( ~isa(range, 'numeric') || numel(range) ~= 2 ) error('EMC_assert_numeric: third argument must be numeric with two elements'); end - else + end + if ( nargin > 3 ) error('EMC_assert_numeric: too many input arguments'); end end diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 02e08051..3662cdac 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -45,13 +45,13 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% if isfield(emc, 'nGPUs') - EMC_assert_integer(emc.nGPUs, 1, [1, 1000]); + EMC_assert_numeric(emc.nGPUs, 1, [1, 1000]); else error('nGPUs is a required parameter'); end if isfield(emc, 'nCpuCores') - EMC_assert_integer(emc.nCpuCores, 1, [1, 1000]); + EMC_assert_numeric(emc.nCpuCores, 1, [1, 1000]); else error('nCpuCores is a required parameter'); end @@ -103,7 +103,7 @@ % Early development parameter, used to store more than one orientation during template matching % and use for further refinement. if isfield(emc, 'nPeaks') - EMC_assert_integer(emc.nPeaks, 1); + EMC_assert_numeric(emc.nPeaks, 1); else emc.('nPeaks') = 1; end @@ -113,7 +113,7 @@ % When used in average3d, this value is stored in the subTomoMeta. if isfield(emc, 'CUTPADDING') - EMC_assert_integer(emc.CUTPADDING, 1); + EMC_assert_numeric(emc.CUTPADDING, 1); else emc.('CUTPADDING') = 20; end @@ -251,7 +251,7 @@ EMC_assert_boolean(emc.fsc_with_chimera); emc = EMC_assert_deprecated_substitution(emc, 0.1, 'minimum_particle_for_fsc_weighting', 'minimumparticleVolume'); -EMC_assert_numeric(emc.minimum_particle_for_fsc_weighting, 0, [0.01, 1.0]); +EMC_assert_numeric(emc.minimum_particle_for_fsc_weighting, 1, [0.01, 1.0]); emc = EMC_assert_deprecated_substitution(emc, 1.0, 'fsc_shape_mask', 'flgFscShapeMask'); EMC_assert_numeric(emc.fsc_shape_mask, 1, [0.0, 2.0]); @@ -279,17 +279,81 @@ emc.('n_scale_spaces') = numel(emc.pca_scale_spaces); if isfield(emc, 'Pca_maxEigs') - EMC_assert_integer(emc.Pca_maxEigs, 1, [1, 1000]); + EMC_assert_numeric(emc.Pca_maxEigs, 1, [1, 1000]); else emc.Pca_maxEigs = 36; end if isfield(emc, 'Pca_randSubset') - EMC_assert_integer(emc.Pca_randSubset, 1); + EMC_assert_numeric(emc.Pca_randSubset, 1); else emc.Pca_randSubset = 0; end +clusterVector= emc.('Pca_clusters'); + +% Allowed values are validated inside BH_clusterPub.m +if ~isfield(emc, 'Pca_distMeasure'); + emc.distance_metric = 'sqeuclidean'; +end + +if isfield(emc, 'Pca_nReplicates'); + EMC_assert_numeric(emc.Pca_nReplicates, 1, [100, 1000]); + emc.n_replicates = 256; +end + +if isfield(emc, 'Pca_refineKmeans') + EMC_assert_boolean(emc.Pca_refineKmeans); +else + emc.Pca_refineKmeans = false; +end + +if isfield(emc, 'Pca_flattenEigs') + EMC_assert_boolean(emc.Pca_flattenEigs); +else + emc.Pca_flattenEigs = true; +end + + +if isfield(emc, 'Pca_som_coverSteps') + EMC_assert_numeric(emc.Pca_som_coverSteps, 1, [1, 1000]); +else + emc.Pca_som_coverSteps = 100; +end + +if isfield(emc, 'Pca_som_initNeighbor') + EMC_assert_numeric(emc.Pca_som_initNeighbor, 1, [1, 32]); +else + emc.Pca_som_initNeighbor = 3; +end + +if ~isfield(emc, 'Pca_som_topologyFcn') + % TODO: assert on allowed values + emc.Pca_som_topologyFcn = 'hextop'; +end + +if isfield(emc, 'spike_prior') + EMC_assert_boolean(emc.spike_prior); +else + emc.spike_prior = false; +end + + +emc = EMC_assert_deprecated_substitution(emc, false, 'update_class_by_ccc', 'updateClassByBestReferenceScore'); +EMC_assert_boolean(emc.update_class_by_ccc); +if (~emc.multi_reference_alignment) + % update by ccc only makes sense for multi reference alignment + emc.update_class_by_ccc = false; +end + +emc = EMC_assert_deprecated_substitution(emc, true, 'move_reference_by_com', 'flgCenterRefCOM'); +EMC_assert_boolean(emc.move_reference_by_com); + +if isfield(emc, 'use_new_grid_search') + EMC_assert_boolean(emc.use_new_grid_search); +else + emc.use_new_grid_search = true; +end end diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index 86d13183..1ce8894d 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -1,44 +1,5 @@ function [ ] = BH_clusterPub(PARAMETER_FILE, CYCLE) -%Kmeans based classification -% -% -% Input Variables: -% -% GEOMETRY = -% -% COEFF_MAT = matfile with previous pca decomposition -% -% nCLUSTERS = vector with number of clusters to try. -% -% COEFFS = cell with 1x2 vectors giving ranges of coeffs to try -% e.g. {[2,40], [7,40]} -% -% kDIST = distance measure to use. I have observed some improved seperation -% for my data using 'cosine' rather than the default. -% -% 'sqeuclidean', 'cityblock', 'cosine', 'correlation' -% -% kREP = number of replicates for each -% -% Output Variables: -% -% None - writes out an updated geometry for each combination of nClUSTERS and -% COEFFS -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% Goals & limitations: -% -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% TODO -% -% Change parpool to 48 prior to testing on archer. Also take a look into the -% available GPU accelerated K means. Check the memory used for coeff and -% whether or not this is limiting. It should not be. -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + if (nargin ~= 2) error('PARAMETER_FILE, CYCLE') end @@ -77,56 +38,8 @@ error('There should be a set of indices for each pca_scale_spaces, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') end -clusterVector= emc.('Pca_clusters'); - -try - kDIST = emc.('Pca_distMeasure'); -catch - kDIST = 'sqeuclidean'; -end -try - kREP = emc.('Pca_nReplicates'); -catch - kREP = 256; -end - -try - flgRefineKmeans = emc.('Pca_refineKmeans') -catch - flgRefineKmeans = false -end - nCores = BH_multi_parallelWorkers(emc.('nCpuCores')); -% try -% relativeScale = emc.('Pca_relativeScale') -% catch -% relativeScale= ones(size(clusterVector,1),1); -% end - -try - flgFlattenEigs = emc.('Pca_flattenEigs') -catch - flgFlattenEigs=1 -end - - -try - coverSteps = emc.('Pca_som_coverSteps'); -catch - coverSteps = 100; -end -try - initNeighbor = emc.('Pca_som_initNeighbor'); -catch - initNeighbor = 3; -end -try - topologyFcn = emc.('Pca_som_topologyFcn'); -catch - % 'hextop' (default) | 'randtop' | 'gridtop' | 'tritop' - topologyFcn = 'hextop'; -end load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); masterTM = subTomoMeta; clear subTomoMeta @@ -165,10 +78,8 @@ kAlgorithm = 'kMeans'; % kAlgorithm = 'neuralNetwork' - - kDist = sprintf('%s', kDIST) - - switch kDist + + switch emc.distance_metric case 'sqeuclidean' kDistMeasure = 'sqeuclidean' case 'cityblock' @@ -184,16 +95,13 @@ kDistMeasure = 'neural' kAlgorithm = 'neuralNetwork' fprintf('Input params for neural network are %d %d %s\n', ... - coverSteps, initNeighbor, topologyFcn); + emc.coverSteps, emc.Pca_som_initNeighbor, emc.topologyFcn); otherwise kDistMeasure = 'sqeuclidean' fprintf(['\nDefaulting to sqeuclidean b/c %s was not recognized'] ... , kDist); end - kReplicates = kREP; - - %kDistMeasure = 'euclidean' try oldPca = load(coeffMatrix); @@ -249,7 +157,7 @@ % % % coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) ./ ... % % % repmat(rms(coeffMat(1+nAdded:nAdded+nFeatures(iScale),:),2),1,nJ).*iScale; - if (flgFlattenEigs) + if (emc.Pca_flattenEigs) coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) = ... coeffMat(1+nAdded:nAdded+nFeatures(iScale),:) ./ ... repmat(rms(coeffMat(1+nAdded:nAdded+nFeatures(iScale),:),2),1,nJ); @@ -271,14 +179,14 @@ if strcmpi(kAlgorithm, 'kMeans') [class, classCenters, sumd, D] = kmeans(coeffMat', nClusters, ... - 'replicates', kReplicates, ... + 'replicates', emc.n_replicates, ... 'Distance', kDistMeasure, ... 'MaxIter', 50000, ... % Default was 100 'Options', statset('UseParallel', 1) ); elseif strcmpi(kAlgorithm, 'kMedoids') [class, classCenters, sumd, D] = kmedoids(coeffMat', nClusters, ... - 'replicates', kReplicates, ... + 'replicates', emc.n_replicates, ... 'Distance', kDistMeasure, ... 'Options', statset('UseParallel', 1, ... 'MaxIter', 50000) ); @@ -293,7 +201,7 @@ elseif strcmpi(kAlgorithm, 'neuralNetwork') - net = selforgmap([1 nClusters], coverSteps, initNeighbor, topologyFcn); + net = selforgmap([1 nClusters], emc.coverSteps, emc.Pca_som_initNeighbor, emc.topologyFcn); [net, tr] = train(net, coeffMat); y = net(coeffMat) class = vec2ind(y) @@ -310,7 +218,7 @@ fprintf('Total kmeans dist = %g\n', totSum1) fprintf('Total kmeans std = %g\n', totStd1) - if (flgRefineKmeans) + if (emc.Pca_refineKmeans) % Using the postions found, refine the original estimates kMin = min(classCenters,[],1); @@ -346,14 +254,14 @@ % 'Options', statset('UseParallel', 1) ); if strcmpi(kAlgorithm, 'kMeans') [class, classCenters, sumd,D] = kmeans(coeffMat', nClusters, ... - 'replicates', kReplicates, ... + 'replicates', emc.n_replicates, ... 'Distance', kDistMeasure, ... 'MaxIter', 50000, ... % Default was 100 'Options', statset('UseParallel', 1) ); elseif strcmpi(kAlgorithm, 'kMedoids') [class, classCenters, sumd,D] = kmedoids(coeffMat', nClusters, ... - 'replicates', kReplicates, ... + 'replicates', emc.n_replicates, ... 'Distance', kDistMeasure, ... 'Options', statset('UseParallel', 1, ... 'MaxIter', 50000) ); diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 9a877408..ee37ccf9 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -687,7 +687,7 @@ % Try to parse the parameter file make sure it's okay. % % Add some actual error handling here to help trouble shoot. -try +% try emc = BH_parseParameterFile( paramTest ); @@ -884,9 +884,9 @@ bh_global_binary_mask_low_pass, ... bh_global_binary_mask_threshold, ... bh_global_binary_pcaMask_threshold); -catch - error('error parsing parameter file %s\n', paramTest) -end +% catch +% error('error parsing parameter file %s\n', paramTest) +% end @@ -936,6 +936,8 @@ function print_experimental_options + % FIXME: make these match the corrections to deprecated options in parseParemeterFile and extend to include + % the (many) that are currently ommitted. fprintf('\n\n\tExperimental Options: use at your own RISK\n'); fprintf('(\t\tOr better yet, check with ben!\t\t)\n'); fprintf('\nIf you do use/change any of these, please mention in your methods and EMDB entry!\n'); @@ -948,7 +950,7 @@ fprintf('\nflgQualityWeight\t Downweight high-freq of low scoring sub-tomos. Def:\t4\n'); fprintf('\nlimit_to_one_core\t For OOM issues in averaging. Boolean Def:\t0\n'); -fprintf('\nflgCenterRefCOM\tShift reference to center of mass. Boolean Def:\t1\n'); +fprintf('\move_reference_by_com\tShift reference to center of mass. Boolean Def:\t1\n'); fprintf('\nconserveDiskSpace\n'); fprintf('\nPca_distMeasure\tMeasure for difference. euclidean, cityblock, correlation, cosine Def:\t sqeuclidean\n'); fprintf('\nPca_nReplicates\tThe number of times Kmeans is intialized. Def:\t 128\n'); diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 53b5309b..bd2416dd 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -437,11 +437,7 @@ end maxCCC = 0; -try - spike_prior = emc.('spike_prior') -catch - spike_prior = false -end + spike_info = struct(); spike_info.('std_dev') = nan; if (emc.flgQualityWeight) @@ -452,7 +448,7 @@ angVect = []; chiVect = []; - if (spike_prior) + if (emc.spike_prior) tiltList_tmp = fieldnames(subTomoMeta.mapBackGeometry); tiltList_tmp = tiltList_tmp(~ismember(tiltList_tmp,{'viewGroups','tomoName'})); @@ -559,7 +555,7 @@ end min_weight = 1e-6; - if (spike_prior) + if (emc.spike_prior) for iSubTomo = 1:size(geometry.(tomoList{iTomo}) , 1) peakList = false(emc.nPeaks,1); for iPeak = 1:emc.nPeaks @@ -621,7 +617,7 @@ end subTomoMeta.(cycleNumber).('score_sigma') = std(cccVect); - if (spike_prior) + if (emc.spike_prior) % spike_info.('normalization_factor') = 1;%nVolumes ./ (emc.nPeaks * addedWeight); % fprintf('From %d possible volumes the total weight is %3.3e\n',nVolumes,addedWeight); end @@ -849,7 +845,7 @@ peakWgt = 1; end - % % if (spike_prior) + % % if (emc.spike_prior) % % peakWgt = peakWgt.*spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,1).*spike_info.('normalization_factor'); % % end From d9ef47d2e9e8a0b73e2986f113d017e0efe41164 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 16:06:17 -0500 Subject: [PATCH 017/151] through basic avg and fsc, single ref align is running --- alignment/BH_alignRaw3d_v2.m | 58 ++++------------- ctf/BH_ctf_Correct3d.m | 31 +++++---- metaData/BH_geometryAnalysis.m | 6 +- metaData/BH_parseParameterFile.m | 8 +-- statistics/BH_clusterPub.m | 2 +- statistics/BH_fscGold_class.m | 90 +++++++++++---------------- statistics/BH_multi_cRef_VnormApply.m | 20 +----- statistics/BH_pcaPub.m | 18 ++---- synthetic/BH_synthetic_mapBack.m | 21 ------- transformations/BH_average3d.m | 74 ++++++++++------------ 10 files changed, 110 insertions(+), 218 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index fe729bd7..48cd83bb 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -107,7 +107,7 @@ % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search -if ( doHelical ) +if ( emc.doHelical ) rotConvention = 'Helical'; end @@ -248,10 +248,10 @@ end - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) + imgNAME = sprintf('class_%d_Locations_Ref_%s', refName, halfSet) - weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', refName, halfSet); + weightNAME = sprintf('class_%d_Locations_Ref_%s_Wgt', refName, halfSet); imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; @@ -369,12 +369,9 @@ end if (emc.classification || emc.multi_reference_alignment) for iRef = 1:nReferences(1) - if (emc.classification) - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('REF%d',iRef)); - else - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Raw%d',iRef)); % % % % - end - + + fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Ref%d',iRef)); + [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... 'GPU', {'none'}, 1, 0, 1 ); radialGrid = single(radialGrid./emc.pixel_size_angstroms); @@ -394,7 +391,7 @@ for iRef = 1 - fscINFO = masterTM.(cycleNumber).('fitFSC').('Raw1'); + fscINFO = masterTM.(cycleNumber).('fitFSC').('Ref1'); [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... 'GPU', {'none'}, 1, 0, 1 ); radialGrid = single(radialGrid./emc.pixel_size_angstroms); @@ -592,8 +589,11 @@ parVect = 1:nParProcesses; fprintf('Starting main loopwith N references %d\n', nReferences(1)); + +% This may be modified in the parfor loop (tho that prob isn't really necessary) + parfor iParProc = parVect - + symmetry = emc.symmetry; bestAngles_tmp = struct(); geometry_tmp = geometry; @@ -956,15 +956,7 @@ estPeakCoord = bestOfRefs(1,8:10); - - - - - % fprintf('Symmetry confirmation %d\n',symmetry); - % [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... - % estPeakCoord,... - % {'Bah',symmetry,'linear',1,volBinary_tmp}, ... - % 'GPU', 'inv',inputVectors); + [ iTrimParticle ] = particleInterpolator.interp3d(... RotMat,... estPeakCoord,rotConvention ,... @@ -974,23 +966,11 @@ if (getInitialCCC) - % [ iTrimInitial ] = BH_resample3d(iparticle, ... - % reshape(angles,3,3),... - % shiftVAL,... - % {rotConvention ,symmetry,'linear',1,volBinary_tmp}, ... - % 'GPU', 'inv',inputVectors); [ iTrimInitial ] = particleInterpolator.interp3d(... reshape(angles,3,3),... shiftVAL,rotConvention ,... 'inv',symmetry); - - % % % powerInitial = sum(abs(iTrimInitial(volBinary_tmp))).^2; - % % % - - % iWedgeInitial = BH_resample3d(iMaxWedgeMask, reshape(angles,3,3), [0,0,0], ... - % {rotConvention ,symmetry,'linear',1,wdgBinary_tmp}, ... - % 'GPU', 'inv',inputWgtVectors); [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... reshape(angles,3,3),... [0,0,0],rotConvention ,... @@ -1269,30 +1249,16 @@ case 2 - % Assuming if class specific symmetry, then some not just 1 if (force_no_symmetry) symmetry = 'C1'; end - % [ iTrimParticle ] = BH_resample3d(iparticle, RotMat,... - % rXYZ,... - % {rotConvention ,symmetry,'linear',1,volBinary_tmp}, ... - % 'GPU', 'inv',inputVectors); [ iTrimParticle ] = particleInterpolator.interp3d(... RotMat,... rXYZ,rotConvention ,... 'inv',symmetry); - % iTrimParticle = iTrimParticle(... - % padWindow(1,1) + 1:end - padWindow(2,1) , ... - % padWindow(1,2) + 1:end - padWindow(2,2) , ... - % padWindow(1,3) + 1:end - padWindow(2,3) ); - % - % iWedgeMask = BH_resample3d(iMaxWedgeMask, RotMat, [0,0,0], ... - % {rotConvention ,symmetry,'linear',1,wdgBinary_tmp},... - % 'GPU', 'inv',inputWgtVectors); - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... RotMat,... [0,0,0],rotConvention ,... diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index b407e52e..b9e571db 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -926,8 +926,7 @@ end -function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, emc.pixel_size_angstroms,... - nSections,tiltName, ctf3Depth) +function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, pixel_size_angstroms, nSections,tiltName, ctf3Depth) nTomos = length(tomoNumber); sectionList = cell(nTomos,1); @@ -938,7 +937,7 @@ % With rounding this could end up a bit short except the top and bottom are both % half a section larger than minimally needed. -nSec = floor(ctf3Depth*10^10/emc.pixel_size_angstroms) ; +nSec = floor(ctf3Depth*10^10/pixel_size_angstroms) ; nSec = nSec + ~mod(nSec,2); halfSec = (nSec-1)/2; @@ -1031,7 +1030,7 @@ function [correctedStack] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... + avgZ,TLT,pixel_size_angstroms,maskedStack,... maxZ,flgDampenAliasedFrequencies,... preCombDefocus,samplingRate,... applyExposureFilter,surfaceFit,... @@ -1104,12 +1103,12 @@ 'Cylindrical','GPU', ... {'none'},1,0,0); end -radialGrid = {radialGrid./emc.pixel_size_si,0,phi}; +radialGrid = {radialGrid./(pixel_size_angstroms*10^-10),0,phi}; phi = []; -fprintf('%f %f\n',filterProjectionsForTomoCPRBackground,emc.pixel_size_angstroms); +fprintf('%f %f\n',filterProjectionsForTomoCPRBackground,pixel_size_angstroms); if (filterProjectionsForTomoCPRBackground ~= 0) - bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',emc.pixel_size_angstroms); + bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',pixel_size_angstroms); else bpFilter = 1; end @@ -1132,10 +1131,10 @@ - STRIPWIDTH = min(floor((0.5*ctf3dDepth/emc.pixel_size_si)/abs(tand(TLT(iPrj,4)))),512); + STRIPWIDTH = min(floor((0.5*ctf3dDepth/(pixel_size_angstroms*10^-10))/abs(tand(TLT(iPrj,4)))),512); STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); % take at least 1200 Ang & include the taper if equal to STRIPWIDTH - tileSize = floor(max(600./emc.pixel_size_angstroms, STRIPWIDTH + 28)); + tileSize = floor(max(600./pixel_size_angstroms, STRIPWIDTH + 28)); tileSize = tileSize + mod(tileSize,2); %fprintf('stripwidth tilesize %d %d\n',STRIPWIDTH,tileSize); incLow = ceil(tileSize./2); @@ -1185,7 +1184,7 @@ % Transform the specimen plane tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ +oX); tY = round(rA(2).*rX + rA(5).*rY + rA(8).*rZ +oY); - tZ = emc.pixel_size_si.*(rA(3).*rX + rA(6).*rY + rA(9).*rZ) + full_defocusOffset; + tZ = (pixel_size_angstroms*10^-10).*(rA(3).*rX + rA(6).*rY + rA(9).*rZ) + full_defocusOffset; % Some edge pixels can be out of bounds depending on the orientation of % the plan fit. Setting to zero will will ignore them (assuming defocus @@ -1222,7 +1221,7 @@ modHqz = []; else - if emc.pixel_size_si < 2.0e-10 + if (pixel_size_angstroms < 2.0) % use double precision - this is not enabled, but needs to be - % requires changes to radial grid as well. Hqz = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,-1); @@ -1262,7 +1261,7 @@ samplingMask(samplingMask == 0) = 1; if (flgWhitenPS(1)) - correctedStack(:,:,TLT(iPrj,1)) =gather(BH_whitenNoiseSpectrum(correctedPrj./samplingMask,'',emc.pixel_size_angstroms,1)); + correctedStack(:,:,TLT(iPrj,1)) =gather(BH_whitenNoiseSpectrum(correctedPrj./samplingMask,'',pixel_size_angstroms,1)); else correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj./samplingMask); @@ -1276,7 +1275,7 @@ function [avgZ, maxZ, tomoNumber,surfaceFit] = calcAvgZ(masterTM,iCoords, ... tiltName,tomoList,... - nTomos, emc.pixel_size_angstroms,... + nTomos, pixel_size_angstroms,... samplingRate,cycleNumber,... sectionList,calcMaxZ) @@ -1308,7 +1307,7 @@ maxZ = maxZ + (samplingRate*2); -maxZ = maxZ.*emc.pixel_size_angstroms./10; +maxZ = maxZ.*pixel_size_angstroms./10; fprintf('combining thickness and shift on tilt %s, found a maxZ %3.3f nm\n',tiltName,maxZ); if (calcMaxZ) @@ -1371,7 +1370,7 @@ zList = zList - tomoOrigin(3) + micOrigin(3); totalZ = totalZ + sum(zList); fprintf('%s tomo has %d subTomos with mean Z %3.3f nm\n', ... - iTomoName, length(zList), mean(zList)*emc.pixel_size_angstroms./10); + iTomoName, length(zList), mean(zList)*pixel_size_angstroms./10); nSubTomos = nSubTomos + length(zList); for iSection = 1:nSections @@ -1401,7 +1400,7 @@ end % loop over tomos -avgZ = totalZ/nSubTomos*emc.pixel_size_angstroms/10*10^-9; +avgZ = totalZ/nSubTomos*pixel_size_angstroms/10*10^-9; % sf(x,y) = p00 + p10*x + p01*y; % surfaceFit = fit([xFull, yFull],zFull,'poly11'); diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 6038ccf6..6bb5db26 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -87,9 +87,9 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... switch STAGEofALIGNMENT case 'TiltAlignment' - fieldPrefix = 'Raw' + fieldPrefix = 'Ref' case 'RawAlignment' - fieldPrefix = 'Raw'; + fieldPrefix = 'Ref'; case 'Cluster_cls' fieldPrefix = 'Cls'; STAGEofALIGNMENT = 'Cluster'; @@ -156,7 +156,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... if strcmpi(fieldPrefix, 'Cls') geometry = subTomoMeta.(cycleNumber).('ClusterClsGeom'); clusterGeom = 'ClusterClsGeom'; - elseif strcmpi(fieldPrefix, 'ref') + elseif strcmpi(fieldPrefix, 'Ref') geometry = subTomoMeta.(cycleNumber).('ClusterRefGeom'); clusterGeom = 'ClusterRefGeom'; end diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 3662cdac..d1fe6ee0 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -198,11 +198,9 @@ % TODO: these should maybe be two different orthogonal parameters % if > 1 keep this many subtomos % if < 1 keep this fraction -if isfield(emc,'flgCCCcutoff') - EMC_assert_numeric(emc.flgCCCcutoff,1) -else - emc.flgCCCcutoff = 0.0; -end +emc = EMC_assert_deprecated_substitution(emc, 0.0, 'ccc_cutoff', 'flgCCCcutoff'); +EMC_assert_numeric(emc.ccc_cutoff,1) + % TOOD: DOC emc = EMC_assert_deprecated_substitution(emc, false, 'projectVolumes', 'flgProjectVolumes'); diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index 1ce8894d..95000193 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -370,7 +370,7 @@ save(emc.('subTomoMeta'), 'subTomoMeta'); %save(sprintf('%s_pca.mat',OUTPUT_PREFIX), 'nTOTAL','U', 'S', 'V', 'coeffs') - fprintf('Total execution time on set %s: %f seconds\n', halfSet, datetime("now") - startTime); + fprintf('Total execution time on set %s: %f seconds\n', halfSet, seconds(datetime("now") - startTime)); delete(gcp('nocreate')); end % end of Gold loop diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 2dcd2550..2a9d8d91 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -42,7 +42,7 @@ outputPrefix = sprintf('./FSC/%s_%s', cycleNumber, emc.('subTomoMeta')); samplingRate = emc.('Ali_samplingRate'); -emc.pixel_size_si = emc.pixel_size_angstroms .* samplingRate; +emc.pixel_size_angstroms = emc.pixel_size_angstroms .* samplingRate; if ( emc.flgCones ) coneInc = 30; @@ -74,7 +74,7 @@ % Note this is taken from the class section, not Fsc refName = emc.('Cls_className');% emc.('Ref_className'); -peakSearch = floor(emc.('particleRadius')./pixelSize); +peakSearch = floor(emc.('particleRadius')./emc.pixel_size_angstroms); peakCOM =3; global bh_global_MTF @@ -103,22 +103,23 @@ refVector{1} =1; refVector{2}= 1; STAGEofALIGNMENT = 'RawAlignment'; - fieldPrefix = 'REF' + fieldPrefix = 'Ref' end else switch STAGEofALIGNMENT case 'RawAlignment' - savePrefix = 'Raw'; if (emc.classification) - fieldPrefix = 'Raw'; - + % FIXME: This won't be correct for multi-ref alignment if testing out the multi-ref classifications + fieldPrefix = 'Cls'; + savePrefix = 'Cls'; className = 0; classVector = [0;1]; else className = emc.(sprintf('Raw_className')); classVector = emc.(sprintf('Raw_classes_odd')); - fieldPrefix = 'REF'; + fieldPrefix = 'Ref'; + savePrefix = 'Ref'; end nReferences = length(classVector(1,:)) @@ -132,35 +133,14 @@ refVector{1} =1; refVector{2}= 1; outputPrefix = sprintf('%s_Raw', outputPrefix); - - case 'NoAlignment' - savePrefix = 'Raw'; - if (emc.classification) - - fieldPrefix = 'NoA'; - else - fieldPrefix = 'REF'; - end - imageName{1} = sprintf('class_0_Locations_%s_ODD_NoWgt', fieldPrefix); - imageName{2} = sprintf('class_0_Locations_%s_EVE_NoWgt', fieldPrefix); - weightName{1} = sprintf('class_0_Locations_%s_ODD_Wgt', fieldPrefix); - weightName{2} = sprintf('class_0_Locations_%s_EVE_Wgt', fieldPrefix); - - nReferences = 1; - refVector{1} =1; - refVector{2}= 1; - - outputPrefix = sprintf('%s_NoA', outputPrefix); + case 'Cluster' error('Fsc calculation for cluster results is not implemented.') case 'SnrEstimate' savePrefix = 'SNR'; flgEstSNR = 1; - if ( CYCLE ) - fieldPrefix = 'Raw' - else - fieldPrefix = 'NoA' - end + fieldPrefix = 'Ref' + imageName{1} = sprintf('class_%d_Locations_%s_ODD_NoWgt', 25,fieldPrefix); imageName{2} = sprintf('class_%d_Locations_%s_EVE_NoWgt', 25,fieldPrefix); outputPrefix = sprintf('%s_Snr', outputPrefix); @@ -177,7 +157,7 @@ end [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', pixelSize) + BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc] = ... BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) @@ -288,13 +268,13 @@ fprintf('working on %d/ %d references FscGold\n', iRef, nReferences); - [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(refIMG{1}{iRef}), pixelSize, {'fsc', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); - [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(refIMG{2}{iRef}), pixelSize, {'fsc', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); + [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(refIMG{1}{iRef}), emc.pixel_size_angstroms, {'fsc', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); + [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(refIMG{2}{iRef}), emc.pixel_size_angstroms, {'fsc', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); if (emc.shape_mask_test) fprintf('\nSaving your masks and exiting!\n'); SAVE_IMG(shapeMask_1,sprintf('%s-shape_mask_%2.2f_lowpass_%2.2f_threshold.mrc', ... - outputPrefix, emc.shape_mask_lowpass,emc.shape_mask_threshold),pixelSize); + outputPrefix, emc.shape_mask_lowpass,emc.shape_mask_threshold),emc.pixel_size_angstroms); return; end @@ -429,13 +409,13 @@ img1=IMG1; img2=IMG2; - [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(img1), pixelSize, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); - [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(img2), pixelSize, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); + [shapeMask_1, pV1, particleFraction1, ~] = EMC_maskReference(gpuArray(img1), emc.pixel_size_angstroms, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); + [shapeMask_2, pV2, particleFraction2, ~] = EMC_maskReference(gpuArray(img2), emc.pixel_size_angstroms, {'fsc', true; 'lowpass', mask_lowpass; 'threshold', mask_threshold}); if (emc.shape_mask_test) fprintf('\nSaving your masks and exiting!\n'); SAVE_IMG(shapeMask_1,sprintf('%s-shape_mask_%2.2f_lowpass_%2.2f_threshold.mrc', ... - outputPrefix, emc.shape_mask_lowpass,emc.shape_mask_threshold),pixelSize); + outputPrefix, emc.shape_mask_lowpass,emc.shape_mask_threshold),emc.pixel_size_angstroms); return; end @@ -486,7 +466,7 @@ [ fscPAD ] = BH_multi_padVal(size(img1), padDIM(1)); [ rad,~,~,~,~,~ ] = BH_multi_gridCoordinates(padDIM.*[1,1,1], 'Cartesian', 'GPU', ... {'none'}, 1, 0, 1 ); - rad = single(rad)./pixelSize; + rad = single(rad)./emc.pixel_size_angstroms; if (emc.fsc_shape_mask) @@ -498,7 +478,7 @@ % applied during the FSC calculation. If instead it is used to estimate % the particle volume (flgEstSolvent) then no mask is directly applied. fscRandCutoffRes = 3*masterTM.currentResForDefocusError(1); - lowResShift = pixelSize*2 - 10; + lowResShift = emc.pixel_size_angstroms*2 - 10; if lowResShift <= 0 lowResShift = 0 else @@ -507,13 +487,13 @@ % Randomize beyond ~ 20A -- calc so that the cutoff is exactly where a FSC % shell is bound. the 10 in the divisor is set in the calc_shells function,. binDiv = ceil(1.5*padDIM(1)^(1/3)); - shellInc = 0.5/(floor(padDIM(1)/binDiv)*pixelSize); + shellInc = 0.5/(floor(padDIM(1)/binDiv)*emc.pixel_size_angstroms); randCutoff = floor((1/(fscRandCutoffRes)-lowResShift)/ shellInc) * shellInc; fscTcutoff = (floor((1/(fscRandCutoffRes*.95)-lowResShift)/ shellInc)) * shellInc; %Calculate the fsc on phase randomized masked volumes. [randGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(padDIM.*[1,1,1], 'Cartesian', ... 'cpu', {'none'}, 1, 0, 1 ); - randGrid = single(randGrid./pixelSize); + randGrid = single(randGrid./emc.pixel_size_angstroms); randLowRES = (randGrid < randCutoff); randHighRES = (randGrid >= randCutoff); @@ -544,7 +524,7 @@ fou2 = fftn(fou2.*BH_padZeros3d(pV2 , fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); [shellsRandFreq, shellsRandFSC, ~,~] = ... - calc_shells(fou1, fou2, rad, pixelSize, coneList,'rand'); + calc_shells(fou1, fou2, rad, emc.pixel_size_angstroms, coneList,'rand'); clear fou1 fou2 else @@ -569,14 +549,14 @@ [ img2 ] = BH_padZeros3d(img2 , fscPAD(1,:), fscPAD(2,:), 'GPU', 'singleTaper'); fou2 = fftn(img2); %clear img2 [shellsFreq, shellsFSC, shellsNUM,shellsPOWER] = ... - calc_shells(fou1, fou2, rad, pixelSize,coneList, halfAngle); + calc_shells(fou1, fou2, rad, emc.pixel_size_angstroms,coneList, halfAngle); clear fou1 fou2 if (emc.fsc_shape_mask) fou1 = fftn(img1.*BH_padZeros3d(pV1, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); clear pv1 fou2 = fftn(img2.*BH_padZeros3d(pV2, fscPAD(1,:), fscPAD(2,:), 'GPU', 'single')); [tightFreq, tightFSC,~,~] = ... - calc_shells(fou1, fou2, rad, pixelSize,coneList, halfAngle); + calc_shells(fou1, fou2, rad, emc.pixel_size_angstroms,coneList, halfAngle); fitTightFSC = csape(tightFreq(:,1),tightFSC(:,1),'variational'); clear pv2 end @@ -593,7 +573,7 @@ end % Oversampled curve - osX = [0:0.001:0.5]'./pixelSize; + osX = [0:0.001:0.5]'./emc.pixel_size_angstroms; @@ -680,18 +660,18 @@ try oneBitCut(1) = find(fnval(fitFSC{1},osX)-aliBIT < 0 & osX > 1/100, 1, 'first'); catch - oneBitCut(1) = find(osX .* pixelSize > 0.425, 1, 'first'); + oneBitCut(1) = find(osX .* emc.pixel_size_angstroms > 0.425, 1, 'first'); end try halfBitCut(1)= find(fnval(fitFSC{1},osX)-halfBIT < 0 & osX > 1/100, 1, 'first'); catch - halfBitCut(1) = find(osX .* pixelSize > 0.425, 1, 'first'); + halfBitCut(1) = find(osX .* emc.pixel_size_angstroms > 0.425, 1, 'first'); end % Particularly for working at higher binning, this allows using the full % frequency range, which is the most information/calc. if isempty(lowCut1) - lowCut1 = find(osX .* pixelSize > 0.425, 1, 'first'); + lowCut1 = find(osX .* emc.pixel_size_angstroms > 0.425, 1, 'first'); end @@ -707,7 +687,7 @@ % Particularly for working at higher binning, this allows using the full % frequency range, which is the most information/calc. if isempty(lowCutAlign) - lowCutAlign = find(osX .* pixelSize > 0.425, 1, 'first'); + lowCutAlign = find(osX .* emc.pixel_size_angstroms > 0.425, 1, 'first'); end forceMaskAlign{1} = exp(-0.005.^-2 .* (osX-osX(oneBitCut(1))).^2); @@ -726,12 +706,12 @@ try oneBitCut(iCone+1) = find(fnval(fitFSC{iCone+1},osX)- aliBIT < 0 & osX > 1/100, 1, 'first'); catch - oneBitCut(iCone+1) = find(osX .* pixelSize > 0.425, 1, 'first'); + oneBitCut(iCone+1) = find(osX .* emc.pixel_size_angstroms > 0.425, 1, 'first'); end try halfBitCut(iCone+1)= find(fnval(fitFSC{iCone+1},osX)-halfBIT < 0 & osX > 1/100, 1, 'first'); catch - halfBitCut(iCone+1) = find(osX .* pixelSize > 0.425, 1, 'first'); + halfBitCut(iCone+1) = find(osX .* emc.pixel_size_angstroms > 0.425, 1, 'first'); end % first try to use 0.5, if not default to 0.425 cyc/pix @@ -739,7 +719,7 @@ lowCut1 = find(fnval(fitFSC{iCone+1},osX) <= 0.5 & osX > 1/100, 1, 'first'); end if isempty(lowCut1) - lowCut1 = find(osX .* pixelSize > 0.425, 1, 'first'); + lowCut1 = find(osX .* emc.pixel_size_angstroms > 0.425, 1, 'first'); end if lowestRes < 1./osX(lowCut1) @@ -980,7 +960,7 @@ %%% Calc FSC function [shellsFreq, shellsFSC, shellsNsamples, shellsPOWER] = ... - calc_shells(fou1, fou2, rad, pixelSize, coneList, halfAngle) + calc_shells(fou1, fou2, rad, pixel_size_angstroms, coneList, halfAngle) @@ -1026,7 +1006,7 @@ else bin = floor(size(cross,1)/binDiv); end -inc = 0.5 / (bin*pixelSize); +inc = 0.5 / (bin*pixel_size_angstroms); shellsFreq = zeros(bin, nIters, 'gpuArray'); shellsFSC = zeros(bin, nIters, 'gpuArray'); shellsNsamples = zeros(bin, nIters, 'gpuArray'); diff --git a/statistics/BH_multi_cRef_VnormApply.m b/statistics/BH_multi_cRef_VnormApply.m index dcb6ea7b..37fb2e78 100644 --- a/statistics/BH_multi_cRef_VnormApply.m +++ b/statistics/BH_multi_cRef_VnormApply.m @@ -48,23 +48,9 @@ fscArgs = 1; end - try - fscParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('REF%d',fscArgs)); - aliParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('ResampleREF%d',fscArgs)); - mskParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('MaskREF%d',fscArgs)); - catch - fprintf('\nDid not find REF, looking at Raw assuming this is a class avg\n.'); - try - fscParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('Raw%d',fscArgs)); - aliParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('ResampleRaw%d',fscArgs)); - mskParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('MaskRaw%d',fscArgs)); - catch - fprintf('\nJust using fsc info from the Raw and only class 1\n'); - fscParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('Raw%d',1)); - aliParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('ResampleRaw%d',1)); - mskParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('MaskRaw%d',1)); - end - end + fscParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('Ref%d',fscArgs)); + aliParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('ResampleRef%d',fscArgs)); + mskParams = subTomoMeta.(cycleNumber).fitFSC.(sprintf('MaskRef%d',fscArgs)); pixelSize = 0.5/fscParams{4}(end); if mFactor diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index c0778a53..6e171798 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -277,17 +277,11 @@ oddRot = eye(3); else iRefPrev = 1; - try - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','REF',iRefPrev)) - oddRot = reshape(aliParams(1,:),3,3)'; - % refine the translation per particle. - catch - error('This block sshould not be reached.'); - fprintf('\nReverting from %s to Raw in loading fitFSC\n','REF'); - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)) - oddRot = reshape(aliParams(1,:),3,3)'; - % refine the translation per particle. - end + + aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)) + oddRot = reshape(aliParams(1,:),3,3)'; + % refine the translation per particle. + clear iRefPrev end @@ -1092,7 +1086,7 @@ end end - fprintf('Total execution time on %s set: %f seconds\n', halfSet, datetime("now") - startTime); + fprintf('Total execution time on %s set: %f seconds\n', halfSet, seconds(datetime("now") - startTime)); close all force; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 6a6a4ca3..fbea4bd1 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -396,27 +396,6 @@ end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% try -% refNameODD = sprintf('%s_%s_class0_REF_ODD.mrc', ... -% cycleNumber,emc.('subTomoMeta')); -% refNameEVE = sprintf('%s_%s_class0_REF_EVE.mrc', ... -% cycleNumber,emc.('subTomoMeta')); -% refODD = getVolume(MRCImage(refNameODD)); -% refEVE = getVolume(MRCImage(refNameEVE)); -% catch -% fprintf('\nDid not find either %s or %s, trying Raw prefix\n',refNameODD,refNameEVE); -% try -% refNameODD = sprintf('%s_%s_class0_Raw_ODD.mrc', ... -% cycleNumber,emc.('subTomoMeta')); -% refNameEVE = sprintf('%s_%s_class0_Raw_EVE.mrc', ... -% cycleNumber,emc.('subTomoMeta')); -% -% refODD = getVolume(MRCImage(refNameODD)); -% refEVE = getVolume(MRCImage(refNameEVE)); -% catch -% error('\nDid not find either %s or %s\n',refNameODD,refNameEVE) -% end -% end try conserveDiskSpace = emc.('conserveDiskSpace'); diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index bd2416dd..5b370576 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -120,9 +120,10 @@ switch STAGEofALIGNMENT case 'RawAlignment' - fieldPrefix = 'Raw'; - savePrefix = 'REF'; + fieldPrefix = 'Ref'; + savePrefix = 'Ref'; + % FIXME: put these into parser, change to deprecated and use Ref classVector{1} = emc.(sprintf('%s_classes_odd','Raw')); classVector{2} = emc.(sprintf('%s_classes_eve','Raw')); @@ -146,8 +147,8 @@ % Special case for the final cycle. % Assuming RawAlignment already run for this cycle and FSC is calculated % Goal is to re-extract odd-half, applying the xform found in fscGold - fieldPrefix = 'Raw'; - savePrefix = 'REF'; + fieldPrefix = 'Ref'; + savePrefix = 'Ref'; classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); @@ -159,20 +160,10 @@ % "global" or whatever requested class iRefPrev = 1; - - try - - aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','REF',iRefPrev)); - oddRot = reshape(aliParams(1,:),3,3); - % refine the translation per particle. - catch - fprintf('\nReverting from %s to Raw in loading fitFSC\n','REF'); - - aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Raw',iRefPrev)); - - oddRot = reshape(aliParams(1,:),3,3); - % refine the translation per particle. - end + aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)); + oddRot = reshape(aliParams(1,:),3,3); + % refine the translation per particle. + clear iRefPrev @@ -197,8 +188,8 @@ case 'SnrEstimate' - fieldPrefix = 'Raw'; - savePrefix = 'REF'; + fieldPrefix = 'Ref'; + savePrefix = 'Ref'; classVector{1} = [1:25;ones(1,25)]; classVector{2} = [1:25;ones(1,25)]; @@ -603,17 +594,17 @@ end - if (emc.flgCCCcutoff > 1.0) + if (emc.ccc_cutoff > 1.0) sorted_ccc = sort(cccVect); - reqVol = int32(round(cccCutOff)) + reqVol = int32(round(emc.ccc_cutoff)) length(sorted_ccc) - reqVol - emc.flgCCCcutoff = sorted_ccc(length(sorted_ccc) - reqVol); - fprintf('Removing all volumes with score < %2.2f to return the requested %d volumes\n\n',cccCutOff,reqVol); - elseif (emc.flgCCCcutoff > 0.0) + emc.ccc_cutoff = sorted_ccc(length(sorted_ccc) - reqVol); + fprintf('Removing all volumes with score < %2.2f to return the requested %d volumes\n\n',emc.ccc_cutoff,reqVol); + elseif (emc.ccc_cutoff > 0.0) sorted_ccc = sort(cccVect); - reqVol = cccCutoff; - emc.flgCCCcutoff = sorted_ccc(floor(length(cccVect).*(1 - reqVol))); - fprintf('Removing all volumes with score < %2.2f to return the requested percent %2.2f of possible volumes\n\n',cccCutOff,reqVol); + reqVol = emc.ccc_cutoff; + emc.ccc_cutoff = sorted_ccc(floor(length(cccVect).*(1 - reqVol))); + fprintf('Removing all volumes with score < %2.2f to return the requested percent %2.2f of possible volumes\n\n',emc.ccc_cutoff,reqVol); end subTomoMeta.(cycleNumber).('score_sigma') = std(cccVect); @@ -798,12 +789,12 @@ if ( flgEstSNR ) % When the class is for estimating SNR - includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... + includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= emc.ccc_cutoff,2) & ... positionList(:,10) == iClassIDX & ... positionList(:,7) == iGold ); else % When the class is from statistical analysis - includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... + includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= emc.ccc_cutoff,2) & ... positionList(:,26) == iClassIDX & ... positionList(:,7) == iGold ); end @@ -811,7 +802,7 @@ else % if class is 0, pick all non-ignored particles - includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= cccCutOff,2) & ... + includeList = ( any(abs(positionList(:,1:26:26*emc.nPeaks)) >= emc.ccc_cutoff,2) & ... any(positionList(:,26:26:26*emc.nPeaks) ~= -9999,2) & ... positionList(:,7) == iGold ); @@ -877,7 +868,7 @@ TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); if (make_sf3d) - [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, wiener_constant); + [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, emc.wiener_constant); make_sf3d = false; end @@ -923,10 +914,9 @@ sizeWindow, maskRadius, center); end - + if ~ischar(indVAL) - - + if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) try particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); @@ -1419,16 +1409,16 @@ [montOUT, imgLocations] = BH_montage4d(classSum(iGold), ''); imout = sprintf('%s_class%d_%s_%s_NoWgt.mrc',outputPrefix, ... - saveClassSum, 'Raw', halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,'Raw', halfSet); + saveClassSum, savePrefix, halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_NoWgt', saveClassSum,savePrefix, halfSet); SAVE_IMG(montOUT, imout,emc.pixel_size_angstroms); subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; [montOUT, imgLocations] = BH_montage4d(classWgtSum(iGold), ''); imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... - saveClassSum, 'Raw', halfSet); - classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum,'Raw', halfSet); + saveClassSum, savePrefix, halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum, savePrefix, halfSet); SAVE_IMG(montOUT, imout,emc.pixel_size_angstroms); subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; @@ -1453,7 +1443,7 @@ -fprintf('Total execution time : %f seconds\n', datetime("now")-startTime); +fprintf('Total execution time : %f seconds\n', seconds(datetime("now")-startTime)); % clean everything up, since this function is called from other functions. @@ -1528,7 +1518,7 @@ for iRef = 1:nClassesReWgt fprintf('Stage of alignment %s\niRef %d\n',STAGEofALIGNMENT,iRef); if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - savePrefix = 'Raw' + savePrefix = 'Ref' else savePrefix = fieldPrefix end @@ -1622,7 +1612,7 @@ subTomoMeta = masterTM; - subTomoMeta.('CUTPADDING') = CUTPADDING; + subTomoMeta.('CUTPADDING') = emc.CUTPADDING; if (emc.flgCutOutVolumes && volumesNeedToBeExtracted) subTomoMeta.('volumesAreCutOut') = 1; end From 8d32303c68925d6f81942245d80b805813f73f48 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jan 2024 16:34:02 -0500 Subject: [PATCH 018/151] Align, Avg for classification seem to be working, pca with single ref is running. --- metaData/BH_removeDuplicates.m | 11 ++++------- statistics/BH_fscGold_class.m | 4 ++-- statistics/BH_pcaPub.m | 31 +++++++++++++++++++++---------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index b7de1de1..80a95e88 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -44,7 +44,6 @@ system(sprintf('cp %s.mat preDupRemoval_%s.mat',emc.('subTomoMeta'),emc.('subTomoMeta'))); load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); geometry = subTomoMeta.(cycleNumber).RawAlign; -masterTM = subTomoMeta; clear subTomoMeta % Get the number of tomograms to process. @@ -60,11 +59,11 @@ nTotal = nTotal + length(includeList); - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; tomoName = sprintf('%s_%d',tiltName,tomoNumber); - recGeom = masterTM.reconGeometry.(tomoName); + recGeom = subTomoMeta.reconGeometry.(tomoName); % iHeader = getHeader(MRCImage(tomoName)); clear postionMatrix positionIDX @@ -101,7 +100,6 @@ % Positions inbetween particle origins will also be non-zero, so restrict % search to be particle origins that are within radius. duplicateList = find( (positionMatrix) & (overlapMatrix > 1) ); - length(duplicateList) for iDup = duplicateList' % get the positions within radius, get corresponding particle ids, find @@ -135,7 +133,7 @@ end catch - iDup + iDup; end @@ -157,7 +155,6 @@ fprintf('%d of %d particles removed\n', nRemoved, nTotal); -subTomoMeta = masterTM; subTomoMeta.(cycleNumber).RawAlign = geometry; save(emc.('subTomoMeta'), 'subTomoMeta'); diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 2a9d8d91..d1ba53fe 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -111,8 +111,8 @@ case 'RawAlignment' if (emc.classification) % FIXME: This won't be correct for multi-ref alignment if testing out the multi-ref classifications - fieldPrefix = 'Cls'; - savePrefix = 'Cls'; + fieldPrefix = 'Ref'; + savePrefix = 'Ref'; className = 0; classVector = [0;1]; else diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 6e171798..74e67b2f 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -89,7 +89,7 @@ % use the mechanism in place to handle multiple references at different length scales derived from the global average, % to instead be used for multiple distinct classes. If the results are promising, then expand so each ref may also be % looked at over its own scale space. -test_multi_ref_diffmap = true; +test_multi_ref_diffmap = false; test_scale_space_bug_fix = false; startTime = datetime("now"); @@ -333,7 +333,7 @@ - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet); + imgNAME = sprintf('class_%d_Locations_Ref_%s', refName, halfSet); [ averageMotif{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... @@ -358,15 +358,26 @@ error('When combining half sets, the number of references must match') end size(averageMotif) - for iRef = 1:nReferences(1) - averageMotif{1}{iRef} = averageMotif{2}{iRef} + ... - BH_resample3d(gather(averageMotif{1}{iRef}), ... + if (nReferences(1) > 1) + for iRef = 1:nReferences(1) + averageMotif{1}{iRef} = averageMotif{2}{iRef} + ... + BH_resample3d(gather(averageMotif{1}{iRef}), ... + oddRot, ... + aliParams(2,1:3), ... + {'Bah',1,'spline'}, 'cpu', ... + 'forward'); + averageMotif{2}{iRef} = []; + end + else + averageMotif{1} = averageMotif{2} + ... + BH_resample3d(gather(averageMotif{1}), ... oddRot, ... aliParams(2,1:3), ... {'Bah',1,'spline'}, 'cpu', ... 'forward'); - averageMotif{2}{iRef} = []; + averageMotif{2} = []; end + end %%% incomplete, the idea is to generate an antialiased scaled volume for PCA @@ -446,7 +457,7 @@ % include when sets are left 100% separate. % volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, ... - {'pca', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', shape_mask_threshold}); + {'pca', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); end @@ -577,14 +588,14 @@ if (PREVIOUS_PCA) previousPCA = sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet); emc.Pca_randSubset = -1; - [ geometry, nTOTAL, nSUBSET ] = BH_emc.Pca_randSubset( geometry,'pca', -1 , randSet); + [ geometry, nTOTAL, nSUBSET ] = BH_randomSubset( geometry,'pca', -1 , randSet); else if (emc.Pca_randSubset) previousPCA = false; - [ geometry, nTOTAL, nSUBSET ] = BH_emc.Pca_randSubset( geometry,'pca', emc.Pca_randSubset, randSet ); + [ geometry, nTOTAL, nSUBSET ] = BH_randomSubset( geometry,'pca', emc.Pca_randSubset, randSet ); else previousPCA = false; - [ geometry, nTOTAL, nSUBSET ] = BH_emc.Pca_randSubset( geometry,'pca', -1 , randSet); + [ geometry, nTOTAL, nSUBSET ] = BH_randomSubset( geometry,'pca', -1 , randSet); end end From a65b243aa4b55f9c2120189f33480dbb8478008c Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 30 Jan 2024 07:28:42 -0500 Subject: [PATCH 019/151] wip --- alignment/BH_alignRaw3d_v2.m | 38 +---- ctf/BH_ctf_Correct3d.m | 8 +- ctf/BH_ctf_Estimate.m | 5 +- metaData/BH_parseParameterFile.m | 174 +++++++++++++++++++++- metaData/BH_rawAlignmentsApply.m | 2 +- statistics/BH_clusterPub.m | 4 +- statistics/BH_pcaPub.m | 25 ++-- synthetic/BH_synthetic_mapBack.m | 242 +++++++------------------------ synthetic/BH_to_cisTEM_mapBack.m | 5 + testScripts/mCompile.sh | 4 +- transformations/BH_average3d.m | 16 +- 11 files changed, 267 insertions(+), 256 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 48cd83bb..3e0e14ea 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -12,6 +12,8 @@ % TODO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +force_no_symmetry = false; + global bh_global_print_shifts_in_particle_basis; if isempty(bh_global_print_shifts_in_particle_basis) bh_global_print_shifts_in_particle_basis = true; @@ -1277,16 +1279,6 @@ if alignLoop == 1 - % use transpose of RotMat - % try - % iRotRef = BH_resample3d(ref_FT2_tmp{half_set}{rRef}, RotMat', ... - % rXYZ, {rotConvention ,1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); - % catch - % cccPreRefineSort(1,1) - % end - % iRotWdg = BH_resample3d(ref_WGT_rot{half_set}{rRef}, RotMat', ... - % [0,0,0], {rotConvention ,1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - % [ iRotRef ] = refInterpolator.interp3d(... RotMat',... @@ -1358,19 +1350,14 @@ try if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); - - else bestRotPeak = cccPreRefineSort(1,:); bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); - - end catch fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); cccStorageRefine{iPeak}(iSubTomo,:) cccPreRefineSort(1,:) - end finalRef = bestRotPeak(1,1); @@ -1397,17 +1384,10 @@ %%% 2016-11-11 estPeakCoord should have been finalrXYZest in %%% the last writing, but now switching to zeros try - % iRotRef = BH_resample3d(ref_FT2_tmp{half_set}{finalRef}, RotMat', ... - % finalrXYZest, {rotConvention ,1,'linear',1,volBinary_tmp}, 'GPU', 'forward',inputVectors); - % iRotWdg = BH_resample3d(ref_WGT_rot{half_set}{finalRef}, RotMat', ... - % [0,0,0], {rotConvention ,1,'linear',1,wdgBinary_tmp}, 'GPU', 'forward',inputWgtVectors); - [ iRotRef ] = refInterpolator.interp3d(... RotMat',... finalrXYZest,rotConvention ,... 'forward','C1'); - - [ iRotWdg ] = refWdgInterpolator.interp3d(... RotMat',... [0,0,0],rotConvention ,... @@ -1426,13 +1406,6 @@ end - - % iRotRef = ... - % iRotRef(padWindow(1,1) + 1:end - padWindow(2,1) , ... - % padWindow(1,2) + 1:end - padWindow(2,2) , ... - % padWindow(1,3) + 1:end - padWindow(2,3) ); - - iRotRef = BH_bandLimitCenterNormalize(... iRotRef,... bandpassFiltREF_tmp{finalRef} ,'',... @@ -1449,13 +1422,6 @@ iRotMask, peakCOM); - - - % % % end - - - - % Subtract shiftVAL since this is due to windowing, not the actual % position. cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index b9e571db..dcaf3bf1 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -71,13 +71,15 @@ loadSubTomoMeta = true; flgWhitenPS = [0,0,0.0]; if nargin > 2 - if ~isempty(EMC_str2double(varargin{1})) + if isempty(EMC_str2double(varargin{1})) + error('Extra argument to ctf 3d should be a vector [THICKNESS, BINNING] tiltN, or a string templateSearch'); + else reconstructionParameters = EMC_str2double(varargin{1}); recWithoutMat = true; if length(varargin) > 2 % Full recon for tomoCPR - bh_global_turn_on_phase_plate = 0; - filterProjectionsForTomoCPRBackground = 28; + bh_global_turn_on_phase_plate = varargin{3}; + filterProjectionsForTomoCPRBackground = varargin{4}; else loadSubTomoMeta = false; % Default to on for subregion picking diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 5a36b95e..0bc97d33 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -1207,9 +1207,10 @@ std2(bgSubPS(bandpass)) ); end + end -function [ diagnosticIMG ] = make_diagnosticIMG( Hqz, emc.pixel_size_si, bandpass, bg, IMG) +function [ diagnosticIMG ] = make_diagnosticIMG( Hqz, pixel_size_si, bandpass, bg, IMG) iImg = 1; @@ -1219,7 +1220,7 @@ BH_multi_gridCoordinates(size(Hqz),'Cartesian',... 'GPU',{'none'},1,0,1); -radialGrid = radialGrid ./ emc.pixel_size_si; +radialGrid = radialGrid ./ pixel_size_si; lowCut = radialGrid(1, find(bandpass , 1,'first')); highCut= radialGrid(1, find(bandpass , 1,'last')); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index d1fe6ee0..61f5e789 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -288,7 +288,9 @@ emc.Pca_randSubset = 0; end -clusterVector= emc.('Pca_clusters'); +if ~isfield(emc, 'Pca_clusters'); + error('Pca_clusters is a required parameter'); +end % Allowed values are validated inside BH_clusterPub.m if ~isfield(emc, 'Pca_distMeasure'); @@ -297,6 +299,7 @@ if isfield(emc, 'Pca_nReplicates'); EMC_assert_numeric(emc.Pca_nReplicates, 1, [100, 1000]); +else emc.n_replicates = 256; end @@ -354,5 +357,174 @@ end +%%%%%%%%%%%%%%%%%%%%%%%%%% tomoCPR params, mostly experimental + +emc = EMC_assert_deprecated_substitution(emc, false, 'save_mapback_classes', 'flgColorMap'); +EMC_assert_boolean(emc.save_mapback_classes); + + +% These seemed to be necessary at some point to translate between IMOD and emClarity +% coordinate systems, but the should probably be looked at again. TODO: +if isfield(emc, 'flgPreShift') + EMC_assert_numeric(emc.flgPreShift, 3); +else + emc.flgPreShift = [-0.5,-0.5,0.5]; +end + + +% These seemed to be necessary at some point to translate between IMOD and emClarity +% coordinate systems, but the should probably be looked at again. TODO: +if isfield(emc, 'flgPostShift') + EMC_assert_numeric(emc.flgPostShift, 2); +else + emc.flgPostShift = [-0.5,-0.5]; +end + +if isfield(emc, 'prjVectorShift') + EMC_assert_numeric(emc.prjVectorShift, 3); +else + emc.prjVectorShift = [0.5,0.5,1.0]'; +end + +if isfield(emc,'pixelShift') + EMC_assert_numeric(emc.pixelShift, 1); +else + emc.pixelShift = -1; +end + +if isfield(emc, 'pixelMultiplier') + EMC_assert_numeric(emc.pixelMultiplier, 1); +else + emc.pixelMultiplier = 1; +end + +if isfield(emc, 'tomoCPR_random_subset') + EMC_assert_numeric(emc.tomoCPR_random_subset, 1); +else + emc.tomoCPR_random_subset = -1; +end + +% I think this has been removed +if isfield(emc, 'probabilityPeakiness') + EMC_assert_numeric(emc.probabilityPeakiness, 1); +else + emc.probabilityPeakiness = 0; +end + +if isfield(emc, 'whitenProjections') + EMC_assert_numeric(emc.whitenProjections, 1); +else + emc.whitenProjections = 0; +end + +if isfield(emc, 'rot_option_global') + EMC_assert_numeric(emc.rot_option_global, 1); +else + emc.rot_option_global = 1; +end + + +if isfield(emc, 'rot_option_local') + EMC_assert_numeric(emc.rot_option_local, 1); +else + emc.rot_option_local = 1; +end + +if isfield(emc, 'rot_default_grouping_global') + EMC_assert_numeric(emc.rot_default_grouping_global, 1); +else + emc.rot_default_grouping_global = 3; +end + +if isfield(emc, 'rot_default_grouping_local') + EMC_assert_numeric(emc.rot_default_grouping_local, 1); +else + emc.rot_default_grouping_local = 3; +end + +if isfield(emc, 'mag_option_global') + EMC_assert_numeric(emc.mag_option_global, 1); +else + emc.mag_option_global = 1; +end + +if isfield(emc, 'mag_option_local') + EMC_assert_numeric(emc.mag_option_local, 1); +else + emc.mag_option_local = 1; +end + +if isfield(emc, 'mag_default_grouping_global') + EMC_assert_numeric(emc.mag_default_grouping_global, 1); +else + emc.mag_default_grouping_global = 5; +end + +if isfield(emc, 'mag_default_grouping_local') + EMC_assert_numeric(emc.mag_default_grouping_local, 1); +else + emc.mag_default_grouping_local = 5; +end + +if isfield(emc, 'tilt_option_global') + EMC_assert_numeric(emc.tilt_option_global, 1); +else + emc.tilt_option_global = 5; +end + +if isfield(emc, 'tilt_option_local') + EMC_assert_numeric(emc.tilt_option_local, 1); +else + emc.tilt_option_local = 5; +end + +if isfield(emc, 'tilt_default_grouping_global') + EMC_assert_numeric(emc.tilt_default_grouping_global, 1); +else + emc.tilt_default_grouping_global = 5; +end + +if isfield(emc, 'tilt_default_grouping_local') + EMC_assert_numeric(emc.tilt_default_grouping_local, 1); +else + emc.tilt_default_grouping_local = 5; +end + + +if isfield(emc, 'peak_mask_fraction') + EMC_assert_numeric(emc.flgPeakMask,1); +else + emc.peak_mask_fraction = 0.4; +end + +if isfield(emc, 'min_overlap') + EMC_assert_numeric(emc.min_overlap,1); +else + emc.min_overlap = 0.5; +end + + +if isfield(emc, 'k_factor_scaling') + EMC_assert_numeric(emc.k_factor_scaling,1); +else + emc.k_factor_scaling = nan; +end + +if isfield(emc, 'shift_z_to_to_centroid') + EMC_assert_boolean(emc.shift_z_to_to_centroid); +else + emc.shift_z_to_to_centroid = true; +end + +emc = EMC_assert_deprecated_substitution(emc, 500e-9, 'tomo_cpr_defocus_range', 'tomoCprDefocusRange'); +EMC_assert_numeric(emc.tomo_cpr_defocus_range, 1, [0.0, 10000e-9]); + +emc = EMC_assert_deprecated_substitution(emc, 100e-9, 'tomo_cpr_defocus_step', 'tomoCprDefocusStep'); +EMC_assert_numeric(emc.tomo_cpr_defocus_step, 1, [1.0e-9, 10000e-9]); + +emc = EMC_assert_deprecated_substitution(emc, false, 'tomo_cpr_defocus_refine', 'calcCTF'); +EMC_assert_boolean(emc.tomo_cpr_defocus_refine); + + end diff --git a/metaData/BH_rawAlignmentsApply.m b/metaData/BH_rawAlignmentsApply.m index 877ac3bd..9c442754 100755 --- a/metaData/BH_rawAlignmentsApply.m +++ b/metaData/BH_rawAlignmentsApply.m @@ -61,7 +61,7 @@ if ~isempty(newAlignment) for iPeak = 1:nPeaks - includeList = find(positionList(:,26 + 26*(iPeak-1)) ~= -9999) + includeList = find(positionList(:,26 + 26*(iPeak-1)) ~= -9999); for iParticle = includeList' % assuming all classes are sequential, only discarded between cycles. diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index 95000193..11338fab 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -173,8 +173,8 @@ end % Set here so other algs have access sumd = 0; - for iCluster = 1:length(clusterVector) - nClusters = clusterVector(iCluster); + for iCluster = 1:length(emc.Pca_clusters) + nClusters = emc.Pca_clusters(iCluster); if strcmpi(kAlgorithm, 'kMeans') diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 74e67b2f..0afe5f7a 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -90,7 +90,6 @@ % to instead be used for multiple distinct classes. If the results are promising, then expand so each ref may also be % looked at over its own scale space. test_multi_ref_diffmap = false; -test_scale_space_bug_fix = false; startTime = datetime("now"); @@ -504,14 +503,18 @@ % radius, convert Ang to pix , denom = equiv stdv from normal to include, e.g. % for 95% use 1/sig = 1/2 %stdDev = 1/2 .* (emc.pca_scale_spaces ./ pixelSize - 1) .* 3.0./log(emc.pca_scale_spaces) -threeSigma = 1/3 .* (emc.pca_scale_spaces ./ pixelSize) +threeSigma = 1/3 .* (emc.pca_scale_spaces ./ pixelSize); for iScale = 1:emc.n_scale_spaces - kernelSize = ceil(threeSigma(iScale)) + 3; + kernelSize = ceil(threeSigma(iScale).*3) + 3; kernelSize = kernelSize + (1-mod(kernelSize,2)); - masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); + % masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); + masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'gpu', {}); + + + % SAVE_IMG( masks.('scaleMask').(sprintf('s%d',iScale)), ... + % sprintf('%s_scaleMask_s%d.mrc',outputPrefix,iScale),pixelSize); - masks.('scaleMask').(sprintf('s%d',iScale)) end @@ -540,14 +543,7 @@ BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)),... [0,0,0;0,0,0],'single'); - % This reproduces the orginal behavior, which wrote over averageMotif. This is a bug, but who knows, it may be beneficial, so lets for now make it optional. - if (test_scale_space_bug_fix) - if (test_multi_ref_diffmap) - averageMotif{iGold}{iScale} = tmp_avg; - else - averageMotif{iGold} = tmp_avg; - end - end + avgFiltered{iGold, iScale} = real(ifftn(avgMotif_FT{iGold, iScale})); avgFiltered{iGold, iScale} = avgFiltered{iGold, iScale} - mean(avgFiltered{iGold, iScale}(masks.('binary').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); @@ -559,11 +555,10 @@ montOUT = BH_montage4d(avgFiltered(1,:),''); -SAVE_IMG(MRCImage(montOUT), sprintf('test_filt.mrc'),pixelSize); +SAVE_IMG(MRCImage(montOUT), sprintf('%s_filt.mrc', outputPrefix),pixelSize); clear montOUT - % If emc.Pca_randSubset is string with a previous matfile use this, without any % decomposition. diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index fbea4bd1..27e2179a 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -20,44 +20,6 @@ emc = BH_parseParameterFile(PARAMETER_FILE); -try - flgColorMap = emc.('flgColorMap'); -catch - flgColorMap = 0; -end - -try - preShift = emc.('preShift'); -catch - preShift = [-0.5,-0.5,0.5]; -end - -try - postShift = emc.('postShift'); -catch - postShift = [-0.5,-0.5]; -end - -try - prjVectorShift = emc.('prjVectorShift')'; -catch - prjVectorShift = [0.5,0.5,1.0]'; -end - -try - pixelShift = emc.('pixelShift'); -catch - pixelShift = -1; -end - -try - pixelMultiplier = emc.('pixelMultiplier'); -catch - pixelMultiplier = 1; -end - - - CYCLE = EMC_str2double(CYCLE); cycle_numerator = ''; cycle_denominator =''; @@ -92,11 +54,7 @@ MOL_MASS = emc.('particleMass'); molMass = MOL_MASS.*(25/samplingRate); -try - tomoCPR_random_subset = emc.('tomoCPR_randomSubset') -catch - tomoCPR_random_subset = -1 -end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%%%%%%%%%% Parameters I am currently experimenting with as of Jan 2018 @@ -112,26 +70,15 @@ rmsScale = sqrt(MOL_MASS); end -try - probabilityPeakiness = emc.('probPeakiness'); -catch - probabilityPeakiness = 0; -end try - useAverageDefocus = emc.('useAverageDefocus'); + nFiducialsPerPatch = emc.('n_fiducials_per_patch'); catch - % While it seems like using the per fiducial defocus max for the - % refinement makes sense, have the default be the average of the - % projection - useAverageDefocus = 0; + % TODO how smooth should the solutions really be - should multiple + % results be run and compared? + nFiducialsPerPatch = ceil(100./sqrt(molMass)); end -try - whitenProjections = emc.('whitenProjections'); -catch - whitenProjections = 0; -end % Used to calc defocus values using tilt instead of manually. Convention % diff. @@ -140,98 +87,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% %%% Playing around with the model n_surfaces=2; -try - rot_option_global = emc.('rot_option_global'); -catch - rot_option_global = 1; -end -try - rot_option_local = emc.('rot_option_local'); -catch - rot_option_local = 1; -end -try - rot_default_grouping_global = emc.('rot_default_grouping_global'); -catch - rot_default_grouping_global = 3; -end -try - rot_default_grouping_local = emc.('rot_default_grouping_local'); -catch - rot_default_grouping_local = 3; -end -try - mag_option_global = emc.('mag_option_global'); -catch - mag_option_global = 1; -end -try - mag_option_local = emc.('mag_option_local'); -catch - mag_option_local = 1; -end -try - mag_default_grouping_global = emc.('mag_default_grouping_global'); -catch - mag_default_grouping_global = 5; -end -try - mag_default_grouping_local = emc.('mag_default_grouping_local'); -catch - mag_default_grouping_local = 5; -end -try - tilt_option_global = emc.('tilt_option_global'); -catch - tilt_option_global = 5; -end -try - tilt_option_local = emc.('tilt_option_local'); -catch - tilt_option_local = 5; -end -try - tilt_default_grouping_global = emc.('tilt_default_grouping_global'); -catch - tilt_default_grouping_global = 5; -end -try - tilt_default_grouping_local = emc.('tilt_default_grouping_local'); -catch - tilt_default_grouping_local = 5; -end -try - nFiducialsPerPatch = emc.('n_fiducials_per_patch'); -catch - % TODO how smooth should the solutions really be - should multiple - % results be run and compared? - nFiducialsPerPatch = ceil(100./sqrt(molMass)); -end -try - target_patch_size = emc.('target_patch_size'); -catch - target_patch_size = 500; -end -try - peak_mask_fraction = emc.('peak_mask_fraction'); -catch - peak_mask_fraction = 0.4; -end -try - min_overlap = emc.('min_overlap'); -catch - min_overlap = 0.5; -end -try - k_factor_scaling = emc.('k_factor_scaling'); -catch - k_factor_scaling = nan; -end -try - shift_z_to_to_centroid = emc.('shift_z_to_to_centroid'); -catch - shift_z_to_to_centroid = true; -end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -374,7 +230,7 @@ end - imgNAME = sprintf('class_%d_Locations_REF_%s', refName, halfSet) + imgNAME = sprintf('class_%d_Locations_Ref_%s', refName, halfSet) iHeader = getHeader(MRCImage(subTomoMeta.(cycleNumber).(imgNAME){1},0)); sizeWindow = iHeader.nZ.*[1,1,1]; @@ -495,7 +351,7 @@ %PARTICLE_RADIUS = floor(mean(emc.('particleRadius')./pixelSize)); % TODO, is this too restricted? - peakSearchRad = floor(peak_mask_fraction*PARTICLE_RADIUS.*[1,1]); + peakSearchRad = floor(emc.peak_mask_fraction*PARTICLE_RADIUS.*[1,1]); try lowPassCutoff = emc.('tomoCprLowPass'); fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n.',... @@ -630,7 +486,7 @@ avgColor = zeros(reconstructionSize, 'int16'); end - if (flgColorMap) + if (emc.save_mapback_classes) avgColor = zeros(reconstructionSize, 'int16'); end @@ -663,8 +519,11 @@ backgroundName = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec',CWD,tiltNameList{iTiltSeries},1, samplingRate); - % emClarity('internal','ctf','3d',PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries},'dummy'); - BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries},'dummy'); + + % TODO: investigate deviations from the default, which is to shut off the phakePhasePlate and to use a backgroundLowPassResolution of 28 + send_phakePhasePlateOption = [0,0]; + send_backgroundLowPassResolution = 28; + BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution); % re-initialize the parpool for each tilt series to free up mem. delete(gcp('nocreate')) @@ -699,7 +558,7 @@ - if (flgColorMap) + if (emc.save_mapback_classes) avgColor = zeros(reconstructionSize, 'int16'); end @@ -815,7 +674,7 @@ % TODO need to update this. - if (flgColorMap) + if (emc.save_mapback_classes) colorMap = single(getVolume(MRCImage(COLOR_MAP))); % should be the same size as the average @@ -891,7 +750,7 @@ % % % prjVector = prjVector + [0.5,0.0,-0.5]; % prjVector = prjVector + [0.0,0.0,1.0]; - prjVector = prjVector - preShift; + prjVector = prjVector - emc.flgPreShift; recVector = (originPrj + [0,0,ceil((reconstructionSize(3)+1)/2)] + prjVector); % subTomo origin relative to reconLowerLeft %Resample a copy of the average to match the position in the tomogram @@ -920,8 +779,8 @@ iAvgResamp = gather(iMaskResamp.*iAvgResamp); - if (flgColorMap || flgClassAvg) - if (flgColorMap) + if (emc.save_mapback_classes || flgClassAvg) + if (emc.save_mapback_classes) iColorMap = gather(int16(iMaskResamp.* BH_resample3d(colorMap, ... rSubTomo',shiftVAL,'Bah',METHOD,'forward'))); else @@ -973,7 +832,7 @@ % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot*prjVector' + [originRec(1),originRec(3),originRec(2)]'- prjVectorShift([1,3,2]), fidIDX); + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot*prjVector' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2]), fidIDX); for iPrj = 1:nPrjs @@ -1033,7 +892,7 @@ end clear avgTomo - if (flgColorMap || flgClassAvg) + if (emc.save_mapback_classes || flgClassAvg) SAVE_IMG(MRCImage(gather(avgColor)),sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),4.0); clear avgColor end @@ -1050,7 +909,7 @@ % tmpTomoBin,mbOUT{1:3},iSave,mbOUT{1:3},tmpTomoBin,iSave)); % end tmpTomoBin = 2; - if (flgColorMap || flgClassAvg) + if (emc.save_mapback_classes || flgClassAvg) system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomoColor ',... '%smapBack%d/%s.bin%dTomoColor.mrc'], ... tmpTomoBin,mbOUT{1:3},mbOUT{1:3},tmpTomoBin)); @@ -1078,7 +937,7 @@ - % % % % if (flgColorMap) + % % % % if (emc.save_mapback_classes) % % % % SAVE_IMG(MRCImage(avgColor),sprintf('mapBack/%s_colorMap.mrc',tiltBaseName)); % % % % % -90 is assumed for trim vol, so if rotate vol is used add 90 % % % % if (rotateVol) @@ -1365,7 +1224,7 @@ % Need to shift again from the model coordinate system - fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(prjVectorShift(1:2)', size(fidList,1),1); + fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(emc.prjVectorShift(1:2)', size(fidList,1),1); foundNans = sum(isnan(fidList(:,3))); if (foundNans) fprintf('\n\t\tThere are %d NaNs in the projected fiducial list %3.3f\n\n',foundNans, foundNans/size(fidList,1)*100); @@ -1453,15 +1312,15 @@ nUniqueFids = numel(unique(fidList(:,2))); % I think the max val of this column should also be okay (+1) nFidsTotal = nUniqueFids; % nFidsTotal = sum(fidList(:,5) == 1 ); - if tomoCPR_random_subset == -1 || tomoCPR_random_subset > nUniqueFids + if emc.tomoCPR_random_subset == -1 || emc.tomoCPR_random_subset > nUniqueFids fprintf('Using all of the %d available fiducials\n',nUniqueFids); else fprintf('Using a random subset of %d fiducials from the %d available\n',... - tomoCPR_random_subset, nUniqueFids); + emc.tomoCPR_random_subset, nUniqueFids); - keepFids = datasample(0:nUniqueFids-1,tomoCPR_random_subset,'Replace',false); + keepFids = datasample(0:nUniqueFids-1,emc.tomoCPR_random_subset,'Replace',false); fidList(~ismember(fidList(:,2),keepFids),2) = -9999; - nFidsTotal = tomoCPR_random_subset; + nFidsTotal = emc.tomoCPR_random_subset; end @@ -1476,8 +1335,8 @@ %Put back into a natural order TLT = sortrows(TLT,1); - if isnan(k_factor_scaling) - k_factor_scaling = 10 / sqrt(nFidsTotal); + if isnan(emc.k_factor_scaling) + emc.k_factor_scaling = 10 / sqrt(nFidsTotal); end parfor iPrj = 1:nPrjs @@ -1576,7 +1435,7 @@ sRms = rms(dataRMS(:)-mRms); % FIXME - if (whitenProjections) + if (emc.whitenProjections) whitenBP = [2*PARTICLE_RADIUS,lowPassCutoff,pixelSize,PARTICLE_RADIUS]; [dataPrj,NPS] = BH_whitenNoiseSpectrum(dataPrj,'',whitenBP,1); % Create a matched filter. @@ -1684,14 +1543,14 @@ continue end - pixelX = wrkFid(iFid,3) - pixelShift + postShift(1); - pixelY = wrkFid(iFid,4) - pixelShift + postShift(2); + pixelX = wrkFid(iFid,3) - emc.pixelShift + emc.flgPostShift(1); + pixelY = wrkFid(iFid,4) - emc.pixelShift + emc.flgPostShift(2); ox = floor(pixelX) - tileRadius; oy = floor(pixelY) - tileRadius; - sx = pixelMultiplier*(pixelX - floor(pixelX)); - sy = pixelMultiplier*(pixelY - floor(pixelY)); + sx = emc.pixelMultiplier*(pixelX - floor(pixelX)); + sy = emc.pixelMultiplier*(pixelY - floor(pixelY)); % ox = floor(wrkFid(iFid,3)) - tileRadius; % oy = floor(wrkFid(iFid,4)) - tileRadius; @@ -2030,7 +1889,7 @@ % % % % % mbOUT{1:3},outCTF,targetPatchSize,targetPatchSize,... % % % % % nFiducialsPerPatch,floor(nFiducialsPerPatch/3)); - if (shift_z_to_to_centroid) + if (emc.shift_z_to_to_centroid) final_line1 = 'ShiftZFromOriginal'; final_line2 = 'AxisZShift 0.0'; final_line3 = 'LocalOutputOptions 1,1,1'; @@ -2095,18 +1954,25 @@ fullPixelSize,fullPixelSize,... mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... - tilt_script_name,n_surfaces,rot_option_global,... - tilt_option_global,tilt_default_grouping_global,... - mag_option_global,mag_default_grouping_global,... - k_factor_scaling,... - rot_option_local,rot_default_grouping_local,... - tilt_option_local,tilt_default_grouping_local,... - mag_option_local,mag_default_grouping_local, ... - mbOutAlt{1:3},outCTF,targetPatchSize,targetPatchSize,... - nFiducialsPerPatch,floor(nFiducialsPerPatch/3),... - min_overlap,min_overlap,... + tilt_script_name,n_surfaces, ... + emc.rot_option_global, ... + emc.tilt_option_global, ... + emc.tilt_default_grouping_global, ... + emc.mag_option_global, ... + emc.mag_default_grouping_global, ... + emc.k_factor_scaling, ... + emc.rot_option_local, ... + emc.rot_default_grouping_local, ... + emc.tilt_option_local, ... + emc.tilt_default_grouping_local, ... + emc.mag_option_local, ... + emc.mag_default_grouping_local, ... + mbOutAlt{1:3},outCTF,targetPatchSize, ... + targetPatchSize,... + nFiducialsPerPatch, ... + floor(nFiducialsPerPatch/3),... + emc.min_overlap,emc.min_overlap,... final_line1,final_line2,final_line3); - % % % Assume that any backlash was solved well enough that there are no major % % % discontinuities in the coarse alignment. Mag and rot are solved/ tilt in % % % the global solution anyhow, so this shouldn't be a bit deal. diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index b9c76461..3187c58e 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -75,6 +75,11 @@ load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); resForFitting = 1.3*mean(subTomoMeta.currentResForDefocusError); +% % Add error check onrange for reasonable values. +% ctfRange = emc.('tomoCprDefocusRange')*10^10; +% ctfInc = emc.('tomoCprDefocusStep')*10^10; + +% calcCTF = emc.('tomoCprDefocusRefine'); nGPUs = emc.('nGPUs'); pInfo = parcluster(); diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index b4cc727c..e99dae7e 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -43,9 +43,9 @@ outName="$(basename ${mFile} .m)${post}" # For naming. If you are compiling your own version, use something descriptive in teh # bugs line. e.g. buggs=5testingFeature major=1 -minor=7 +minor=8 bugs=0 -nightly=15 +nightly=0 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 5b370576..ac886e4a 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -1455,8 +1455,6 @@ if strcmpi(STAGEofALIGNMENT, 'RawAlignment') BH_fscGold_class(PARAMETER_FILE, num2str(CYCLE), STAGEofALIGNMENT); -else - error('This block should not be reached'); end @@ -1540,10 +1538,16 @@ flgRefCutOff = 0; end - fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRefPrev)); - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); - mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); - + try + fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRefPrev)); + aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); + mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); + catch + fprintf('\nReverting from %s to REf in loading fitFSC\n',savePrefix); + fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d','Ref',iRefPrev)); + aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)); + mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d','Ref',iRefPrev)); + end iOdd = iRef; iEve = iRef; From 7a7517f2301292b6bdc1e0a4d3fd7eb9764ea4ec Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 2 Feb 2024 09:27:24 -0500 Subject: [PATCH 020/151] WIP: Basic update to classification working, needs to be parameterized. --- alignment/BH_alignRaw3d_v2.m | 34 +-- coordinates/BH_multi_angularSearch.m | 424 --------------------------- metaData/BH_parseParameterFile.m | 8 +- statistics/BH_clusterPub.m | 1 + statistics/BH_fscGold_class.m | 4 +- statistics/BH_pcaPub.m | 117 ++++---- testScripts/mCompile.sh | 2 +- transformations/BH_average3d.m | 49 ++-- 8 files changed, 107 insertions(+), 532 deletions(-) delete mode 100755 coordinates/BH_multi_angularSearch.m diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 3e0e14ea..df3585c4 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -12,7 +12,6 @@ % TODO %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -force_no_symmetry = false; global bh_global_print_shifts_in_particle_basis; if isempty(bh_global_print_shifts_in_particle_basis) @@ -593,9 +592,13 @@ fprintf('Starting main loopwith N references %d\n', nReferences(1)); % This may be modified in the parfor loop (tho that prob isn't really necessary) - +particle_symmetry = emc.symmetry; +if (emc.force_no_symmetry) + particle_symmetry = 'C1'; +end parfor iParProc = parVect symmetry = emc.symmetry; + bestAngles_tmp = struct(); geometry_tmp = geometry; @@ -962,7 +965,7 @@ [ iTrimParticle ] = particleInterpolator.interp3d(... RotMat,... estPeakCoord,rotConvention ,... - 'inv',symmetry); + 'inv',particle_symmetry); @@ -971,12 +974,12 @@ [ iTrimInitial ] = particleInterpolator.interp3d(... reshape(angles,3,3),... shiftVAL,rotConvention ,... - 'inv',symmetry); + 'inv',particle_symmetry); [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... reshape(angles,3,3),... [0,0,0],rotConvention ,... - 'inv',symmetry); + 'inv',particle_symmetry); end @@ -985,7 +988,7 @@ [ iWedgeMask ] = imgWdgInterpolator.interp3d(... RotMat,... [0,0,0],rotConvention ,... - 'inv',symmetry); + 'inv',particle_symmetry); end @@ -1250,29 +1253,18 @@ padWindow(1,3) + 1:end - padWindow(2,3) ); case 2 - - if (force_no_symmetry) - symmetry = 'C1'; - end - + [ iTrimParticle ] = particleInterpolator.interp3d(... RotMat,... rXYZ,rotConvention ,... - 'inv',symmetry); + 'inv',particle_symmetry); [ iWedgeMask ] = imgWdgInterpolator.interp3d(... RotMat,... [0,0,0],rotConvention ,... - 'inv',symmetry); - - - - - - - - + 'inv',particle_symmetry); + end diff --git a/coordinates/BH_multi_angularSearch.m b/coordinates/BH_multi_angularSearch.m deleted file mode 100755 index f323e32b..00000000 --- a/coordinates/BH_multi_angularSearch.m +++ /dev/null @@ -1,424 +0,0 @@ -function [ CCC_STORAGE] = BH_multi_angularSearch( ANGLE_STEP, ... - PEAK_LIST, ... - IN_PLANE_SEARCH, ... - iClassImg, iClassWdg, ... - ref_FT, refWDG, ... - refRotAvg_FT, ... - volMask, bandpassFilt, ... - padCalc, padREF,... - peakMask, peakCOM, IDX, ... - refSym) -%UNTITLED Summary of this function goes her -% Detailed explanation goes here - -% Make a normalization factor for wedge weighting. This could be done in the -% begining, but to test I'll put it here so I don't have to change functino I/o - -peakBinary = (peakMask >= 0.01); -volBinary = (volMask >= 0.01); - - - -% If searching the full in plane range, limit based on symmetry. Important for -% wedge bias and also helps with speed. -if IN_PLANE_SEARCH(1) == -180 - limitSymmetry = 1; - fprintf('limiting to symmetry constrained in-plane search.\n') -else - limitSymmetry = 0; -end - - - -iClassTrim = iClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - -%[ iClassImg ] = BH_bandLimitCenterNormalize(unMaskedClassImage, bandpassFilt, volMask); - - - -if length(PEAK_LIST) > 1 - % Search angles based on previously found peaks. - peakList = PEAK_LIST; - - % Either a refinment around top three or top one peaks, or checking all - % inplane angles for top ten axially averaged. - if (size(peakList,2) == 10) - % Refinementd - % Get unique references for peakList wedge weight normalization - referenceList = unique(peakList(:,1)); - nRefs = length(referenceList); - iClassImg2 = cell(nRefs,1); - for iRef = 1:length(bandpassFilt) - iRef - size( iClassTrim) - size(bandpassFilt{iRef}) - [ iClassImg2{iRef} ] = BH_bandLimitCenterNormalize(iClassTrim.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - end - angCount=1; - - nAngles = 46.*size(peakList,1)+1 - pause(4) - cccStorage = zeros(nAngles, 13, 'double', 'gpuArray'); - - for iAngle = 1 : size(peakList,1) - - - iRef = peakList(iAngle, 1); - phi = peakList(iAngle, 2); phiInc = peakList(iAngle,5); - theta= peakList(iAngle, 3); thetaInc = peakList(iAngle,6); - psi = peakList(iAngle, 4); psiInc = peakList(iAngle,7); - - if size(peakList,1) == 1 % This is the final refinement - % superSample = 1; - inPlaneSearch = psi-psiInc : psiInc : psi+psiInc; - polarSearch = theta-thetaInc :thetaInc : theta+thetaInc; - azimuthalSearch= phi-phiInc : phiInc : phi + phiInc; - else - inPlaneSearch = psi-psiInc : psiInc : psi + psiInc; - polarSearch = theta-thetaInc: thetaInc : theta+ thetaInc; - azimuthalSearch= phi-2*phiInc : phiInc : phi + 2*phiInc; - end - - - - - - searchList = zeros(46,3); - nSearch = 1; - for iPhi = azimuthalSearch - for iTheta = polarSearch - for iPsi = inPlaneSearch - searchList(nSearch, :) = [iPhi, iTheta, iPsi]; - nSearch = nSearch + 1; - end - end - end - - - - for iRefine = 1:nSearch-1 - - - RotMat = BH_defineMatrix(searchList(iRefine,:),'Bah', 'forward'); - - - [ rotRef ] = BH_resample3d(ref_FT(:,:,:,iRef),RotMat, ... - peakList(iAngle,8:10), ... - 'Bah', 'GPU', 'forward'); - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - peakList(iAngle,8:10), ... - 'Bah', 'GPU', 'forward')); - else - rotWDG = refWDG; - end - - rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - - - rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef}, volBinary,padCalc,'single'); - rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - - rotRef_FT = conj(rotRef_FT); - rotRef_FT2= conj(rotRef_FT2); - - - % find translational shift using rotationally averaged tightly masked - % reference, to reduce chance of drift to alternate lattice sites. - try - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); - catch - iRef - - - error('sdfsd') - end - - % apply only a tranlational shift to the particle - [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - - rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - [ rotClassImg1 ] = BH_bandLimitCenterNormalize(... - rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,... - padCalc,'single'); - % [ rotClassImg2 ] = BH_bandLimitCenterNormalize(... - % rotClassImg.*peakMask, ... - % bandpassFilt, peakBinary, ... - % padCalc,'single'); - clear rotClassImg - - % now calc CCC, setting sampling shift to zero - [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg, rotWDG, ... - peakMask); - - % [ finalPeakCoord ] = BH_multi_xcf_Translational( ... - % rotClassImg2, rotRef_FT2, ... - % peakMask, peakCOM); - cccStorage(angCount,:) = [iRef, IDX, searchList(iRefine,:), iCCC, ... - iWeight, peakList(iAngle,8:10)+estPeakCoord, ... - phiInc,thetaInc,psiInc]; - - angCount = angCount + 1; - end % end inPlane - end % end search over best peaks - - else - % top ten - % Get unique references for peakList wedge weight normalization - - % referenceList = unique(peakList(:,1)); - % nRefs = length(referenceList); - angCount=1; - - nRefs = size(ref_FT,4); - iClassImg2 = cell(nRefs,1); - for iRef = 1:nRefs - [ iClassImg2{iRef} ] = BH_bandLimitCenterNormalize(iClassTrim.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - end - nAngles = nRefs.*size(peakList,1).*length(IN_PLANE_SEARCH); - cccStorage = zeros(nAngles, 10, 'double', 'gpuArray'); - - for iAngle = 1:size(peakList,1) - % The assumption is the best reference for the axially averaged is - % also the best ref otherwise. Maybe not true. - %iRef = peakList(iAngle, 1); - phi = peakList(iAngle, 2); - theta= peakList(iAngle, 3); - - for iInPlane = IN_PLANE_SEARCH - psi = iInPlane; - - evaluateRef = ones(1,nRefs); - if (limitSymmetry) - evaluateRef = evaluateRef.*((abs(psi).*evaluateRef) < 180 ./ refSym); - end - for iRef = 1:nRefs - if (evaluateRef(iRef)) - RotMat = BH_defineMatrix([phi, theta, psi - phi],'Bah', 'forward'); - - [ rotRef ] = BH_resample3d(ref_FT(:,:,:,iRef), ... - RotMat,peakList(iAngle,4:6), ... - 'Bah', 'GPU', 'forward'); - - - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - peakList(iAngle,4:6), ... - 'Bah', 'GPU', 'forward')); - else - rotWDG = refWDG; - end - - rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - - - rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef} , volBinary,padCalc,'single'); - rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef} , peakBinary,padCalc,'single'); - - rotRef_FT = conj(rotRef_FT); - rotRef_FT2= conj(rotRef_FT2); - - - - - - % find translational shift - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); - - % apply only a tranlational shift to the particle - [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - - rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - [ rotClassImg1 ] = BH_bandLimitCenterNormalize(rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,padCalc,'single'); - % [ rotClassImg2 ] = BH_bandLimitCenterNormalize(rotClassImg.*peakMask, ... - % bandpassFilt{iRef} , peakBinary,padCalc,'single'); - clear rotClassImg - - - % now calc CCC, setting sampling shift to zero - [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg, rotWDG,... - peakMask); - - % [ finalPeakCoord ] = BH_multi_xcf_Translational( ... - % rotClassImg2, rotRef_FT2, ... - % peakMask, peakCOM); - - - cccStorage(angCount,:) = [iRef, IDX, phi, theta, psi - phi, iCCC, ... - iWeight, estPeakCoord + ... - peakList(iAngle,4:6) ]; - angCount = angCount + 1; - end - end - end % end inPlane - end % end search over best peaks - end - -else % search angles based on grideSearchAngles - - referenceList = 1:size(ref_FT,4); - nRefs = length(referenceList); - iClassImg2 = cell(nRefs,1); - - - for iRef = 1:nRefs - - [ iClassImg2{iRef} ] = BH_bandLimitCenterNormalize(iClassTrim.*peakMask, bandpassFilt{iRef}, peakBinary,padCalc,'single'); - end - angCount=1; - nAngles = nRefs.*sum(ANGLE_STEP(:,2)+1).*length(IN_PLANE_SEARCH); - cccStorage = zeros(nAngles, 10, 'double', 'gpuArray'); - for iAngle = 1:size(ANGLE_STEP,1) - - theta = ANGLE_STEP(iAngle,1); - - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. - - phiStep = ANGLE_STEP(iAngle,3); - - % To prevent only searching the same increments each time in a limited - % grid search, radomly offset the azimuthal angle by a random number - % between 0 and 1/2 the azimuthal increment. - azimuthalRandomizer = rand(1)*phiStep/2; - - for iAzimuth = 0:ANGLE_STEP(iAngle,2) - phi = rem(phiStep * (iAzimuth + azimuthalRandomizer),360) ; - - % For axially averaged this is always zero. - for iInPlane = IN_PLANE_SEARCH - psi = iInPlane; - [phi,theta,psi]; - - for iRef = 1:nRefs - - - - RotMat = BH_defineMatrix([phi, theta, psi ],'Bah', 'forward'); - - - [ rotRef ] = BH_resample3d(refRotAvg_FT(:,:,:,iRef),RotMat, ... - [0,0,0], 'Bah', 'GPU', 'forward'); - - - - if isa(refWDG,'cell') - rotWDG = ifftn(BH_resample3d(refWDG{iRef},RotMat, ... - [0,0,0], ... - 'Bah', 'GPU', 'forward')); - else - rotWDG = refWDG; - end - - rotRef = rotRef(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - try - rotRef_FT = ... - BH_bandLimitCenterNormalize(rotRef.*volMask, bandpassFilt{iRef} , volBinary,padCalc,'single'); - rotRef_FT2 = ... - BH_bandLimitCenterNormalize(rotRef.*peakMask, bandpassFilt{iRef} , peakBinary,padCalc,'single'); - catch - size(volMask) - size(peakMask) - size(bandpassFilt{iRef} ) - size(rotRef) - error('size mismatch in multi_angular search') - end - rotRef_FT = conj(rotRef_FT); - rotRef_FT2= conj(rotRef_FT2); - - - % find translational shift - [ estPeakCoord ] = BH_multi_xcf_Translational( ... - iClassImg2{iRef}, rotRef_FT2, ... - peakMask, peakCOM); - - % apply only a tranlational shift to the particle - [ rotClassImg ] = BH_resample3d(iClassImg, [0,0,0], ... - estPeakCoord,'Bah','GPU','inv'); - - rotClassImg = rotClassImg(padREF(1,1)+1 : end - padREF(2,1), ... - padREF(1,2)+1 : end - padREF(2,2), ... - padREF(1,3)+1 : end - padREF(2,3) ); - - - [ rotClassImg1 ] = BH_bandLimitCenterNormalize(rotClassImg.*volMask, ... - bandpassFilt{iRef} , volBinary,padCalc,'single'); - % [ rotClassImg2 ] = BH_bandLimitCenterNormalize(rotClassImg.*peakMask, ... - % bandpassFilt{iRef} , peakBinary,padCalc,'single'); - clear rotClassImg - - - % now calc CCC, setting sampling shift to zero - [ iCCC, iWeight ] = ... - BH_multi_xcf_Rotational( rotClassImg1, rotRef_FT, ... - iClassWdg,rotWDG, ... - peakMask); - - % [ finalPeakCoord ] = BH_multi_xcf_Translational( ... - % rotClassImg2, rotRef_FT2, ... - % peakMask, peakCOM); - % - - cccStorage(angCount,:) = [iRef, IDX, phi, theta, psi, iCCC, ... - iWeight, estPeakCoord]; - - angCount = angCount + 1; - end % peak search over refs - - - end % end inPlane - end % azimuthal - - end % polar - - % end of else clause, which is a search over axially averaged reference. -end - -%[ cccStorage ] = BH_multi_peakSearch(referenceList, cccStorage); -cccStorage = cccStorage(( ~(sum(isnan(cccStorage),2)) ),:); -cccStorage = cccStorage(( cccStorage(:,6) ~= 0 ),:); - -CCC_STORAGE = sortrows(gather(cccStorage),-6); - -clear rotRef rotWDG refWDG rotRef_FT rotRef_FT2 clear rotClassImg1 iClassImg2 -end % end angularSearch function - diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 61f5e789..903627be 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -212,8 +212,8 @@ EMC_assert_boolean(emc.classification); -emc = EMC_assert_deprecated_substitution(emc, false, 'multi_reference_alignment', 'flgMultiRefAlignment'); -EMC_assert_boolean(emc.multi_reference_alignment); +emc = EMC_assert_deprecated_substitution(emc, 0, 'multi_reference_alignment', 'flgMultiRefAlignment'); +EMC_assert_numeric(emc.multi_reference_alignment, 1, [0, 2]); % Zero padding of the volumes before alignment/other FFT ops emc = EMC_assert_deprecated_substitution(emc, 1.5, 'scale_calc_size', 'scaleCalcSize'); @@ -232,11 +232,11 @@ error('force_no_symmetry must be after symmetry check'); end % Warning must be after symmetry check - if (force_no_symmetry) + if (emc.force_no_symmetry) emc.symmetry='C1'; end else - force_no_symmetry = false; + emc.force_no_symmetry = true; end if isfield(emc, 'Pca_constrain_symmetry') diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index 11338fab..a66eed73 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -10,6 +10,7 @@ emc = BH_parseParameterFile(PARAMETER_FILE); +% Only used in PCA pub right now test_multi_ref_diffmap=true; %%% For general release, I've disabled class average alignment and diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index d1ba53fe..6d48054e 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -18,6 +18,8 @@ error('args = PARAMETER_FILE, CYCLE, STAGEofALIGNMENT') end +% FIXME +test_multi_ref_diffmap = true; % Explicit reference to location of variables in main memory, or on the GPU. cpuVar = struct(); @@ -109,7 +111,7 @@ switch STAGEofALIGNMENT case 'RawAlignment' - if (emc.classification) + if (emc.classification && ~(test_multi_ref_diffmap )) % FIXME: This won't be correct for multi-ref alignment if testing out the multi-ref classifications fieldPrefix = 'Ref'; savePrefix = 'Ref'; diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 0afe5f7a..774d4920 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -89,7 +89,7 @@ % use the mechanism in place to handle multiple references at different length scales derived from the global average, % to instead be used for multiple distinct classes. If the results are promising, then expand so each ref may also be % looked at over its own scale space. -test_multi_ref_diffmap = false; +test_multi_ref_diffmap = true; startTime = datetime("now"); @@ -159,7 +159,7 @@ nCores = BH_multi_parallelWorkers(emc.('nCpuCores')); pInfo = parcluster(); -nTempParticles = emc.('PcaGpuPull'); +nParticlesOnGpuBeforeTransfer = emc.('PcaGpuPull'); outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); @@ -229,10 +229,7 @@ CUTPADDING=20 end -% % -% % pathList= subTomoMeta.mapPath; -% % extList = subTomoMeta.mapExt; -masterTM = subTomoMeta; clear subTomoMeta + [ useGPU ] = BH_multi_checkGPU( -1 ); gDev = gpuDevice(useGPU); @@ -277,7 +274,7 @@ else iRefPrev = 1; - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)) + aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)) oddRot = reshape(aliParams(1,:),3,3)'; % refine the translation per particle. @@ -336,8 +333,8 @@ [ averageMotif{iGold} ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... + subTomoMeta.(cycleNumber).(imgNAME){1}, ... + subTomoMeta.(cycleNumber).(imgNAME){2},... preSizeMask); @@ -347,7 +344,7 @@ end if (flgLoadMask) && (iGold == 1) fprintf('\n\nLoading external mask\n'); - externalMask = getVolume(MRCImage(sprintf('%s-pcaMask',masterTM.(cycleNumber).(imgNAME){1}))); + externalMask = getVolume(MRCImage(sprintf('%s-pcaMask',subTomoMeta.(cycleNumber).(imgNAME){1}))); end end @@ -503,19 +500,21 @@ % radius, convert Ang to pix , denom = equiv stdv from normal to include, e.g. % for 95% use 1/sig = 1/2 %stdDev = 1/2 .* (emc.pca_scale_spaces ./ pixelSize - 1) .* 3.0./log(emc.pca_scale_spaces) -threeSigma = 1/3 .* (emc.pca_scale_spaces ./ pixelSize); -for iScale = 1:emc.n_scale_spaces - - kernelSize = ceil(threeSigma(iScale).*3) + 3; - kernelSize = kernelSize + (1-mod(kernelSize,2)); - % masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); - masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'gpu', {}); - - - % SAVE_IMG( masks.('scaleMask').(sprintf('s%d',iScale)), ... - % sprintf('%s_scaleMask_s%d.mrc',outputPrefix,iScale),pixelSize); - +if ~(test_multi_ref_diffmap) + threeSigma = 1/3 .* (emc.pca_scale_spaces ./ pixelSize); + for iScale = 1:emc.n_scale_spaces + + kernelSize = ceil(threeSigma(iScale).*3) + 3; + kernelSize = kernelSize + (1-mod(kernelSize,2)); + % masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); + masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'gpu', {}); + + % SAVE_IMG( masks.('scaleMask').(sprintf('s%d',iScale)), ... + % sprintf('%s_scaleMask_s%d.mrc',outputPrefix,iScale),pixelSize); + + + end end avgMotif_FT = cell(1+flgGold,emc.n_scale_spaces); @@ -595,6 +594,7 @@ end % Extend the random subset to each peak if needed + % FIXME: this should just be in random subset if (emc.nPeaks > 1) for iTomo = 1:nTomograms selectedList = geometry.(tomoList{iTomo})(:,8) > 0; @@ -611,7 +611,7 @@ tempDataMatrix = cell(3,1); for iScale = 1:emc.n_scale_spaces dataMatrix{iScale} = zeros(nPixels(iGold,iScale), nSUBSET, 'single'); - tempDataMatrix{iScale} = zeros(nPixels(iGold,iScale), nTempParticles, 'single', 'gpuArray'); + tempDataMatrix{iScale} = zeros(nPixels(iGold,iScale), nParticlesOnGpuBeforeTransfer, 'single', 'gpuArray'); end % Pull masks onto GPU (which are cleared along with everything else when @@ -627,7 +627,9 @@ gpuArray(masks.('binary').(stHALF).(stSCALE)); gpuMasks.('binaryApply').(stSCALE) = ... gpuArray(masks.('binaryApply').(stHALF).(stSCALE)); - gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); + if ~(test_multi_ref_diffmap) + gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); + end gpuMasks.('highPass').(stSCALE) = BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize); @@ -653,11 +655,12 @@ tomoName = tomoList{iTomo}; + iGPU = 1; - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); + TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); if (emc.flgCutOutVolumes) @@ -670,9 +673,9 @@ end - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; + iTiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoName).tiltName; - tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); + tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); fprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName); % Load in the geometry for the tomogram, and get number of subTomos. @@ -778,10 +781,12 @@ shiftVAL = shiftVAL + aliParams(2,1:3)./samplingRate; end + using_this_subtomo = true; + particleIDX = positionList(iSubTomo, 4); % Same for all peaks if ~ischar(indVAL) % Read in and interpolate at single precision as the local values % in the interpolant suffer from any significant round off errors. - particleIDX = positionList(iSubTomo, 4); % Same for all peaks + if (emc.flgCutOutVolumes) @@ -834,8 +839,11 @@ for iScale = 1:emc.n_scale_spaces - - iPrt = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); + if (test_multi_ref_diffmap) + iPrt = iTrimParticle; + else + iPrt = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); + end iPrt = BH_bandLimitCenterNormalize( ... iPrt .* ... @@ -844,36 +852,33 @@ gpuMasks.('binary').(sprintf('s%d',iScale)),... [0,0,0;0,0,0],'single'); - - - - [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iPrt,ifftshift(iWedge),... flgNorm,pixelSize,radialMask, padWdg); - + % using_this_subtomo is set true for each particle, but if we are not finite for ANY scale space, we want to skip this particle for all + % scale spaces. if all(isfinite(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale))))) - keepTomo = 1; tempDataMatrix{iScale}(:,nTemp) = single(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale)))); else - fprintf('inf or nan in subtomo %d scalePace %d',iSubTomo,iScale); - keepTomo = 0; + fprintf('inf or nan in subtomo %d scalePace %d',particleIDX, iScale); + using_this_subtomo = false; end - end + end % loop on scale spaces clear iAvg iWmd iTrimParticle - if (keepTomo) + if (using_this_subtomo) idxList(1, nExtracted) = particleIDX; + fprintf("Adding subtomo %d to idxList\n",particleIDX); peakList(1,nExtracted) = iPeak+1; nExtracted = nExtracted + 1; nTemp = nTemp + 1; % pull data of the gpu every 1000 particls (adjust this to max mem) - if nTemp == nTempParticles - 1 + if nTemp - 1 == nParticlesOnGpuBeforeTransfer for iScale = 1:emc.n_scale_spaces dataMatrix{iScale}(:,1+nTempPrev:nTemp+nTempPrev-1) = ... gather(tempDataMatrix{iScale}(:,1:nTemp-1)); @@ -884,21 +889,27 @@ end else nIgnored = nIgnored + 1; - fprintf('Ignoring subtomo %d from %s\n',iSubTomo, tomoList{iTomo}); - masterTM.(cycleNumber).(geom_name).(tomoList{iTomo})(iSubTomo, 26+iPeak*26) = -9999; + fprintf('Ignoring subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; + subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; end else nIgnored = nIgnored + 1; - fprintf('Ignoring subtomo %d from %s\n',iSubTomo, tomoList{iTomo}); - masterTM.(cycleNumber).(geom_name).(tomoList{iTomo})(iSubTomo, 26+iPeak*26) = -9999; + fprintf('Ignoring inside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; + subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; - end % end of ignore new particles + end % end of update data matrix or if ignoring update metaData end % end of loop over peaks - - end % end of ignore if statment + else + nIgnored = nIgnored + 1; + fprintf('Ignoring outside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; + subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; + end % end of ignore if statment from extracted window out of bounds (ischar(indVAL)) if ~rem(iSubTomo,100) fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... iSubTomo, nSubTomos,iPeak+1,emc.nPeaks, iTomo,nTomograms); @@ -919,9 +930,9 @@ end - masterTM.(cycleNumber).('newIgnored_PCA').(halfSet) = gather(nIgnored); + subTomoMeta.(cycleNumber).('newIgnored_PCA').(halfSet) = gather(nIgnored); - subTomoMeta = masterTM; + subTomoMeta = subTomoMeta; save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); for iScale = 1:emc.n_scale_spaces diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index e99dae7e..529390b4 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=0 -nightly=0 +nightly=1 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index ac886e4a..dabf4f35 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -111,6 +111,10 @@ peakBinary = []; peakCOM = []; peakSearch = []; + +% FIXME: It doesn' tmake any sense to average the classes then also average those together. +% This is a waste of memory as we could just create a single average. I think this must have been +% a hack override for multi-ref alignment with classification or something. saveClassSum = -1; % The prefix of the variable read in from the parameter file. @@ -131,15 +135,18 @@ samplingRate = emc.('Ali_samplingRate'); if (emc.multi_reference_alignment && (test_multi_ref_diffmap || ~emc.classification)) - className = emc.(sprintf('Raw_className')) - saveClassSum = emc.(sprintf('Raw_className')) - elseif (emc.multi_reference_alignment && emc.classification) - fprintf('\n\nMutliRef and Classify enabled.\n'); - fprintf('Only creating the global class average for PCA\n\n.'); - className = 0; - saveClassSum = 0; - classVector{1} = [0;1]; - classVector{2} = [0;1]; + fprintf('\n\nMutliRef enabled.\n'); + pause(2); + className = emc.(sprintf('Raw_className')); + else + if (emc.multi_reference_alignment && emc.classification) + fprintf('\n\nMutliRef and Classify enabled.\n'); + fprintf('Only creating the global class average for PCA\n\n.'); + className = 0; + saveClassSum = 0; + classVector{1} = [0;1]; + classVector{2} = [0;1]; + end end @@ -506,12 +513,12 @@ end nVol = nVol + 1; - end - end + end % iPeak + end % iSubTomo angVect = [angVect tmpTomo]; - end + end % iTomo spike_info.('std_dev') = std(angVect); @@ -535,7 +542,7 @@ save('spike_hist.mat','angVect','chiVect'); % figure, - end + end % if spike_prior nVolumes = 0; addedWeight = 0; @@ -720,7 +727,7 @@ tomoCount = 0; end - sprintf('gpu %d working on %d/%d volumes\n',iParProc,iTomo,nTomograms) + fprintf('gpu %d working on %d/%d volumes\n',iParProc,iTomo,nTomograms); tomoName = tomoList{iTomo}; @@ -744,7 +751,6 @@ tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - fprintf('Loading tomo %d from tilt %s \n',tomoNumber,tiltName); reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) @@ -768,8 +774,6 @@ % Work on each class seperately pushing to main memory when finished. for iGold = 1:2-flgFinalAvg - iGold - size(classVector{iGold},2); for iClassPos = 1:size(classVector{iGold},2) @@ -1192,9 +1196,7 @@ nIgnored = nIgnored + cntResults{iParProc}(1,2); for iVol = 1:maxClasses - iVol for iHalf = 1:2-flgFinalAvg - iHalf if iParProc == 1 avgVolume{iVol, iHalf} = avgResults{iParProc}{iVol, iHalf}; avgWedge{iVol, iHalf} = wgtResults{iParProc}{iVol, iHalf}; @@ -1285,7 +1287,6 @@ sum(nExtracted(iClassPos,iGold)); end - fprintf('flgGold = %d\n',flgGold); for iGold = 1:2-flgFinalAvg if iGold == 1 halfSet = 'ODD'; @@ -1362,15 +1363,11 @@ for iGold = 1:2-flgFinalAvg - %%% if( flgGold ) if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - %%% else - %%% halfSet = 'STD'; - %%% end imgCounts = gather([classVector{iGold}(1,:) ; nExtracted(:,iGold)']); @@ -1478,15 +1475,11 @@ %%%%%%%%%%%%%55 Reweight now that the FSC is calculated for iGold = 1:2 - %%% if( flgGold ) if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - %%% else - %%% halfSet = 'STD'; - %%% end imgIN = sprintf('class_%d_Locations_%s_%s_NoWgt', ... className, fieldPrefix, halfSet); wgtIN = sprintf('class_%d_Locations_%s_%s_Wgt', ... From 83f338f970fff1f73b82b55e729dd68ac81f23e2 Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 12 Feb 2024 12:58:22 -0500 Subject: [PATCH 021/151] Update to cuda12 --- metaData/BH_skipClassAlignment.m | 27 ++--- mexFiles/mexCompile.m | 23 ++-- mexFiles/mexSF3D.cu | 180 ++++++++++++++----------------- mexFiles/mexXform3d.cu | 18 +--- statistics/BH_fscGold_class.m | 30 +++--- synthetic/BH_synthetic_mapBack.m | 11 +- testScripts/mCompile.sh | 2 +- 7 files changed, 130 insertions(+), 161 deletions(-) diff --git a/metaData/BH_skipClassAlignment.m b/metaData/BH_skipClassAlignment.m index 21f71813..9bea8149 100755 --- a/metaData/BH_skipClassAlignment.m +++ b/metaData/BH_skipClassAlignment.m @@ -34,26 +34,21 @@ outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); + + + if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - if (emc.multi_reference_alignment && ~emc.classification) - subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('Avg_geometry'); - - elseif (emc.multi_reference_alignment && emc.classification) - subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('ClusterClsGeom'); - else - - try - subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('ClusterClsGeom'); - catch - subTomoMeta.(cycleNumber).('RawAlign') = ... - subTomoMeta.(cycleNumber).('ClusterRefGeom'); + if (emc.multi_reference_alignment) + if (emc.classification) + subTomoMeta.(cycleNumber).('RawAlign') = subTomoMeta.(cycleNumber).('ClusterClsGeom'); + else + subTomoMeta.(cycleNumber).('RawAlign') = subTomoMeta.(cycleNumber).('ClusterRefGeom'); end - + else + subTomoMeta.(cycleNumber).('RawAlign') = subTomoMeta.(cycleNumber).('Avg_geometry'); end + else error(['STAGEofALIGNMENT to skip may be RawAlignment'],... ['the former requires the latter to exist.\n']); diff --git a/mexFiles/mexCompile.m b/mexFiles/mexCompile.m index 7de3cf64..0deb6b29 100644 --- a/mexFiles/mexCompile.m +++ b/mexFiles/mexCompile.m @@ -24,22 +24,29 @@ % --warn-on-double-precision-use % --warn-on-spills % -Wno-deprecated-gpu-targets +% '-lcublas_static' ... % Link to cuBLAS + mexcuda_opts = { ... CUDA_LIB ... -'-lcublas' ... % Link to cuBLAS +'-lcuda' ... '-lmwlapack' ... % Link to LAPACK -'-lcufft' ... % Link to cuFFT -['NVCCFLAGS= --use_fast_math --default-stream per-thread -m64 '... +'-lcufft' ... +'-lculibos' ... +'-lcudart' ... +'-ldl' ... +'-lrt' ... % Link to cuFFT +['NVCCFLAGS= --use_fast_math --default-stream per-thread -m64 --extra-device-vectorization --expt-relaxed-constexpr -t8 '... '--gpu-architecture=compute_86 ' ... '--restrict -Xptxas --warn-on-spills ' ... - '-gencode=arch=compute_60,code=sm_70 ' ... - '-gencode=arch=compute_61,code=sm_75 ' ... - '-gencode=arch=compute_70,code=sm_80 ' ... - '-gencode=arch=compute_75,code=sm_86 '] ...% the optimizations are default anyway when I checked + '-gencode=arch=compute_70,code=sm_70 ' ... + '-gencode=arch=compute_80,code=sm_80 ' ... + '-gencode=arch=compute_75,code=sm_75 ' ... + '-gencode=arch=compute_86,code=sm_86 ' ... + '-gencode=arch=compute_89,code=sm_89 '] ...% the optimizations are default anyway when I checked }; - +disp(mexcuda_opts); if isfile(sprintf('%s/compiled',mexPATH)) system(sprintf('rm -rf %s/compiled',mexPATH)); system(sprintf('mkdir -p %s/compiled',mexPATH)); diff --git a/mexFiles/mexSF3D.cu b/mexFiles/mexSF3D.cu index 479a22ce..b1620241 100644 --- a/mexFiles/mexSF3D.cu +++ b/mexFiles/mexSF3D.cu @@ -12,22 +12,22 @@ // Texture reference for 2D float texture -texture tex; -const float slice_thickness_pixel_radius = 4; // TODO should +// texture tex; +const float slice_thickness_pixel_radius = 3; // TODO should const float cosine_edge_arg = PI / (float)slice_thickness_pixel_radius; -const float cosine_edge_norm = 1.0f / 4.0f; +const float cosine_edge_norm = 1.0f;// / 3.0f; //////////////////////////////////////////////////////////////////////////////// //! Transform an image using texture lookups //! @param outputData output data in global memory //////////////////////////////////////////////////////////////////////////////// -__global__ void sf3dKernel(float *outputData, +__global__ void sf3dKernel(const cudaTextureObject_t tex_obj, + float *outputData, float3 size_shift, uint3 dims, float2 sinAcosA - ) -{ + ) { // Assuming a single-Y-axis tilt such that the rotation is // [ c, 0, s, @@ -38,42 +38,37 @@ __global__ void sf3dKernel(float *outputData, if (x >= dims.x) { return ; } int y = blockIdx.y*blockDim.y + threadIdx.y; if (y >= dims.y) { return ; } -// int z = blockIdx.z*blockDim.z + threadIdx.z; -// if (z >= dims.z) { return ; } - float u,w,tv,tu,tw,tw_pre,zWeight; + // For every 3D voxel invert the tilt transformation to see where the data would be in the xy plane of the 2D image + float tu,tw,tw_pre,zWeight; - // First calc the Z-dimension, and check that we are close enough to the plane - u = (float)x - (float)dims.x/2; - tw_pre = -u*sinAcosA.x + size_shift.z; - tv = ((float)y - (float)(dims.y/2) + size_shift.y) / (float)dims.y + 0.5f; - for (int z = 0; z < dims.z; z++) - { - w = (float)z - (float)(dims.z/2); - tw = tw_pre + w*sinAcosA.y; + // The centered coordinate in the 3d volume + float x_centered = (float)x - (float)dims.x/2; + tw_pre = -x_centered * sinAcosA.x + size_shift.z; + float y_centered_normalized = ((float)y - (float)(dims.y/2) + size_shift.y) / (float)dims.y + 0.5f; - if (tw < -slice_thickness_pixel_radius | tw > slice_thickness_pixel_radius) - { + for (int z = 0; z < dims.z; z++) { + + float z_centered = float(z - dims.z/2); + tw = tw_pre + z_centered * sinAcosA.y; + + if (tw < -slice_thickness_pixel_radius || tw > slice_thickness_pixel_radius) { continue; } - // FIXME this should approximate a sinc - zWeight = (0.5 + 0.5*cosf(tw * cosine_edge_arg )) * cosine_edge_norm; + // FIXME this should approximate a sinc + zWeight = (0.5 + 0.5*cosf(tw * cosine_edge_arg )) * cosine_edge_norm; - tu = u*sinAcosA.y + w*sinAcosA.x + size_shift.x; - tu /= (float)dims.x; // Normalized coords - tw /= (float)dims.z; + tu = x_centered*sinAcosA.y + z_centered*sinAcosA.x + size_shift.x; + tu /= (float)dims.x; // Normalized coords + tw /= (float)dims.z; - // FIXME this extra division could go - // u /= (float)(dims.x/2); - //100.0f; - tu += 0.5f; - tw += 0.5f; + outputData[ (z*dims.y + y) * dims.x + x ] += ( zWeight * tex2D(tex_obj, tu + 0.5f, y_centered_normalized)); - outputData[ (z*dims.y + y) * dims.x + x ] += ( zWeight * tex2D(tex, tu, tv)); + // outputData[ (z*dims.y + y) * dims.x + x ] += ( zWeight * tex2D(tex, tu, tv)); } } @@ -95,7 +90,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) if ( nrhs != 15 & nrhs != 16 ) { mexErrMsgIdAndTxt("MATLAB:mexSF3D:rhs", - "requires 14 inputs."); + "requires 14 inputs.\n"); } bool * doHalfGrid = (bool *) mxGetData(prhs[0]); @@ -161,7 +156,6 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) mwSize output_size[3] = {dims.x, dims.y, dims.z}; // Allocate device memory for the weights - // Create MX array and init with zeros outputArray = mxGPUCreateGPUArray(output_dims, output_size, @@ -170,83 +164,74 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) MX_GPU_INITIALIZE_VALUES); d_output_img = (float *)(mxGPUGetData(outputArray)); + cudaTextureObject_t tex_obj = 0; + cudaArray* cuArray = 0; - cudaArray *cuArray; + //////////////////////////////////////////////////////////////////// - // TODO where does the 32 come from? - cudaChannelFormatDesc channelDesc = - cudaCreateChannelDesc(32, 0, 0, 0, cudaChannelFormatKindFloat); + // Allocate array and copy image data + cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(32, 0, 0, 0, cudaChannelFormatKindFloat); + checkCudaErrors(cudaMallocArray(&cuArray, &channelDesc, ctf_dims.x, ctf_dims.y)); -// mexit(__LINE__); - (cudaMallocArray(&cuArray, - &channelDesc, - ctf_dims.x, - ctf_dims.y)); - // Set texture parameters - // cudaAddressModeWrap cudaAddressModeClamp cudaAddressModeMirror cudaAddressModeBorder - tex.addressMode[0] = cudaAddressModeBorder; - tex.addressMode[1] = cudaAddressModeBorder; + // TODO checkout cudaCreateChannelDescHalf https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#sixteen-bit-floating-point-textures + struct cudaResourceDesc resDesc; + (memset(&resDesc, 0, sizeof(resDesc))); + resDesc.resType = cudaResourceTypeArray; + resDesc.res.array.array = cuArray; - tex.filterMode = cudaFilterModeLinear; + struct cudaTextureDesc texDesc; + (memset(&texDesc, 0, sizeof(texDesc))); + + texDesc.filterMode = cudaFilterModeLinear; + texDesc.readMode = cudaReadModeElementType; + texDesc.normalizedCoords = 1; + texDesc.addressMode[0] = cudaAddressModeBorder; + texDesc.addressMode[1] = cudaAddressModeBorder; + texDesc.addressMode[2] = cudaAddressModeBorder; - tex.normalized = true; // access with normalized texture coordinates - // Params for the 2d ctf - mwSize const ctf_number_of_dims = 2; - mxComplexity ctf_type = mxREAL; - mxClassID ctf_class = mxSINGLE_CLASS; - mwSize ctf_size[2]; - ctf_size[0] = (mwSize) ctf_dims.x; - ctf_size[1] = (mwSize) ctf_dims.y; - - mxGPUArray *ctfArray; - // TODO it would be nice not to init all the zeros, but then the fourier padding would need to be dealt with. - ctfArray = mxGPUCreateGPUArray(ctf_number_of_dims, - ctf_size, - ctf_class, - ctf_type, - MX_GPU_INITIALIZE_VALUES); - d_ctf_img = (float *)(mxGPUGetData(ctfArray)); + size_t n_to_copy = ctf_dims.x * ctf_dims.y * sizeof(float); + checkCudaErrors(cudaMallocAsync(&d_ctf_img, n_to_copy, cudaStreamPerThread)); + // d_ctf_img = (float *)(mxGPUGetData(ctfArray)); // Would specifying a 3d grid speed up by improving drop out over a block? - dim3 threads_per_block = dim3(32,32,1); // max is 1024 threads/block for 2.x --> 7.5 compute capability - dim3 dimGrid = dim3((dims.x + threads_per_block.x -1) / threads_per_block.x, - (dims.y + threads_per_block.y -1) / threads_per_block.y, - 1);//(dims.z + threads_per_block.z -1) / threads_per_block.z); + dim3 threads_per_block = dim3(32,32,1); // max is 1024 threads/block for 2.x --> 7.5 compute capability + dim3 dimGrid = dim3((dims.x + threads_per_block.x -1) / threads_per_block.x, + (dims.y + threads_per_block.y -1) / threads_per_block.y, + 1);//(dims.z + threads_per_block.z -1) / threads_per_block.z); - dim3 ctfBlock(32, 32, 1); - dim3 ctfGrid((ctf_dims.x + ctfBlock.x - 1) / ctfBlock.x, (ctf_dims.y + ctfBlock.y - 1) / ctfBlock.y, 1); - bool calc_centered = true; + dim3 ctfBlock(32, 32, 1); + dim3 ctfGrid((ctf_dims.x + ctfBlock.x - 1) / ctfBlock.x, (ctf_dims.y + ctfBlock.y - 1) / ctfBlock.y, 1); + bool calc_centered = true; float3 size_shift = make_float3(0.0f, 0.0f, 0.0f); if (IsEven(dims.x)) size_shift.x = 0.5f; if (IsEven(dims.y)) size_shift.y = 0.5f; if (IsEven(dims.z)) size_shift.z = 0.5f; - // Calculate this prior to any half-dim reduction - float2 fourierVoxelSize; - fourierVoxelSize = make_float2( 1.0f/(pixelSize[0] * (float)ctf_dims.x), - 1.0f/(pixelSize[0] * (float)ctf_dims.y)); + // Calculate this prior to any half-dim reduction + float2 fourierVoxelSize; + fourierVoxelSize = make_float2( 1.0f/(pixelSize[0] * (float)ctf_dims.x), + 1.0f/(pixelSize[0] * (float)ctf_dims.y)); + bool is_initialized = false; for (int iAng = 0 ; iAng < *nTilts ; iAng ++) { - if (iAng > 0) - { + if (iAng > 0) { fourierVoxelSize.x = 1.0f/(pixelSize[iAng] * (float)ctf_dims.x); fourierVoxelSize.y = 1.0f/(pixelSize[iAng] * (float)ctf_dims.y); } - ctfParams b_ctf(*doHalfGrid,*doSqCTF,pixelSize[iAng],waveLength[iAng],CS[iAng],AmpContrast[iAng], defocus1[iAng], defocus2[iAng], defocusAst[iAng]); @@ -256,45 +241,36 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) calc_centered, occupancy[iAng], exposure[iAng], *wiener_constant); - // Put the ctf in tex2 - (cudaMemcpyToArray(cuArray, - 0, - 0, - d_ctf_img, - (long)(ctf_dims.x * ctf_dims.y * sizeof(float)), - cudaMemcpyDeviceToDevice)); - - - // Bind the array to the texture - (cudaBindTextureToArray(tex, cuArray, channelDesc)); + // FIXME + checkCudaErrors(cudaMemcpyToArrayAsync(cuArray, 0, 0, d_ctf_img, n_to_copy, cudaMemcpyDeviceToDevice, cudaStreamPerThread)); + // checkCudaErrors(cudaMemcpy2DToArrayAsync(cuArray, 0, 0, d_ctf_img, ctf_dims.x * sizeof(float), ctf_dims.x, ctf_dims.y, cudaMemcpyDeviceToDevice, cudaStreamPerThread)); + if (! is_initialized) { + cudaStreamSynchronize(cudaStreamPerThread); + checkCudaErrors(cudaCreateTextureObject(&tex_obj, &resDesc, &texDesc, NULL)); + is_initialized = true; + } // Call the sf3d kernel - sf3dKernel<<>>(d_output_img, size_shift, - dims, sinAcosA[iAng]); - - + sf3dKernel<<>>(tex_obj, d_output_img, size_shift, dims, sinAcosA[iAng]); - // FIXME if you could bin an array of texture objects, you could launch the sf3dKernel outside the loop once. } - (cudaStreamSynchronize(cudaStreamPerThread)); - -// (cudaUnbindTexture(tex)); - - - + checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); // Check if kernel execution generated an error - getLastCudaError("Kernel execution failed"); + getLastCudaError("Kernel execution failed\n"); plhs[0] = mxGPUCreateMxArrayOnGPU(outputArray); - (cudaFreeArray(cuArray)); + checkCudaErrors(cudaFreeArray(cuArray)); + checkCudaErrors(cudaDestroyTextureObject(tex_obj)); + mxGPUDestroyGPUArray(outputArray); - mxGPUDestroyGPUArray(ctfArray); + // mxGPUDestroyGPUArray(ctfArray); + cudaFreeAsync(d_ctf_img, cudaStreamPerThread); // (cudaFree(d_input_img)); diff --git a/mexFiles/mexXform3d.cu b/mexFiles/mexXform3d.cu index 128cdb32..09c1d4be 100644 --- a/mexFiles/mexXform3d.cu +++ b/mexFiles/mexXform3d.cu @@ -202,7 +202,9 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) mxClassID output_data_class = {mxUINT64_CLASS}; mxComplexity output_data_complexity = {mxREAL}; - // Forward and inverse transforms + // We want to allocate memory that Matlab knows about so it can be used outside of mex (and recycled within) + // Then we point the texture object to it. + // FIXME: The textrure object should not be as big as the cudaArray, right?! WHy is this setup like this. plhs[1] = mxCreateNumericArray(ptr_dims, ptr_size, output_data_class, @@ -225,7 +227,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) // mexMakeArrayPersistent(plhs[2]); // -// cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(); + // cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(); cudaChannelFormatDesc channelDesc = cudaCreateChannelDesc(32, 0, 0, 0, cudaChannelFormatKindFloat); checkCudaErrors(cudaMalloc3DArray(cuArray, @@ -248,18 +250,6 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) resDesc.res.array.array = *cuArray; -// size_t n_bytes = dims.x*dims.y*dims.z*sizeof(float); -// float* buffer; -// checkCudaErrors(cudaMalloc(&buffer, n_bytes ); -// struct cudaResourceDesc resDesc; - -// memset(&resDesc, 0, sizeof(cudaResourceDesc)); -// resDesc.resType = cudaResourceTypeLinear; -// resDesc.res.linear.devPtr = buffer; -// resDesc.res.linear.desc.f = cudaChannelFormatKindFloat; -// resDesc.res.linear.desc.x = 32; // bits per channel -// resDesc.res.linear.sizeInBytes = n_bytes; - struct cudaTextureDesc texDesc; memset(&texDesc,0,sizeof(texDesc)); diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index 6d48054e..bc339903 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -870,19 +870,23 @@ %%%%%%%%%%%%%%%%%%%%%%%%% % For comparing results from a tight mask and solvent normalized fsc % The tight mask calculation is not so reliable (which is by it isn't used in the first place. - % This is used for the figure S3 in the Nature Methods paper, but not in regular us. - figure('Visible','off'), plot(osX,fnval(fitTightFSC,osX),'k-.',... - osX,fnval(fscTrue,osX),'k',... - osX,fnval(fitFSC{1},osX),'b',... - osX, 0.*osX+0.143, 'k--',... - osX,zeros(length(osX)),'k'); - - fTightGold = osX(find(fnval(fscTrue,osX) < 0.143 & osX > 1/100, 1, 'first')); - - title({'FSC',sprintf('0.143 - %3.2f, %3.2f\n(paritcleVolume,tightMask)',1./fgold,1./fTightGold)}); - xlabel('Spatial Freq'); ylabel('fsc'); - ylim([-.05 1.025]) - file_out = sprintf('%s-%d-fscFull_%s', outputPrefix, iRef, halfSet); + % This is used for the figure S3 in the Nature Methods paper, but not in regular use. + plot_compareable_tight_fsc = false; + if (plot_compareable_tight_fsc) + figure('Visible','off'), plot(osX,fnval(fitTightFSC,osX),'k-.',... + osX,fnval(fscTrue,osX),'k',... + osX,fnval(fitFSC{1},osX),'b',... + osX, 0.*osX+0.143, 'k--',... + osX,zeros(length(osX)),'k'); + + fTightGold = osX(find(fnval(fscTrue,osX) < 0.143 & osX > 1/100, 1, 'first')); + + title({'FSC',sprintf('0.143 - %3.2f, %3.2f\n(paritcleVolume,tightMask)',1./fgold,1./fTightGold)}); + xlabel('Spatial Freq'); ylabel('fsc'); + ylim([-.05 1.025]) + file_out = sprintf('%s-%d-fscFull_%s', outputPrefix, iRef, halfSet); + saveas(gcf, file_out,'pdf') + end figure('Visible','off'), plot(osX,cRef{1}(osX),'kd','MarkerSize',3); hold on; if (emc.flgCones) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 27e2179a..a917c44b 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -163,14 +163,11 @@ load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; +% FIXME: rename this to something more descriptive +ctfRange = emc.('tomo_cpr_defocus_range')*10^10; +ctfInc = emc.('tomo_cpr_defocus_step')*10^10; - - -% Add error check onrange for reasonable values. -ctfRange = emc.('tomoCprDefocusRange')*10^10; -ctfInc = emc.('tomoCprDefocusStep')*10^10; - -calcCTF = emc.('tomoCprDefocusRefine'); +calcCTF = emc.('tomo_cpr_defocus_refine'); [tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 529390b4..fb25a0c2 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=0 -nightly=1 +nightly=2 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 725839e61dca007954670ad51850aa5d3c38bced Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 12 Feb 2024 15:51:07 -0500 Subject: [PATCH 022/151] Improved options for visualizing tomoCPR results on phakePhasePlate tomos and also error handling when no subTomos are available. --- ctf/BH_ctf_Correct3d.m | 3 +- synthetic/BH_synthetic_mapBack.m | 96 ++++++++++++++++++++------------ 2 files changed, 60 insertions(+), 39 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index dcaf3bf1..058bc3a9 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -78,7 +78,7 @@ recWithoutMat = true; if length(varargin) > 2 % Full recon for tomoCPR - bh_global_turn_on_phase_plate = varargin{3}; + bh_global_turn_on_phase_plate = varargin{3} filterProjectionsForTomoCPRBackground = varargin{4}; else loadSubTomoMeta = false; @@ -93,7 +93,6 @@ if strcmpi(varargin{1},'templateSearch') recWithoutMat = true; loadSubTomoMeta = false; - flgWhitenPS = [0,0,1.0]; if (bh_global_turn_on_phase_plate(1)) fprintf('WARNING: the filtered tomogram should only be used for viz, not template matching.'); end diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index a917c44b..ee7c75dd 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -16,7 +16,11 @@ buildTomo=1;% % % % % % % METHOD = 'GPU'; flgRunAlignments = true; -COLOR_MAP= '0'; + +% Default true, we don't need this after projection +delete_background_estimate = true; + + emc = BH_parseParameterFile(PARAMETER_FILE); @@ -262,7 +266,7 @@ mbOUT = {[tmpCache],[mapBackIter+1],'dummy'}; fprintf('\nmBOUT name is %smapBack%d/%s\n',mbOUT{1:3}); - +tiltStart=1; for iTiltSeries = tiltStart:nTiltSeries if (skip_to_the_end_and_run) continue; @@ -277,7 +281,9 @@ % altogether. continue end - + + skip_this_tilt_series_because_it_is_empty = false(nTomograms,1); + tiltList = cell(nTomograms,1); @@ -430,14 +436,14 @@ - % re-initialize the parpool for each tilt series to free up mem. - if ~isempty(gcp('nocreate')) - delete(gcp('nocreate')) - EMC_parpool(nWorkers); - else - EMC_parpool(nWorkers); - end - fprintf('init with %d workers\n',nWorkers); + % % re-initialize the parpool for each tilt series to free up mem. + % if ~isempty(gcp('nocreate')) + % delete(gcp('nocreate')) + % EMC_parpool(nWorkers); + % else + % EMC_parpool(nWorkers); + % end + % fprintf('init with %d workers\n',nWorkers); outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); @@ -516,10 +522,16 @@ backgroundName = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec',CWD,tiltNameList{iTiltSeries},1, samplingRate); + send_backgroundLowPassResolution = 28; % TODO: investigate deviations from the default, which is to shut off the phakePhasePlate and to use a backgroundLowPassResolution of 28 + % Default false, we don't apply this filter + % if enabled, it currently only saves the filtered background estimate for visualization in addition to the normal version + if (emc.save_mapback_classes) + BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3); + end + send_phakePhasePlateOption = [0,0]; - send_backgroundLowPassResolution = 28; BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution); % re-initialize the parpool for each tilt series to free up mem. @@ -529,7 +541,9 @@ avgTomo{1} = getVolume(MRCImage(backgroundName)); - system(sprintf('rm %s',backgroundName)); + if (delete_background_estimate) + system(sprintf('rm %s',backgroundName)); + end for iRef = 1:nRefs refVol{1}{iRef} = gpuArray(refVol{1}{iRef}); @@ -569,6 +583,7 @@ % Track the number of fiducials in order to scale the K-factor to more or less % aggressivley downweight outliers in the alignment nFidsTotal = 0; + fidIDX = 0; for iTomo = 1:nTomograms TLT = tiltGeometry.(tomoList{iTomo}); @@ -668,20 +683,28 @@ nPrjs = size(TLT,1) nSubTomos = size(positionList,1); + + if (nSubTomos == 0) + % No points were saved after template matching so skip this tilt series + % altogether. + skip_this_tilt_series_because_it_is_empty(iTomo) = true; + continue; + end + - % TODO need to update this. - if (emc.save_mapback_classes) - colorMap = single(getVolume(MRCImage(COLOR_MAP))); - % should be the same size as the average + % % TODO need to update this. + % if (emc.save_mapback_classes) + % colorMap = single(getVolume(MRCImage(COLOR_MAP))); + % % should be the same size as the average - if any(size(refVol{1})-size(colorMap)) - error('Color map and average vol must be the same size.\n'); - end - colorMap = colorMap(avgOrigin(1)-maxRad:avgOrigin(1)+maxRad,... - avgOrigin(2)-maxRad:avgOrigin(2)+maxRad,... - avgOrigin(3)-maxRad:avgOrigin(3)+maxRad); - end + % if any(size(refVol{1})-size(colorMap)) + % error('Color map and average vol must be the same size.\n'); + % end + % colorMap = colorMap(avgOrigin(1)-maxRad:avgOrigin(1)+maxRad,... + % avgOrigin(2)-maxRad:avgOrigin(2)+maxRad,... + % avgOrigin(3)-maxRad:avgOrigin(3)+maxRad); + % end @@ -711,9 +734,7 @@ - if (iTomo == 1) - fidIDX = 0; - end + %%%%%%%%%% if (buildTomo) @@ -777,7 +798,7 @@ iAvgResamp = gather(iMaskResamp.*iAvgResamp); if (emc.save_mapback_classes || flgClassAvg) - if (emc.save_mapback_classes) + if ~(emc.save_mapback_classes) iColorMap = gather(int16(iMaskResamp.* BH_resample3d(colorMap, ... rSubTomo',shiftVAL,'Bah',METHOD,'forward'))); else @@ -805,7 +826,6 @@ end - try avgTomo{1}(indVAL(1,1):indVAL(2,1), ... indVAL(1,2):indVAL(2,2), ... @@ -824,9 +844,7 @@ continue end - - - + % Reproject using tilt, so just save the 3d coords. fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot*prjVector' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2]), fidIDX); @@ -870,9 +888,13 @@ end %%%% temp condition to skip building full tomo - end - + end % end of loop over tomograms on this tilt-series + % No subtomos remain + if all( skip_this_tilt_series_because_it_is_empty) + continue; + end + if (buildTomo) fclose(coordOUT); fclose(coordSTART); @@ -890,7 +912,7 @@ clear avgTomo if (emc.save_mapback_classes || flgClassAvg) - SAVE_IMG(MRCImage(gather(avgColor)),sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),4.0); + SAVE_IMG(MRCImage(gather(avgColor)),sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),pixelSize); clear avgColor end % If not planning on visualization, save only a binned copy of the synthetic @@ -905,8 +927,8 @@ % '%smapBack%d/%s.bin%dTomo%d.mrc'], ... % tmpTomoBin,mbOUT{1:3},iSave,mbOUT{1:3},tmpTomoBin,iSave)); % end - tmpTomoBin = 2; - if (emc.save_mapback_classes || flgClassAvg) + tmpTomoBin = 1; + if (emc.save_mapback_classes || flgClassAvg && tmpTomoBin > 1) system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomoColor ',... '%smapBack%d/%s.bin%dTomoColor.mrc'], ... tmpTomoBin,mbOUT{1:3},mbOUT{1:3},tmpTomoBin)); From 81d8fc6331c2e17c343f84b26c68caefbbcd21ee Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 15 Feb 2024 08:21:51 -0500 Subject: [PATCH 023/151] Move mex to static compilation for cuda libs --- alignment/BH_alignRaw3d_v2.m | 18 +- alignment/BH_templateSearch3d_2.m | 2 +- ctf/BH_ctf_Estimate.m | 67 +--- ctf/BH_ctf_Updatefft.m | 3 - ctf/BH_runCtfFind.m | 2 +- metaData/BH_geometryAnalysis.m | 4 +- metaData/BH_geometryInitialize.m | 43 +- metaData/BH_parseParameterFile.m | 1 - mexFiles/mexCompile.m | 8 +- statistics/BH_fscGold_class.m | 25 +- statistics/BH_pcaPub.m | 5 +- synthetic/BH_synthetic_mapBack.m | 527 ++++++++----------------- synthetic/BH_to_cisTEM_mapBack.m | 11 +- transformations/BH_average3d.m | 8 +- transformations/BH_multi_loadOrBin.m | 12 +- transformations/BH_multi_loadOrBuild.m | 23 +- 16 files changed, 239 insertions(+), 520 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index df3585c4..e729458a 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -724,12 +724,14 @@ tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); + reconGeometry = (masterTM.reconGeometry.(tomoList{iTomo}) ./ samplingRate); + TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); if (emc.flgCutOutVolumes) volumeData = []; else - [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... + [ volumeData, ~ ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... reconCoords, mapBackIter, ... samplingRate,iGPUidx,reconScaling,0); volHeader = getHeader(volumeData); @@ -792,6 +794,7 @@ % end % Align all valid subtomos, even if the do not belong to the classes we've selected as references. % To ignore particles, remove them with geometry RemoveClases.m + % FIXME: what was this for? flgAllClasses = true; @@ -952,24 +955,18 @@ padWindow(1,2) + 1:end - padWindow(2,2) , ... padWindow(1,3) + 1:end - padWindow(2,3) ); - case 2 - bestOfRefs = sortrows(gather(cccStorageTrans), -6); %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); estPeakCoord = bestOfRefs(1,8:10); - [ iTrimParticle ] = particleInterpolator.interp3d(... RotMat,... estPeakCoord,rotConvention ,... 'inv',particle_symmetry); - - - if (getInitialCCC) [ iTrimInitial ] = particleInterpolator.interp3d(... reshape(angles,3,3),... @@ -980,18 +977,13 @@ reshape(angles,3,3),... [0,0,0],rotConvention ,... 'inv',particle_symmetry); - - end - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... RotMat,... [0,0,0],rotConvention ,... 'inv',particle_symmetry); - - - end + end % switch on align loop switch emc.multi_reference_alignment diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index de3e355e..fc370d91 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -222,7 +222,7 @@ -[ tomogram, reconGeometry ] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... +[ tomogram, ~] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... reconCoords, mapBackIter, samplingRate,... gpuIDX, reconScaling,1,'',super_sample); diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 0bc97d33..59797657 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -324,7 +324,7 @@ mbEST = load(sprintf('%s.xf',mapBackPrfx)); mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); -outputStackName = sprintf('aliStacks/%s%s',stackNameOUT,extension) +outputStackName = sprintf('aliStacks/%s%s',stackNameOUT,extension); try erase_beads_after_ctf = emc.('erase_beads_after_ctf'); @@ -505,45 +505,7 @@ end - d3 = size(STACK,3) - % Check for extra large (8k) data which will be too big for the gpu. - % Should set this up to be a hybrid where each slice is on GPU but - % But then pull to the cpu and store there in stack. - if d1C > 4096 || d2C > 4096 || d3 > 40 - prjMaskMethod = 'GPU' - else - prjMaskMethod = 'GPU' - end - - % % % % evalMask = zeros(d1C,d2C,d3,'single'); - % % % % for iPrj = 1:d3 - % % % % tmpTLT = tltForExp(iPrj,:); - % % % % % need to write over the projections position in the stack to not expand beyond 2d - % % % % tmpTLT(1) = 1; - % % % % [ iEvalMask, ~ ] = BH_multi_projectionMask([d1C,d2C,1;d1C,d2C,1], tmpTLT, ... - % % % % 'GPU', [zShift,deltaZTolerance] ); - % % % % - % % % % evalMask(:,:,tltForExp(iPrj,1)) = gather(iEvalMask); - % % % % end - % % % % - % % % % - % % % % %evalMask = gather(evalMask); - % % % % - % % % % - % % % % nTiles = zeros(size(STACK,3),1); - % % % % - % % % % - % % % % for i = 1+tileSize/2:overlap:d1C-tileSize/2 - % % % % for j = 1+tileSize/2:overlap:d2C-tileSize/2 - % % % % for k = 1:size(STACK,3) - % % % % if evalMask(i,j,k) - % % % % nTiles(k) = nTiles(k) + 1; - % % % % end - % % % % end - % % % % end - % % % % end - - + d3 = size(STACK,3); [radialForCTF,phi,~,~,~,~] = ... BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU', ... @@ -555,15 +517,11 @@ inc = (0.5 - FIXED_FIRSTZERO) / (paddedSize/2); freqVector = [inc+FIXED_FIRSTZERO:inc:0.5 ]; - % % % % clear sumVector radialAvg - % % % % sumVector(length(freqVector)) = gpuArray(double(0)); - % % % % radialAvg(length(freqVector)) = gpuArray(double(0)); - tic - nT = 1; - nT2=0; - nT3= 0; + nT = 1; + nT2 = 0; + nT3 = 0; halfX = floor(paddedSize/2) + 1; % % % % psTile = zeros([(paddedSize).*[1,1],3],'single','gpuArray'); @@ -612,10 +570,6 @@ iProjection = iProjection ./ ... BH_movingRMS(iProjection,[tileSize,tileSize]); - - - - for i = 1+tileSize/2:overlap:d1C-tileSize/2 if min([nT,nT2,nT3])< maxNumberOfTiles && (iEvalMask(i) || iEvalPos(i) || iEvalNeg(i)) for j = 1+tileSize/2:overlap:d2C-tileSize/2 @@ -626,18 +580,9 @@ j-tileSize/2+1:j+tileSize/2)),... padVAL(1,:),padVAL(2,:),... 'GPU','singleTaper'))); - - % % % % - % % % % thisTile = abs(fftn( ... - % % % % BH_padZeros3d(... - % % % % (iProjection( ... - % % % % i-tileSize/2+1:i+tileSize/2,... - % % % % j-tileSize/2+1:j+tileSize/2)),... - % % % % padVAL(1,:),padVAL(2,:),... - % % % % 'GPU','singleTaper'))); + tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; - if (iEvalMask(i)) nT = nT+1; tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index cecad6cd..aa77c119 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -73,7 +73,6 @@ % For some reason matlab was geeking out about calling this in the parfor % loop, getting confused about whether it is a variable or a function. -recGeomForThickness = subTomoMeta.reconGeometry; parfor iGPU = 1:nGPUs % for iTilt = 1:length(ITER_LIST{iGPU}) @@ -159,8 +158,6 @@ % FIXME, this should be stored from previous mask calc and accessed there. % For now just take based on tomogram (which will be larger than the true specimen thickness) - %THICKNESS = recGeomForThickness.(sprintf('%s_1',STACK_PRFX)); - %THICKNESS = min(10,abs(THICKNESS(1,3)-THICKNESS(2,3)).*emc.pixel_size_si.*10^9); THICKNESS = 100; % Assuming all extreme pixels have already been removed from the stack. %PRJ_STACK = {sprintf('%s_local04_18.mrc',mjIDX)};%,sprintf('%s_local14_18.mrc',mjIDX),sprintf('%s_local24_18.mrc',mjIDX),sprintf('%s_local34_18.mrc',mjIDX)}; diff --git a/ctf/BH_runCtfFind.m b/ctf/BH_runCtfFind.m index 6dd8dbf1..369f6d4c 100644 --- a/ctf/BH_runCtfFind.m +++ b/ctf/BH_runCtfFind.m @@ -77,7 +77,7 @@ for iPrj = 1:d3 - system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10,3.141592/180*$4,-1*(($2+$3)/2)*10^-10 }'' >> %s', baseName,iPrj,tmpName)); + system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10,3.1415926535/180.0*$4,-1*(($2+$3)/2)*10^-10 }'' >> %s', baseName,iPrj,tmpName)); end diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 6bb5db26..9bc1e3fb 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -87,9 +87,9 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... switch STAGEofALIGNMENT case 'TiltAlignment' - fieldPrefix = 'Ref' + fieldPrefix = 'Raw' case 'RawAlignment' - fieldPrefix = 'Ref'; + fieldPrefix = 'Raw'; case 'Cluster_cls' fieldPrefix = 'Cls'; STAGEofALIGNMENT = 'Cluster'; diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index 955a4a15..29103e12 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -140,9 +140,8 @@ nTomogramsTotal = length(getPath); fileInfo = cell(nTomogramsTotal,4); -getCoords = dir('recon/*.coords') -getCoords(1).name -nStacks = length(getCoords) +getCoords = dir('recon/*.coords'); +nStacks = length(getCoords); % There is no real need to have a separate data here. The main difference % is these have all tomos from one tilt-series and instead of the origin on @@ -152,11 +151,7 @@ % this could lead to bugs if one is changed and the other not. The only % other concern is then linking each tomogram to the parent tilt-series. for iStack = 1:nStacks - - - - sprintf('recon/%s',getCoords(iStack).name) - + [ recGeom, tiltName, nTomosPossible] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name) ); % Initialize @@ -164,7 +159,7 @@ subTomoMeta.('mapBackGeometry').(tiltName).('coords') = zeros(nTomosPossible,6); if (doImport) - iPath = dir(sprintf('convmap/%s_*.csv',tiltName)) + iPath = dir(sprintf('convmap/%s_*.csv',tiltName)); else iPath = dir(sprintf('convmap/%s_*.mod',tiltName)); end @@ -192,8 +187,7 @@ end - subTomoMeta.('mapBackGeometry').(tiltName).('coords')(tomoNumber,:) = ... - recGeom(tomoNumber,:); + subTomoMeta.('mapBackGeometry').(tiltName).('coords')(tomoNumber,:) = recGeom(tomoNumber,:); subTomoMeta.('mapBackGeometry').('tomoName').(... sprintf('%s_%d',tiltName,tomoNumber)).('tiltName') = tiltName; subTomoMeta.('mapBackGeometry').('tomoName').(... @@ -202,21 +196,19 @@ -end +end % end of loop over stacks for iTomo = 1:nTomogramsTotal - iTomo - sprintf('convmap/%s',getPath(iTomo).name) - modName = strsplit(getPath(iTomo).name,'_') + modName = strsplit(getPath(iTomo).name,'_'); if (doImport) - tiltName = modName{1} + tiltName = modName{1}; tomoNumber = strsplit(modName{2},'.csv'); tomoNumber = EMC_str2double(tomoNumber{1}) else tiltName = strjoin(modName(1:end-2),'_'); - tomoNumber = EMC_str2double(modName{end-1}) + tomoNumber = EMC_str2double(modName{end-1}); end fileInfo{iTomo,1} = tiltName; @@ -226,8 +218,8 @@ subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2}) = load(fileInfo{iTomo,4}); - recCoords = importdata(sprintf('./recon/%s_recon.coords',tiltName)) - recCoords = recCoords.data + recCoords = importdata(sprintf('./recon/%s_recon.coords',tiltName)); + recCoords = recCoords.data; % The reconstruction could be defined based on the aliStacks or the % fixedStacks. the dimensions @@ -239,17 +231,8 @@ floor((recCoords(4 + (tomoNumber-1)*6) + recCoords(3 + (tomoNumber-1)*6) - 1)/2 - (subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2})(1,21))/2),... % oY -- need the tilt series size recCoords(7 + (tomoNumber-1)*6)]; %OZ (negative shift Z in imod reconstruction command -- but rotated during reconstruction so the -1 is implicit); - subTomoMeta.('reconGeometry').(fileInfo{iTomo,2}) = ... - [recGeom(1:3);recGeom(4:6)]; - % try - % recGeom = load(sprintf('./recon/%s_recon.txt',fileInfo{iTomo,2})); - % subTomoMeta.('reconGeometry').(fileInfo{iTomo,2}) = ... - % [recGeom(1:3);recGeom(4:6)]; - % - % catch - % fileInfo{iTomo,2} - % error('error loading ./recon/%s_recon.txt',fileInfo{iTomo,2}); - % end + subTomoMeta.('reconGeometry').(fileInfo{iTomo,2}) = [recGeom(1:3);recGeom(4:6)]; + % Check to make sure no out of bounds conditions were created in X Y diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 903627be..29c6edfd 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -525,6 +525,5 @@ emc = EMC_assert_deprecated_substitution(emc, false, 'tomo_cpr_defocus_refine', 'calcCTF'); EMC_assert_boolean(emc.tomo_cpr_defocus_refine); - end diff --git a/mexFiles/mexCompile.m b/mexFiles/mexCompile.m index 0deb6b29..9bc41862 100644 --- a/mexFiles/mexCompile.m +++ b/mexFiles/mexCompile.m @@ -2,7 +2,7 @@ fprintf("\n\nCompile here\n\n"); mexPATH = '/sa_shared/git/emClarity/mexFiles'; -CUDA_LIB = '-L/usr/local/cuda/lib64'; ... % NOTE if you leave a space at the end of this string, MATLAB does not parse the option correctly (which wouldn't matter in a normal compile line!) +CUDA_LIB = '-L/usr/local/cuda/lib64 '; ... % NOTE if you leave a space at the end of this string, MATLAB does not parse the option correctly (which wouldn't matter in a normal compile line!) getenv('MW_NVCC_PATH') getenv('CUDA_HOME') @@ -25,14 +25,12 @@ % --warn-on-spills % -Wno-deprecated-gpu-targets % '-lcublas_static' ... % Link to cuBLAS - mexcuda_opts = { ... CUDA_LIB ... -'-lcuda' ... '-lmwlapack' ... % Link to LAPACK -'-lcufft' ... +'-lcufft_static_nocallback' ... '-lculibos' ... -'-lcudart' ... +'-lcudart_static' ... '-ldl' ... '-lrt' ... % Link to cuFFT ['NVCCFLAGS= --use_fast_math --default-stream per-thread -m64 --extra-device-vectorization --expt-relaxed-constexpr -t8 '... diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index bc339903..c370df9e 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -105,7 +105,7 @@ refVector{1} =1; refVector{2}= 1; STAGEofALIGNMENT = 'RawAlignment'; - fieldPrefix = 'Ref' + fieldPrefix = 'Ref'; end else @@ -130,7 +130,6 @@ imageName{2} = sprintf('class_%d_Locations_%s_EVE_NoWgt', className,fieldPrefix); weightName{1} = sprintf('class_%d_Locations_%s_ODD_Wgt', className,fieldPrefix); weightName{2} = sprintf('class_%d_Locations_%s_EVE_Wgt', className,fieldPrefix); - imageName{1} refVector{1} =1; refVector{2}= 1; @@ -141,7 +140,7 @@ case 'SnrEstimate' savePrefix = 'SNR'; flgEstSNR = 1; - fieldPrefix = 'Ref' + fieldPrefix = 'Ref'; imageName{1} = sprintf('class_%d_Locations_%s_ODD_NoWgt', 25,fieldPrefix); imageName{2} = sprintf('class_%d_Locations_%s_EVE_NoWgt', 25,fieldPrefix); @@ -159,10 +158,10 @@ end [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms) + BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms); [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc] = ... - BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); padDIM = max(max(sizeWindow),384); padREF = [0,0,0;0,0,0]; @@ -243,8 +242,8 @@ % end fprintf('%d\n',nargin); if nargin > 3 - sLow = EMC_str2double(varargin{1}) - sTop = EMC_str2double(varargin{2}) + sLow = EMC_str2double(varargin{1}); + sTop = EMC_str2double(varargin{2}); else sLow = 1e-6; sTop = 0.999; @@ -482,7 +481,7 @@ fscRandCutoffRes = 3*masterTM.currentResForDefocusError(1); lowResShift = emc.pixel_size_angstroms*2 - 10; if lowResShift <= 0 - lowResShift = 0 + lowResShift = 0; else lowResShift = 1/(fscRandCutoffRes+lowResShift); end @@ -642,13 +641,11 @@ gridSearch = eulerSearch(refSymmetry,180,5,360,5,0.0,1,true); fprintf('Adjusting the one/half bit cutoffs for the %d asymmetric unit(s)\n',gridSearch.number_of_asymmetric_units); end - nCones nEffective = fnval(fitNUM{1},osX).*(3/2.*DbyL).^2 ./ (2.*gridSearch.number_of_asymmetric_units); oneBIT = ( 0.5+2.4142./sqrt(nEffective) ) ./ ... ( 1.5 + 1.4142./sqrt(nEffective) ); - halfBIT= ( 0.207+1.9102./sqrt(nEffective) ) ./ ... ( 1.2071 + 0.9102./sqrt(nEffective) ); @@ -656,8 +653,6 @@ % Find the two common cutoff values -- need a better way to determine the second % value that handles non-monotonic curves and is still smooth/gentle without % falling off too slowly. - - lowCut1 = find(fnval(fitFSC{1},osX) <= 0.143 & osX > 1/100, 1, 'first'); try oneBitCut(1) = find(fnval(fitFSC{1},osX)-aliBIT < 0 & osX > 1/100, 1, 'first'); @@ -726,7 +721,7 @@ if lowestRes < 1./osX(lowCut1) lowestRes = 1./osX(lowCut1); - elseif highestRes > 1./ osX(lowCut1) + elseif highestRes > 1 ./ osX(lowCut1) highestRes = 1./osX(lowCut1); end @@ -803,8 +798,8 @@ clear fout famp1 famp2 fphase1 fphase2 - fmid = osX(find(fnval(fitFSC{1},osX) < 0.5 & osX > 1/100, 1, 'first')) - fgold = osX(find(fnval(fitFSC{1},osX) < 0.143 & osX > 1/100, 1, 'first')) + fmid = osX(find(fnval(fitFSC{1},osX) < 0.5 & osX > 1/100, 1, 'first')); + fgold = osX(find(fnval(fitFSC{1},osX) < 0.143 & osX > 1/100, 1, 'first')); fprintf('\n0.5 = 1/%f\n0.143 = 1/%f\n', fmid, fgold) diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 774d4920..ce8179dd 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -660,13 +660,15 @@ tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); + reconGeometry = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); + TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); if (emc.flgCutOutVolumes) volumeData = []; else - [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... + [ volumeData, ~ ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... reconCoords, mapBackIter, ... samplingRate, iGPU, reconScaling,0); volHeader = getHeader(volumeData); @@ -872,7 +874,6 @@ if (using_this_subtomo) idxList(1, nExtracted) = particleIDX; - fprintf("Adding subtomo %d to idxList\n",particleIDX); peakList(1,nExtracted) = iPeak+1; nExtracted = nExtracted + 1; nTemp = nTemp + 1; diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index ee7c75dd..f5d55df7 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -13,9 +13,11 @@ % Some flags that are worth keeping as options, but not accessible % directlyCT % by the users (private methods-ish) + +% FIXME: is this even relevant any more? buildTomo=1;% % % % % % % -METHOD = 'GPU'; -flgRunAlignments = true; + + % Default true, we don't need this after projection delete_background_estimate = true; @@ -27,32 +29,36 @@ CYCLE = EMC_str2double(CYCLE); cycle_numerator = ''; cycle_denominator =''; + +% When tomoCPR is run on one node (start to finish) or is being finalized on a many node run using the [cycle, nodeIDX, totalNodes] syntax with we [ cycle, 0, 0 ] +% this is set to true. For the many node alignment, we want to defer running the final alignment until all the nodes have finished their work, which we do by setting this to false. +flgRunAlignments = true; +multi_node_run = false; skip_to_the_end_and_run = false; if numel(CYCLE) == 3 - + multi_node_run = true; % After splitting, run the alignments while skipping everything else if CYCLE(2) == 0 && CYCLE(3) == 0 skip_to_the_end_and_run = true; - flgRunAlignments = true; else flgRunAlignments = false; end cycle_numerator = CYCLE(2); cycle_denominator = CYCLE(3); CYCLE = CYCLE(1); - flgAltRun = 1; - -else - flgAltRun = 0; % Could just use one flag for RunAlignments and ALt Run end +EMC_assert_numeric(CYCLE, 1, [0, inf]); -cycleNumber = sprintf('cycle%0.3u', CYCLE); +% skip_to_the_end_and_run is only relevant when running on multiple nodes +if (skip_to_the_end_and_run && ~multi_node_run) + error('You are trying to skip to the end and run, but you are not running on multiple nodes'); +end +cycleNumber = sprintf('cycle%0.3u', CYCLE); -reconScaling = 1; samplingRate = emc.('Ali_samplingRate'); % used to determine the number of fiducials/patch for local area. MOL_MASS = emc.('particleMass'); @@ -96,7 +102,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% try - use_PCF = emc.('use_PCF') + use_PCF = emc.('use_PCF'); catch use_PCF = 0; end @@ -177,7 +183,7 @@ [tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); -if (flgAltRun && ~skip_to_the_end_and_run) +if (multi_node_run && ~skip_to_the_end_and_run) nParts = ceil(nTiltSeries ./ cycle_denominator); tiltStart = 1+(cycle_numerator - 1)*nParts; nTotal = nTiltSeries; @@ -214,12 +220,9 @@ classVector{1} = emc.('Raw_classes_odd')(1,:); -classSymmetry{1}= emc.('Raw_classes_odd')(2,:); classVector{2} = emc.('Raw_classes_eve')(1,:); -classSymmetry{2}= emc.('Raw_classes_eve')(2,:); - -nRefs = length(classVector{1}) +nRefs = length(classVector{1}); particleMask = cell(nRefs,1); for iGold = 1:2 @@ -261,18 +264,19 @@ end tiltGeometry = subTomoMeta.tiltGeometry; -outCTF = ''; +% TODO: this is a bit of an aritfact, can be removed. +outCTF = '_ctf'; + is_first_run = true; mbOUT = {[tmpCache],[mapBackIter+1],'dummy'}; -fprintf('\nmBOUT name is %smapBack%d/%s\n',mbOUT{1:3}); tiltStart=1; for iTiltSeries = tiltStart:nTiltSeries if (skip_to_the_end_and_run) continue; end + - tiltNameList mapBackRePrjSize = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize'); % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos @@ -283,8 +287,6 @@ end skip_this_tilt_series_because_it_is_empty = false(nTomograms,1); - - tiltList = cell(nTomograms,1); % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); @@ -295,16 +297,10 @@ if any(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords(iTomo,:)) tomoList{tomoIDX} = sprintf('%s_%d',tiltNameList{iTiltSeries},iTomo); - - - tiltList{tomoIDX} = sprintf('%saliStacks/%s_ali%d.fixed',... - CWD,tiltNameList{iTiltSeries},mapBackIter+1); - outCTF='_ctf'; - + tiltList{tomoIDX} = sprintf('%saliStacks/%s_ali%d.fixed', CWD, tiltNameList{iTiltSeries}, mapBackIter + 1); % Only increment if values found. tomoIDX = tomoIDX + 1; end - end [~,tiltBaseName,~] = fileparts(tiltList{1}); @@ -312,36 +308,28 @@ if (mapBackIter) - localFile = sprintf('%smapBack%d/%s_ali%d_ctf.local', ... - CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter) + localFile = sprintf('%smapBack%d/%s_ali%d_ctf.local', CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter); else - localFile = sprintf('%sfixedStacks/%s.local',CWD,tiltNameList{iTiltSeries}) + localFile = sprintf('%sfixedStacks/%s.local',CWD,tiltNameList{iTiltSeries}); end if exist(localFile,'file') - fprintf('Found local file\n.'); + fprintf('Found local file %s\n.', localFile); else fprintf('No local transforms requested.\n'); localFile = 0; end - % For now assume that all of these are the same - this is a shitty way to - % handle it, but keeps things general and simple. - reconRotation = zeros(nTomograms,3); - % Remove this option (always assume -rx) - for iTomo = 1:nTomograms - reconRotation(iTomo,:) = [0,0,-90]; - end + % The model is scaled to full sampling prior to passing to tiltalign, % make sure the header in the synthetic stack is set appropriately. - fullPixelSize = emc.pixel_size_angstroms; - pixelSize = fullPixelSize .* samplingRate; + unsampled_pixel_size = emc.pixel_size_angstroms; + pixel_size = unsampled_pixel_size .* samplingRate; try eraseMaskType = emc.('Peak_mType'); - eraseMaskRadius = emc.('Peak_mRadius')./pixelSize; - fprintf('Further restricting peak search to radius %f %f %f\n',... - eraseMaskRadius); + eraseMaskRadius = emc.('Peak_mRadius') ./ pixel_size; + fprintf('Further restricting peak search to radius of [%f %f %f] pixels\n', eraseMaskRadius); eraseMask = 1; catch eraseMask = 0; @@ -349,16 +337,16 @@ end - [ ~,~,maskRadius,~ ] = BH_multi_maskCheck(emc,'Ali',pixelSize) - PARTICLE_RADIUS = floor(max(emc.('particleRadius')./pixelSize)); + particle_radius = floor(max(emc.('particleRadius')./pixel_size)); - %PARTICLE_RADIUS = floor(mean(emc.('particleRadius')./pixelSize)); % TODO, is this too restricted? - peakSearchRad = floor(emc.peak_mask_fraction*PARTICLE_RADIUS.*[1,1]); + % current default peak_mask_fraction = 0.4 + peak_search_radius = floor(emc.peak_mask_fraction .* particle_radius .* [1,1]); + + % FIXME: this should be in parseParameterFile try lowPassCutoff = emc.('tomoCprLowPass'); - fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n.',... - lowPassCutoff); + fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n.', lowPassCutoff); catch % TODO are these range limits okay? lowPassCutoff = 1.5.*mean(subTomoMeta.currentResForDefocusError); @@ -370,11 +358,14 @@ fprintf('Using an internatlly determined lowpass cutoff of %3.3f Ang\n.',... lowPassCutoff); end - if lowPassCutoff < 2* pixelSize + + % FIXME: this can also be in parseParameterFile + if lowPassCutoff < 2* pixel_size fprintf('Psych, the cutoff is being set to Nyquist'); - lowPassCutoff = 2*pixelSize; + lowPassCutoff = 2*pixel_size; end + % FIXME: this should be in parseParameterFile min_res_for_ctf_fitting = 10.0; if (calcCTF) try @@ -382,44 +373,23 @@ catch end - if sqrt(2)*pixelSize > min_res_for_ctf_fitting + if sqrt(2)*pixel_size > min_res_for_ctf_fitting fprintf('Warning the current resolution is too low to refine the defocus. Turning off this feature'); calcCTF = false; end end - % % % % % targetPatchSize = max(500, ceil(2.*(PARTICLE_RADIUS).*sqrt(nFiducialsPerPatch))) - - nFiducialsPerPatch = ceil(100./sqrt(molMass)) - targetPatchSize = max(500, ceil(2.*(PARTICLE_RADIUS).*sqrt(nFiducialsPerPatch))) - - - % % % - % % % % Check to see if this tilt has already been worked on, if so skip - % % % aliCmdFileCheck = sprintf('%smapBack%d/%s.align',mbOUT{1:3}); - % % % if exist(aliCmdFileCheck,'file') - % % % fprintf('\n\nFound aliCmdFileCheck, skipping rather than overwrite.\n'); - % % % continue - % % % end - - + nFiducialsPerPatch = ceil(100./sqrt(molMass)); + targetPatchSize = max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch))); + if (samplingRate > 1) for iTomo = 1:nTomograms [~, tltName, tltExt] = fileparts(tiltList{iTomo}); - - % Resample the tilt if necessary, then modify the tilt list - BH_multi_loadOrBin(tiltList{iTomo},-1.*samplingRate, 2); - tiltList{iTomo} = sprintf('%scache/%s_bin%d%s', ... - CWD,tltName, samplingRate,tltExt); - - - - - + tiltList{iTomo} = sprintf('%scache/%s_bin%d%s', CWD, tltName, samplingRate, tltExt); end end @@ -436,23 +406,18 @@ - % % re-initialize the parpool for each tilt series to free up mem. - % if ~isempty(gcp('nocreate')) - % delete(gcp('nocreate')) - % EMC_parpool(nWorkers); - % else - % EMC_parpool(nWorkers); - % end - % fprintf('init with %d workers\n',nWorkers); + % re-initialize the parpool for each tilt series to free up mem. + if ~isempty(gcp('nocreate')) + delete(gcp('nocreate')) + EMC_parpool(nWorkers); + else + EMC_parpool(nWorkers); + end + fprintf('init with %d workers\n',nWorkers); outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); - - - - - - + % Get the thickest for recon maxZ = 0; overSampleZforProjection = 1.0; @@ -478,8 +443,8 @@ % xyzproj assumes centered in Z, so add extra height for z offsets to create % the true "in microsope" dimension - reconstructionSize = [tiltHeader.nX,tiltHeader.nY,maxZ] - originRec = ceil((reconstructionSize+1)./2) + reconstructionSize = [tiltHeader.nX,tiltHeader.nY,maxZ]; + originRec = ceil((reconstructionSize+1)./2); avgTomo = cell(3,1); @@ -493,10 +458,7 @@ avgColor = zeros(reconstructionSize, 'int16'); end - % as the projection of the 3dModel with tilt will use this file and it - % must match the zCoords in the defAng file. - tomoList{1} - pause(3) + TLT = tiltGeometry.(tomoList{1}); @@ -527,9 +489,9 @@ % TODO: investigate deviations from the default, which is to shut off the phakePhasePlate and to use a backgroundLowPassResolution of 28 % Default false, we don't apply this filter % if enabled, it currently only saves the filtered background estimate for visualization in addition to the normal version - if (emc.save_mapback_classes) - BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3); - end + % if (emc.save_mapback_classes) + % BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3); + % end send_phakePhasePlateOption = [0,0]; BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution); @@ -576,7 +538,9 @@ if (buildTomo) coordOUT = fopen(sprintf('%smapBack%d/%s.coord',mbOUT{1:3}),'w'); coordSTART = fopen(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3}),'w'); - + if (emc.save_mapback_classes) + coordCLASS = fopen(sprintf('%smapBack%d/%s.coord_class',mbOUT{1:3}),'w'); + end defOUT = fopen(sprintf('%smapBack%d/%s.defAng',mbOUT{1:3}),'w'); end @@ -631,8 +595,6 @@ tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); - % [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); - binShift = [0,0,0]; positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); @@ -648,40 +610,17 @@ sTX = floor(tiltHeader.nX ); sTY = floor(tiltHeader.nY ); iTLT = floor(tiltHeader.nZ); - % FIXME the z-dimension should be 1 right? - originPrj = ceil(([sTX,sTY,0]+1)./2); + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - - - % iGPU=1; - - if (buildTomo) - % [tomo,tomoReconCoords] = BH_multi_loadOrBuild(tomoList{iTomo}, ... - % reconCoords, mapBackIter, ... - % samplingRate, iGPU,reconScaling,1); - - doRecon = 0; - doLoad = false; - reconCoords - [~,tomoReconCoords] = BH_multi_loadOrBuild(tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate, doRecon,reconScaling,... - doLoad, 'tomoCPR'); - - - - originVol = ceil((tomoReconCoords(1,1:3)+1)./2); - - reconShift = tomoReconCoords(2,1:3); - - end - - + % reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); + tomoReconCoords = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); + + tomo_origin_in_tomo_frame = ceil((tomoReconCoords(1,1:3)+1)./2); + tomo_origin_wrt_tilt_origin = tomoReconCoords(2,1:3); - nPrjs = size(TLT,1) + nPrjs = size(TLT,1); nSubTomos = size(positionList,1); if (nSubTomos == 0) @@ -706,101 +645,76 @@ % avgOrigin(3)-maxRad:avgOrigin(3)+maxRad); % end - - - % Switch from maskRadius to particleRadius 20180129 sizeAvgVol = size(refVol{1}{1}); - - - - - for iRef = 1:nRefs - % FIXME change to EMC_maskreference + % FIXME: change to EMC_maskreference refVol{1}{iRef} = gpuArray(refVol{1}{iRef}); refVol{2}{iRef} = gpuArray(refVol{2}{iRef}); - particleMask{iRef} = BH_mask3d('sphere',sizeAvgVol,PARTICLE_RADIUS.*[1,1,1],[0,0,0]).* ... - BH_mask3d(refVol{1}{iRef} + refVol{2}{iRef} ,pixelSize,'',''); - - % binaryMask = particleMask{iRef} > 0.01; - % for rV = 1:2 - % refVol{rV}{iRef} = refVol{rV}{iRef} - mean(refVol{rV}{iRef}(binaryMask)); - % refVol{rV}{iRef} = refVol{rV}{iRef} ./ (0.5.*rms(refVol{rV}{iRef}(binaryMask))); - % refVol{rV}{iRef} = refVol{rV}{iRef} .* particleMask{iRef}; - % end + particleMask{iRef} = BH_mask3d('sphere',sizeAvgVol,particle_radius.*[1,1,1],[0,0,0]).* ... + BH_mask3d(refVol{1}{iRef} + refVol{2}{iRef} ,pixel_size,'',''); end - - - - - - %%%%%%%%%% if (buildTomo) + % We need to rotate the model 90 degrees around X to match the "natural" reconstruction reference frame of imod + % that is [x,z,-y] modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector'); for iSubTomo = 1:nSubTomos - - - % - % prjVector = (positionList(iSubTomo,11:13)./samplingRate + binShift) - ... - % originVol + reconShift; - % - rSubTomo = reshape(positionList(iSubTomo,17:25),3,3); - prjVector = (positionList(iSubTomo,11:13)./samplingRate) - originVol + reconShift; + + subtomo_rot_matrix = reshape(positionList(iSubTomo,17:25),3,3); + subtomo_origin_in_tomo_frame = (positionList(iSubTomo,11:13) ./ samplingRate); + subtomo_origin_wrt_tilt_origin = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_tilt_origin; iRefIDX = 1; - iClassIDX = 1; + iClassIDX = positionList(iSubTomo,26); if (nRefs > 1) % Assuming generally there are fewer classes seleceted as references than there are total classes - % For those that aren't on of the select ones, we could try to track the best matched reference from the most recent + % For those that aren't one of the select ones, we could try to track the best matched reference from the most recent % alignment - iClassIDX = positionList(iSubTomo,26); + % FIXME: having a class occupancy factor would be better than just picking a random one. + if ~(ismember(iClassIDX,classVector{1}) || ismember(iClassIDX,classVector{2})) - iClassIDX = datasample(classVector{1},1); + use_class = datasample(classVector{1},1); + else + use_class = iClassIDX; end - iRefIDX = find(classVector{1} == iClassIDX); + iRefIDX = find(classVector{1} == use_class); end + % This extra shift came from experiments with real data but is both annoying and not understood. + subtomo_origin_wrt_tilt_origin = subtomo_origin_wrt_tilt_origin - emc.flgPreShift; + + % subTomo origin relative to reconLowerLeft + subtomo_origin_in_sample = originRec + subtomo_origin_wrt_tilt_origin; - % % % prjVector = prjVector + [0.5,0.0,-0.5]; - % prjVector = prjVector + [0.0,0.0,1.0]; - prjVector = prjVector - emc.flgPreShift; - recVector = (originPrj + [0,0,ceil((reconstructionSize(3)+1)/2)] + prjVector); % subTomo origin relative to reconLowerLeft - - %Resample a copy of the average to match the position in the tomogram + % Resample a copy of the average to match the position in the tomogram % The third entry is a dummy, normally used to make sure at least the % particle was being extracted even if the surrounding density (where % some delocalized values may be located) are not. - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(reconstructionSize, sizeAvgVol, sizeAvgVol./5, recVector); - + [ indVAL, padVAL, shiftVAL ] = BH_isWindowValid(reconstructionSize, sizeAvgVol, sizeAvgVol./5, subtomo_origin_in_sample); - - if ~ischar(indVAL) - - + + if ischar(indVAL) + fprintf('ignoring subTomo %d for out of bounds conditions.\n', iSubTomo); + else if positionList(iSubTomo,7) == 1 - iAvgResamp = BH_resample3d(refVol{1}{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); + iAvgResamp = BH_resample3d(refVol{1}{iRefIDX},subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'); elseif positionList(iSubTomo,7) ==2 - iAvgResamp = BH_resample3d(refVol{2}{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); + iAvgResamp = BH_resample3d(refVol{2}{iRefIDX},subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'); else error('positionList iSubtomo %d col 7 is %d',iSubTomo,positionList(iSubTomo,7)); end - iMaskResamp = BH_resample3d(particleMask{iRefIDX},rSubTomo',shiftVAL,'Bah',METHOD,'forward'); - - - + iMaskResamp = BH_resample3d(particleMask{iRefIDX},subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'); iAvgResamp = gather(iMaskResamp.*iAvgResamp); - if (emc.save_mapback_classes || flgClassAvg) + if (emc.save_mapback_classes) if ~(emc.save_mapback_classes) iColorMap = gather(int16(iMaskResamp.* BH_resample3d(colorMap, ... - rSubTomo',shiftVAL,'Bah',METHOD,'forward'))); + subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'))); else % Set value to class average number iColorMap = iMaskResamp; @@ -808,12 +722,7 @@ iColorMap(iColorMap >= 0.05) = iRefIDX; iColorMap = gather(int16(iColorMap)); end - - - if ( flgClassAvg ) - - end - + avgColor(indVAL(1,1):indVAL(2,1), ... indVAL(1,2):indVAL(2,2), ... indVAL(1,3):indVAL(2,3)) = avgColor(indVAL(1,1):indVAL(2,1), ... @@ -823,7 +732,6 @@ 1+padVAL(1,2):end-padVAL(2,2),... 1+padVAL(1,3):end-padVAL(2,3)); - end try @@ -844,30 +752,34 @@ continue end - - % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot*prjVector' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2]), fidIDX); + + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_tilt_origin' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2]), fidIDX); + % Save a non-rotated model with each class on its own object for visualization + if (emc.save_mapback_classes) + fprintf(coordCLASS,'%d 1 %0.4f %0.4f %0.4f\n', iClassIDX, subtomo_origin_wrt_tilt_origin' + originRec'- emc.prjVectorShift); + end + for iPrj = 1:nPrjs iPrj_nat = find(TLT(:,1) == iPrj); % imod is indexing from zero - % imod is indexing from zero zCoord = iPrj_nat; rTilt = BH_defineMatrix([90,1.*TLT(iPrj_nat,4),-90],'Bah','forwardVector'); - prjCoords = rTilt*prjVector'; + prjCoords = rTilt*subtomo_origin_wrt_tilt_origin'; - fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)); - % d1 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; - % d2 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; + fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, samplingRate.*prjCoords(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15)); + % d1 = -1.*((samplingRate.*prjCoords(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; + % d2 = -1.*((samplingRate.*prjCoords(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; - d1 = -1.*(samplingRate.*prjVector(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15))*10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm + d1 = -1.*(samplingRate.*subtomo_origin_wrt_tilt_origin(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15))*10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value - fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',fidIDX, tomoNumber,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(rSubTomo,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); + fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n', ... + fidIDX, tomoNumber,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(subtomo_rot_matrix,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); % These shifts are a record of transformation from the raw data, but here % we are comparing with [CTF] corrected data, from which the @@ -875,15 +787,8 @@ end % loop over tilt projections - fidIDX = fidIDX + 1; - else - fprintf('ignoring subTomo %d for out of bounds conditions.\n', iSubTomo); - reconstructionSize - sizeAvgVol - sizeAvgVol./5 - recVector - end + end % if condition for valid subTomo windowing end % loop over subtomos end %%%% temp condition to skip building full tomo @@ -891,42 +796,38 @@ end % end of loop over tomograms on this tilt-series % No subtomos remain - if all( skip_this_tilt_series_because_it_is_empty) + if all( skip_this_tilt_series_because_it_is_empty ) continue; end if (buildTomo) + fclose(coordOUT); fclose(coordSTART); - + if (emc.save_mapback_classes) + fclose(coordCLASS); + p2m = sprintf(['point2model -sphere 6 -thick 6 -scat ',... + '%smapBack%d/%s.coord_class %smapBack%d/%s_classIdx.3dfid'], ... + mbOUT{1:3},mbOUT{1:3}); + system(p2m); + end p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 ',... - '%smapBack%d/%s.coord %smapBack%d/%s.3dfid'], ... - mbOUT{1:3},mbOUT{1:3}) + '%smapBack%d/%s.coord %smapBack%d/%s.3dfid'], ... + mbOUT{1:3},mbOUT{1:3}); system(p2m); for iSave = 1 - SAVE_IMG(MRCImage(gather(avgTomo{iSave})),sprintf('%smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave),pixelSize); + SAVE_IMG(MRCImage(gather(avgTomo{iSave})),sprintf('%smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave),pixel_size); avgTomo{iSave} = []; end clear avgTomo if (emc.save_mapback_classes || flgClassAvg) - SAVE_IMG(MRCImage(gather(avgColor)),sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),pixelSize); + SAVE_IMG(MRCImage(gather(avgColor)),sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),pixel_size); clear avgColor end - % If not planning on visualization, save only a binned copy of the synthetic - % tomo. - - % tmpTomoBin = floor(1/samplingRate*6); - % TODO make this an adjustable parameter - % tmpTomoBin = ceil(6/pixelSize); - % - % for iSave = 1:1+(3*testSubtraction) - % system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomo%d ',... - % '%smapBack%d/%s.bin%dTomo%d.mrc'], ... - % tmpTomoBin,mbOUT{1:3},iSave,mbOUT{1:3},tmpTomoBin,iSave)); - % end + tmpTomoBin = 1; if (emc.save_mapback_classes || flgClassAvg && tmpTomoBin > 1) system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomoColor ',... @@ -936,10 +837,7 @@ end - - - - rotSize = [tiltHeader.nX,maxZ,tiltHeader.nY] + rotSize = [tiltHeader.nX,maxZ,tiltHeader.nY]; for iSave = 1 rotCMD = sprintf(['rotatevol -angles 0,0,90 -size %d,%d,%d ',... @@ -949,24 +847,8 @@ system(rotCMD); system(sprintf('rm %smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave)); - - end - - - - % % % % if (emc.save_mapback_classes) - % % % % SAVE_IMG(MRCImage(avgColor),sprintf('mapBack/%s_colorMap.mrc',tiltBaseName)); - % % % % % -90 is assumed for trim vol, so if rotate vol is used add 90 - % % % % if (rotateVol) - % % % % system(sprintf('rotatevol -angles 0,0,%d mapBack/%s_colorMap.mrc mapBack/%s_colorMap.rot',reconRotation(iTomo,3)+90.0,tiltBaseName,tiltBaseName)); - % % % % system(sprintf('mv mapBack/%s_colorMap.rot mapBack/%s_colorMap.mrc',tiltBaseName,tiltBaseName)); - % % % % system(sprintf('rm mapBack/%s_colorMap.rot',tiltBaseName)); - % % % % end - % % % % end - - clear avgTomo{1} wgt end % % % % It may be faster to work with a rotated vol since the reading in may cause @@ -1006,7 +888,7 @@ cpuLastLine = ''; end - if strcmpi(METHOD, 'GPU') + if strcmpi('GPU', 'GPU') if (lastLine1) lastLine2 = 'UseGPU 0'; lastLine3 = 'ActionIfGPUFails 2,2'; @@ -1023,7 +905,6 @@ % Break this up into chunks since things hang even with the % ActionIfGPUFails option. Try 3 times 512,256,128 - % refPrj = zeros(sTX,sTY,iTLT, 'single'); for iSave = 1 keepItRunning = 1; @@ -1031,26 +912,23 @@ while (keepItRunning) - inc = 0:mapBackRePrjSize:sTY-1; if inc(end) < sTY-1 inc = [inc,sTY-1]; end - inc nChunks = length(inc)-1; for iChunk = 1:nChunks if iChunk == 1 - if exist(outputStackName,'file') fprintf('removing %s\n',outputStackName); system(sprintf('rm %s',outputStackName)); end % Special case, initialize the full sized volume and the % header but don't actually reproject anything. - fprintf('Initializing volume %d/%d with size %d\n',... - iChunk,nChunks,mapBackRePrjSize); + fprintf('Initializing volume %d/%d with size %d\n', iChunk, nChunks, mapBackRePrjSize); + rePrjFileName = sprintf('%smapBack%d/%s_%d_rePrj.sh',mbOUT{1:3},iSave); reModFileName = sprintf('%smapBack%d/%s_%d_reMod.sh',mbOUT{1:3},iSave); reProjFile = fopen(rePrjFileName,'w'); @@ -1199,7 +1077,7 @@ 'EOF'],tiltList{1}, mbOUT{1:3}, maxZ, ... mbOUT{1:3},... mbOUT{1:3},... - pixelSize./10, flgInvertTiltAngles,... % Ang --> nm + pixel_size./10, flgInvertTiltAngles,... % Ang --> nm mbOUT{1:3},... mbOUT{1:3},... mbOUT{1:3},... @@ -1257,12 +1135,13 @@ % for center of mass COM = 3; - [bx,by] = ndgrid(-COM:COM,-COM:COM) + [bx,by] = ndgrid(-COM:COM,-COM:COM); + % add optional half radius for edge case and make the padding more % logical, twice the particle radius, and then CTF size using mulit_iter % with an optimization step particlePad = 1.5; - tileRadius = floor(particlePad.*PARTICLE_RADIUS); + tileRadius = floor(particlePad.*particle_radius); tileSize = (2.*tileRadius + 1).*[1,1]; CTFSIZE = BH_multi_iterator([2.*tileSize,1], 'fourier'); @@ -1276,14 +1155,14 @@ if (eraseMask) peakMask = BH_mask3d(eraseMaskType,CTFSIZE,eraseMaskRadius,[0,0],'2d'); else - peakMask = BH_mask3d('sphere',CTFSIZE,peakSearchRad,[0,0],'2d'); + peakMask = BH_mask3d('sphere',CTFSIZE,peak_search_radius,[0,0],'2d'); end peakMask(peakMask < 0.99) = 0; - bandPassPrj = BH_bandpass3d([sTX,sTY,1],0,0,lowPassCutoff,'cpu',pixelSize); + bandPassPrj = BH_bandpass3d([sTX,sTY,1],0,0,lowPassCutoff,'cpu',pixel_size); diagnosticCell = cell(nPrjs,1); evalMaskCell = cell(nPrjs,1); @@ -1294,13 +1173,13 @@ % Any large shifts should be obvious in the original alignment, so only % look around +/- this value - globalPeak = max(2,ceil(10/pixelSize)); + globalPeak = max(2,ceil(10/pixel_size)); globalPeak = globalPeak + mod(globalPeak,2); globalPeakMask = zeros([sTX,sTY,1],'single'); - globalPeakMask(originPrj(1) -globalPeak : originPrj(1) + globalPeak,... - originPrj(2) -globalPeak : originPrj(2) + globalPeak) = 1; + globalPeakMask(originRec(1) -globalPeak : originRec(1) + globalPeak,... + originRec(2) -globalPeak : originRec(2) + globalPeak) = 1; globalBinary = ( globalPeakMask > 0 ); % Zero and only changed if CTF is refined. @@ -1455,7 +1334,7 @@ % FIXME if (emc.whitenProjections) - whitenBP = [2*PARTICLE_RADIUS,lowPassCutoff,pixelSize,PARTICLE_RADIUS]; + whitenBP = [2*particle_radius,lowPassCutoff,pixel_size,particle_radius]; [dataPrj,NPS] = BH_whitenNoiseSpectrum(dataPrj,'',whitenBP,1); % Create a matched filter. refPrj = refPrj ./ NPS; NPS = []; @@ -1524,11 +1403,11 @@ - estPeak = [mRx, mRy] - originPrj(1:2) + [comPRJX, comPRJY]; + estPeak = [mRx, mRy] - originRec(1:2) + [comPRJX, comPRJY]; glbList = fopen(sprintf('%smapBack%d/%s_%03d.global',mbOUT{1:3},iPrj),'w'); % Add unique indicies to prevent ambiquity when comparing with paral fprintf(glbList,'%f degree tilt at %f %f\n', TLT(iPrj,4),estPeak); - dataPrj = BH_resample2d(dataPrj,[0,0,0],[estPeak,0],'Bah',METHOD,'inv',1,size(dataPrj)); + dataPrj = BH_resample2d(dataPrj,[0,0,0],[estPeak,0],'Bah','GPU','inv',1,size(dataPrj)); %mapBack%d, imshow3D(gather(fftshift(real(ifftn(fftn(dataPrj).*conj(fftn(refPrj))))))); @@ -1573,8 +1452,8 @@ % ox = floor(wrkFid(iFid,3)) - tileRadius; % oy = floor(wrkFid(iFid,4)) - tileRadius; - oxEval = [floor(wrkFid(iFid,3) - PARTICLE_RADIUS),floor(wrkFid(iFid,3) + PARTICLE_RADIUS)]; - oyEval = [floor(wrkFid(iFid,4) - PARTICLE_RADIUS),floor(wrkFid(iFid,4) + PARTICLE_RADIUS)]; + oxEval = [floor(wrkFid(iFid,3) - particle_radius),floor(wrkFid(iFid,3) + particle_radius)]; + oyEval = [floor(wrkFid(iFid,4) - particle_radius),floor(wrkFid(iFid,4) + particle_radius)]; % it would be good to try a smaller tile. % First check that the data are found in this given projection @@ -1623,11 +1502,11 @@ dfA = wrkPar(iFid,6); if (calcCTF) - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixelSize),pixelSize]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixelSize),pixelSize])); + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixel_size),pixel_size]),'fwd'); + refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixel_size),pixel_size])); else - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixelSize]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixelSize])); + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixel_size]),'fwd'); + refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixel_size])); end bestScore = -1000000; @@ -1646,7 +1525,7 @@ if (use_PCF) cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); else - % % % % % cccMap = peakMask.*real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(dataTile,1,0,[0,300,lowPassCutoff,pixelSize]).*conj(bhF.fwdFFT(refTile,1,0) .* iCTF),'fwd'))); + % % % % % cccMap = peakMask.*real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(dataTile,1,0,[0,300,lowPassCutoff,pixel_size]).*conj(bhF.fwdFFT(refTile,1,0) .* iCTF),'fwd'))); end cccMap = peakMask.*real(bhF.invFFT(cccMap)); @@ -1697,8 +1576,8 @@ dCTF = imDefC; % fprintf('New best score %3.6f for defocus shift %3.3eAng\n', bestScore, defShiftVect(dCTF)); - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixelSize]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixelSize])); + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixel_size]),'fwd'); + refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixel_size])); iRefCTF = refFT .* ... mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... @@ -1762,19 +1641,7 @@ fprintf(defShifts,'%6.6e\n',defocusShifts{iPrj}); end end - % evalMaskStack = zeros(sTX,sTY,nPrjs); - % diagnosticStack = zeros([(globalPeak.*2+1).*[1,1],nPrjs],'single'); - % for iPrj = 1:nPrjs - % diagnosticStack(:,:,iPrj) = gather(diagnosticCell{iPrj}); - % evalMaskStack(:,:,iPrj) = int16(gather(evalMaskCell{iPrj})); - % end - % if (bh_global_save_tomoCPR_diagnostics) - % diagnosticStack = zeros([(globalPeak.*2+1).*[1,1],nPrjs],'single'); - % for iPrj = 1:nPrjs - % diagnosticStack(:,:,iPrj) = gather(diagnosticCell{iPrj}); - % evalMaskStack(:,:,iPrj) = int16(gather(evalMaskCell{iPrj})); - % end - % end + clear diagnosticCell evalMaskCell % if ~(conserveDiskSpace) && bh_global_save_tomoCPR_diagnostics @@ -1812,7 +1679,7 @@ fFull = fCombine; fDefFull = [fCombine,zeros(size(fCombine,1),1)]; - % % % % % fDefFull(:,2:3) = fDefFull(:,2:3).*pixelSize; + % % % % % fDefFull(:,2:3) = fDefFull(:,2:3).*pixel_size; fDefFull(:,2:3) = fDefFull(:,2:3).*samplingRate; for iPrj = 1:nPrjs % Create a file that has the X,Y,defocus positions for all fiducials @@ -1838,7 +1705,7 @@ % The model ends up seeing the pixel size as 1, so even though it loads % properly on the full aligned stack, these coords need to be scaled by % the pixel size since this is the input to tiltalign. - fFull(:,2:3) = fFull(:,2:3).*pixelSize; + fFull(:,2:3) = fFull(:,2:3).*pixel_size; fprintf(fidCombine,'%d %4.4f %4.4f %d\n',fCombine'); fclose(fidCombine); @@ -1857,56 +1724,6 @@ RotDef = 5; TltDef = 4; aliCom = fopen(sprintf('%smapBack%d/%s.align',mbOUT{1:3}),'w'); - % % % % % fprintf(aliCom,['#!/bin/bash\n\n',... - % % % % % 'tiltalign -StandardInput << EOF\n',... - % % % % % 'ModelFile %smapBack%d/%s_fit-full.fid\n',... - % % % % % 'ImageSizeXandY %d,%d\n',... - % % % % % 'ImagePixelSizeXandY %f,%f\n',... - % % % % % 'ImagesAreBinned 1\n',... - % % % % % 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... - % % % % % 'OutputResidualFile %smapBack%d/%s%s.resid\n',... - % % % % % 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... - % % % % % 'OutputTiltFile %smapBack%d/%s%s.tlt\n',... - % % % % % 'OutputXAxisTiltFile %smapBack%d/%s%s.xtilt\n',... - % % % % % 'OutputTransformFile %smapBack%d/%s%s.tltxf\n',... - % % % % % 'RotationAngle 0.00\n',... % assumed to be rotated already - % % % % % 'TiltFile %s\n',... - % % % % % 'SurfacesToAnalyze 2\n',... - % % % % % 'RotOption 1\n',... % def solve all rotations - % % % % % 'RotDefaultGrouping 3\n',... % if rot option --> 5 use def group size - % % % % % 'TiltOption %d\n',... % Tilts are harder use automapping - % % % % % 'TiltDefaultGrouping %d\n',... - % % % % % 'MagOption 1\n',... % def solve all mags - % % % % % 'MagDefaultGrouping 3\n',... - % % % % % 'XStretchOption 0\n',... - % % % % % 'SkewOption 0\n',... - % % % % % 'BeamTiltOption 0\n',... - % % % % % 'XTiltOption 0\n',... - % % % % % 'ResidualReportCriterion 0.001\n',... - % % % % % 'ShiftZFromOriginal\n',... - % % % % % 'AxisZShift 0.0\n',... - % % % % % 'RobustFitting\n',... - % % % % % 'KFactorScaling %3.3f\n',... - % % % % % 'LocalAlignments\n',... - % % % % % 'LocalRotOption 1\n',... - % % % % % 'LocalRotDefaultGrouping 3\n',... - % % % % % 'LocalTiltOption %d\n',... - % % % % % 'LocalTiltDefaultGrouping %d\n',... - % % % % % 'LocalMagOption %d\n',... - % % % % % 'LocalMagDefaultGrouping 5\n',... - % % % % % 'OutputLocalFile %smapBack%d/%s%s.local\n',... - % % % % % 'TargetPatchSizeXandY %d,%d\n', ... - % % % % % 'MinFidsTotalAndEachSurface %d,%d\n',... - % % % % % 'MinSizeOrOverlapXandY 0.5,0.5\n',... - % % % % % 'LocalOutputOptions 1,1,1\n', ... - % % % % % 'EOF'],mbOUT{1:3},fullTiltSizeXandY,... - % % % % % fullPixelSize,fullPixelSize,... - % % % % % mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,... - % % % % % mbOUT{1:3},outCTF,mbOUT{1:3},outCTF,mbOUT{1:3},outCTF, ... - % % % % % iRawTltName,tiltAliOption(1:2),... - % % % % % 10 / sqrt(nFidsTotal),tiltAliOption(3:4),flgLocalMag, ... - % % % % % mbOUT{1:3},outCTF,targetPatchSize,targetPatchSize,... - % % % % % nFiducialsPerPatch,floor(nFiducialsPerPatch/3)); if (emc.shift_z_to_to_centroid) final_line1 = 'ShiftZFromOriginal'; @@ -1920,7 +1737,7 @@ mbOutAlt = mbOUT; tilt_script_name = iRawTltName; - if (flgAltRun) + if (multi_node_run) mbOutAlt{1} = 'cache/'; [~,tn2,tn3] = fileparts(iRawTltName); tilt_script_name = sprintf('cache/mapBack%d/%s%s',mbOUT{2},tn2,tn3); @@ -1970,7 +1787,7 @@ '%s\n',... '%s\n',... 'EOF'],iTiltSeries,mbOutAlt{1:3},fullTiltSizeXandY,... - fullPixelSize,fullPixelSize,... + unsampled_pixel_size,unsampled_pixel_size,... mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... tilt_script_name,n_surfaces, ... @@ -2004,7 +1821,7 @@ system(sprintf('chmod a=wrx %smapBack%d/%s.align',mbOUT{1:3})); if (is_first_run) - if ( flgAltRun ) + if ( multi_node_run ) fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'w'); fprintf(fOUT,['%smapBack%d/%s.align > ',... '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); @@ -2022,7 +1839,7 @@ fclose(fOUT); is_first_run = false; else - if ( flgAltRun ) + if ( multi_node_run ) fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'a'); else fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}), 'a'); @@ -2051,13 +1868,13 @@ end % loop over tilts + if ( flgRunAlignments ) - + mainFile = sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}); altFiles = sprintf('%smapBack%d/runAlignments_*.sh',mbOUT{1:2}); - - if (flgAltRun) + if (multi_node_run) fOUT = fopen(mainFile,'w'); fprintf(fOUT,'#!/bin/bash\n\n'); fclose(fOUT); @@ -2066,13 +1883,11 @@ system(sprintf('cat %s >> %s',altFiles,mainFile)); end - fOUT = fopen(mainFile,'a'); fprintf(fOUT,'\nwait\n'); fclose(fOUT); system(sprintf('chmod a=wrx %smapBack%d/runAlignments.sh', mbOUT{1:2})); - system(sprintf('%smapBack%d/runAlignments.sh', mbOUT{1:2})); end diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 3187c58e..2879998e 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -297,13 +297,10 @@ tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - - flgLoad = 0; - [~,tomoReconCoords] = BH_multi_loadOrBuild(tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate, 1,reconScaling,flgLoad, 'tomoCPR'); - + % reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); + tomoReconCoords = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); + + originVol = floor(tomoReconCoords(1,1:3)./2) + 1; reconShift = tomoReconCoords(2,1:3); diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index dabf4f35..4646c837 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -76,7 +76,6 @@ emc.track_stats = false; end end -fprintf('track stats is %d\n',emc.track_stats) %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of @@ -363,7 +362,7 @@ end [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc] = ... - BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, emc.nGPUs, sizeCalc(1), emc.nCpuCores); @@ -752,13 +751,14 @@ tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - + reconGeometry = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); + if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) volumeData = []; else reconScaling = 1; - [ volumeData, reconGeometry ] = BH_multi_loadOrBuild( ... + [ volumeData, ~ ] = BH_multi_loadOrBuild( ... tomoList{iTomo}, ... reconCoords, mapBackIter, ... samplingRate,iGPUidx,reconScaling,0); diff --git a/transformations/BH_multi_loadOrBin.m b/transformations/BH_multi_loadOrBin.m index a8e7b820..b4140875 100755 --- a/transformations/BH_multi_loadOrBin.m +++ b/transformations/BH_multi_loadOrBin.m @@ -1,5 +1,4 @@ -function [ IMG_OUT, iPixelHeader, iOriginHeader, imgExt ] = ... - BH_multi_loadOrBin( IMG, SAMPLING,DIMENSION, varargin ) +function [ IMG_OUT, iPixelHeader, iOriginHeader, imgExt ] = BH_multi_loadOrBin( IMG, SAMPLING, DIMENSION, varargin ) %Check to see if a cached binned image exists, either load or bin and load. % Switched to using imod's newstack and binvol to create binning and % removed inline binning from my workflow. @@ -9,12 +8,6 @@ imgExt = ''; flgLoad = 0; -if nargin > 3 - flgMedianFilter = varargin{1}; -else - flgMedianFilter = 0; -end - if SAMPLING < 0 samplingRate = abs(SAMPLING); IMG_OUT = ''; @@ -37,7 +30,7 @@ randIDX = randi([1,10^10],1); if samplingRate > 1 - nameOUT = sprintf('cache/%s_bin%d%s', imgName, samplingRate,imgExt); + nameOUT = sprintf('cache/%s_bin%d%s', imgName, samplingRate, imgExt); doCalc = 0; if exist(nameOUT,'file') fprintf('Using cached file %s_bin%d%s\n', imgName, samplingRate,imgExt); @@ -60,7 +53,6 @@ system(sprintf('binvol -BinningFactor %d -antialias 6 %s cache/%s_bin%d%s > /dev/null', ... samplingRate,IMG, imgName, samplingRate,imgExt)); case 2 - sprintf('%s',IMG) try tiltObj = MRCImage(IMG,0); catch diff --git a/transformations/BH_multi_loadOrBuild.m b/transformations/BH_multi_loadOrBuild.m index 9cd567d1..e7d3f05c 100755 --- a/transformations/BH_multi_loadOrBuild.m +++ b/transformations/BH_multi_loadOrBuild.m @@ -1,7 +1,10 @@ -function [ IMG_OUT, reconGeom ] = BH_multi_loadOrBuild(tomoName, ... - rCoords, mapBackIter, ... - SAMPLING, gpuIDX,... - reconScaling, varargin) +function [ IMG_OUT, reconGeom ] = BH_multi_loadOrBuild( tomoName, ... + rCoords, ... + mapBackIter, ... + SAMPLING, ... + gpuIDX,... + reconScaling, ... + varargin) %Check to see if a cached binned image exists, either load or reconstruct % Switched to using imod's newstack and binvol to create binning and % removed inline binning from my workflow. @@ -21,8 +24,6 @@ if nargin > 6 flgLoad = varargin{1}; -else - flgLoad = varargin{1}; end recon = ''; @@ -48,8 +49,8 @@ tomoName = strjoin(nameSplit(1:end-1),'_') tomoNumber = EMC_str2double(nameSplit{end}) -% % % rCoords = subTomoMeta.mapBackGeometry.(tomoName).coords(tomoNumber,:); rCoords = rCoords ./ SAMPLING; +% fix is like floor but rounds towards zero, not sure why I'm doing this here anymore. rCoords(1:4) = fix(rCoords(1:4)); @@ -202,10 +203,14 @@ function [ reconGeom ] = calc_rg( header, rCoords ) - oY = ceil((header.nY +1)./2); + % Origin in the binned tilt series + oY = 1 + floor(header.nY ./ 2); + % Size in the binned tilt series nY = rCoords(3) - rCoords(2) + 1; - dY = floor(rCoords(2)+nY/2) -oY; + % Origin in the reconstructed area (active shift from origin in tilt series) + dY = floor(rCoords(2) + nY/2) - oY; reconGeom = zeros(2,3); + % FIXME index 4 and 5 should be 3 and 4 reconGeom(1,1:3) = [rCoords(1), nY, rCoords(4)]; % value specify location of origin, but SHIFT in IMOD's tilt takes the % location to shift the origin too, so multiply oX by -1. The notion for Z is From 4e3465b362a5d4ece126e21f58bb669a505f4bf8 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 15 Feb 2024 13:29:30 -0500 Subject: [PATCH 024/151] Fixing ctf convention, emClarity will NOT work with overfocus now as the abs value is taken to be compatible with existing projects. This can be removed at some point in the future or with a allow over focus option --- alignment/BH_templateSearch3d_2.m | 47 +-- coordinates/BH_multi_calcBinShift.m | 27 +- ctf/BH_ctfCalc.m | 7 +- ctf/BH_ctf_Correct3d.m | 293 ++++++------- ctf/BH_ctf_Estimate.m | 10 +- ctf/BH_ctf_Refine2.m | 8 +- masking/BH_weightMask3d.m | 475 ---------------------- masking/BH_weightMaskMex.m | 2 +- mexFiles/include/core_headers.cuh | 8 +- mexFiles/utils/ctf.cu | 6 +- statistics/BH_multi_loadAndMaskStack.m | 8 +- statistics/BH_pcaPub.m | 15 +- synthetic/BH_synthetic_mapBack.m | 107 ++--- testScripts/fourierCtfRec.m | 249 ------------ testScripts/fourierCtfRecTex.m | 338 --------------- transformations/BH_multi_loadOrBin.m | 52 +-- transformations/BH_multi_loadOrBuild.m | 2 +- transformations/emc_get_max_specimen_NZ.m | 40 ++ transformations/emc_get_origin_index.m | 12 + 19 files changed, 262 insertions(+), 1444 deletions(-) delete mode 100755 masking/BH_weightMask3d.m delete mode 100644 testScripts/fourierCtfRec.m delete mode 100644 testScripts/fourierCtfRecTex.m create mode 100644 transformations/emc_get_max_specimen_NZ.m create mode 100644 transformations/emc_get_origin_index.m diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index fc370d91..0f43cbc2 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -172,16 +172,6 @@ scale_mip = false; end -% Limit to the first zero if we are NOT using the CTF rec -if ~( use_ctf3d_templateSearch ) - TLT = load(sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tomoName,mapBackIter+1)); - def = mean(-1.*TLT(:,15))*10^6; %TODO if you switch to POSITIVEDEFOCUS this will be wrong - firstZero = -0.2*def^2 +5.2*def +11; - % FIXME: if you have a ctf 3d tomo why limit to the firsts zero? - % Take the lower of firstZero lowResCut or Nyquist - bp_vals(3) = max(bp_vals(3), firstZero); - fprintf('\nUsing max (%f) of specified resolution cutoff of %f and first ctf zero %f Angstrom\n',bp_vals(3), wantedCut, firstZero); -end if pixelSize*2 > bp_vals(3) fprintf('\nLimiting to Nyquist (%f) instead of user requested low pass cutoff %f Angstrom\n',pixelSize*2,bp_vals(3)); @@ -232,8 +222,7 @@ -[template, tempPath, tempName, tempExt] = ... - BH_multi_loadOrBin( TEMPLATE, 1, 3 ); +[template, tempPath, tempName, tempExt] = BH_multi_loadOrBin( TEMPLATE, 1, 3, true ); % Bandpass the template so it is properly normalized bp_vals @@ -417,40 +406,6 @@ [ OUTPUT ] = BH_multi_iterator( [sizeChunk;kVal.*[1,1,1]], 'extrapolate' ); -% -% switch wedgeType -% case 1 -% % make a binary wedge -% [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... -% 'binaryWedgeGPU',particleThickness,... -% 1, 1, samplingRate); -% case 2 -% % make a non-CTF wedge -% [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... -% 'applyMask',particleThickness,... -% 2, 1, samplingRate); -% case 3 -% % make a CTF without exposure weight -% [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... -% 'applyMask',particleThickness,... -% 3, 1, samplingRate); -% case 4 -% % make a wedge with full-ctf -% [ wedgeMask ]= BH_weightMask3d(-1.*OUTPUT(1,:), tiltGeometry, ... -% 'applyMask',particleThickness,... -% 4, 1, samplingRate); -% otherwise -% error('wedgeType must be 1-4'); -% end -% -% wedgeMask = (ifftshift(wedgeMask)); -% -% % Now just using the mask to calculate the power remaining in the template, -% % without actually applying. -% wedgeMask = gather(find(ifftshift(wedgeMask))); - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Preprocess the tomogram tomoIDX = 1; nTomograms = prod(nIters); diff --git a/coordinates/BH_multi_calcBinShift.m b/coordinates/BH_multi_calcBinShift.m index 5bc41d50..9aadda23 100644 --- a/coordinates/BH_multi_calcBinShift.m +++ b/coordinates/BH_multi_calcBinShift.m @@ -1,4 +1,4 @@ -function [ binSize, binShift ] = BH_multi_calcBinShift(coords, isTilt, samplingRate) +function [ binSize, binShift ] = BH_multi_calcBinShift(coords, samplingRate) % Address fractional shifts on binning % Coordinates are stored relative to the lower left corner of the full % tilt/tomo. On binning a shift is needed to keep that origin in the same @@ -7,24 +7,17 @@ % way to do this is to shift the data on binning in 2d, and leave the % coordinates alone. -if (isTilt) - % Expecting just the x,y,z for a tilt series and the binning. Also may - % shift to have an odd dimension so that Imod origin is always the same. - binSize = floor(coords./samplingRate); - binSize = binSize - (1-mod(binSize,2)); - - originFull = floor(coords ./2) + 1; - originBin = floor(binSize./2) + 1; - % This is the shift we need to apply to the binned image to make sure - % that the origin is in the same place. - binShift = -1.*(samplingRate.*originBin - originFull) ./ samplingRate; - -end +% Expecting just the x,y,z for a tilt series and the binning. Also may +% shift to have an odd dimension so that Imod origin is always the same. +binSize = floor(coords./samplingRate); +binSize = binSize - (1-mod(binSize,2)); -% tmpShift = (coords-fix(coords./samplingRate).*samplingRate); -% binShiftTemplateSearch = [tmpShift(1), tmpShift(3) + tmpShift(2),tmpShift(4)]; -% binShift = binShiftTemplateSearch ./ samplingRate; +originFull = floor(coords ./2) + 1; +originBin = floor(binSize./2) + 1; +% This is the shift we need to apply to the binned image to make sure +% that the origin is in the same place. +binShift = -1.*(samplingRate.*originBin - originFull) ./ samplingRate; end diff --git a/ctf/BH_ctfCalc.m b/ctf/BH_ctfCalc.m index dbb360ec..dcd700cf 100755 --- a/ctf/BH_ctfCalc.m +++ b/ctf/BH_ctfCalc.m @@ -129,9 +129,12 @@ % df1 should be defocus of greater mag and phi0 -90/90 % phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 + DF.*WL.*(radialGrid).^2); +if (df1 - df2 < 0) + error('df1 must be greater than df2') +end + dfTerm = 0.5.*( (df1+df2) + (df1-df2)*cos(2.*(phi-phi0)) ); -phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 + ... - WL.*(radialGrid).^2 .* dfTerm); +phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 - WL.*(radialGrid).^2 .* dfTerm); % dPdQ = 2*pi*CS*WL^3.*radialGrid.^3 + 2*WL.*radialGrid.*dfTerm; if ( flgComplex ) ctfMask = exp(-1i.*(phasePerturbation-atan2(AMPCONT,sqrt(1+AMPCONT)))); diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 058bc3a9..104eee59 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -378,15 +378,18 @@ iterList{gpuList(iGPU)} = iGPU+(tiltStart-1):nGPUs:nTilts; iterList{gpuList(iGPU)}; end -try - EMC_parpool(nGPUs) -catch - delete(gcp('nocreate')) - EMC_parpool(nGPUs) -end - +% FIXME +% try +% EMC_parpool(nGPUs) +% catch +% delete(gcp('nocreate')) +% EMC_parpool(nGPUs) +% end + +% FIXME +% parfor iGPU = 1:nGPUs +for iGPU = 1:nGPUs -parfor iGPU = 1:nGPUs for iTilt = iterList{gpuList(iGPU)} iTomoList = {}; @@ -443,8 +446,9 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. -parfor iGPU = 1:nGPUs% - %for iGPU = 1:nGPUs +% FIXME +% parfor iGPU = 1:nGPUs + for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); % Loop over each tilt for iTilt = iterList{gpuList(iGPU)} @@ -524,7 +528,7 @@ % % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s ',samplingRate,fullStack,inputStack); % % system(binCMD); - BH_multi_loadOrBin(fullStack,-1.*samplingRate,2); + BH_multi_loadOrBin(fullStack, samplingRate, 2, false); end else @@ -567,7 +571,7 @@ else dampeningMax = 0.90; [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... - TLT(1,17),TLT(1,18),TLT(1,15), ... + TLT(1,17),TLT(1,18),abs(TLT(1,15)), ... 2048, TLT(1,19), ... resTarget,maxZ*10, ... dampeningMax,CYCLE); @@ -584,27 +588,22 @@ % max odd number nSections = nSections + ~mod(nSections,2); end - fprintf('with %3.3f nm sections, correcting %d tilt-series\n',ctf3dDepth*10^9,nSections); + fprintf('with %3.3f nm sections, correcting %d tilt-series\n', ctf3dDepth*10^9, nSections); % For each tomo create a list of slices that are to be reconstructed % for every section section. - [ sectionList ] = calcTomoSections(iCoords, tomoNumber,emc.pixel_size_angstroms, ... - nSections,tiltList{iTilt}, ctf3dDepth); + [ sectionList ] = calcTomoSections(iCoords, tomoNumber, emc.pixel_size_angstroms, nSections, tiltList{iTilt}, ctf3dDepth); if (recWithoutMat) avgZ = 0; surfaceFit = 0; - else - [ avgZ, maxZ, tomoNumber, surfaceFit ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, emc.pixel_size_angstroms, ... - samplingRate, cycleNumber,... - sectionList,0); - - + iTomoList,nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + sectionList, 0); end if ( shiftDefocusOrigin ) @@ -616,36 +615,11 @@ end - - - - % Correct a tilt series for earch section which requires writing each to - % disk for use of IMOD. - if (mapBackIter) - tiltErrorFile = sprintf('mapBack%d/%s_ali%d_ctf.beamTiltError', ... - mapBackIter,tiltList{iTilt}, mapBackIter); - try - tiltError = load(tiltErrorFile); - if numel(tiltError) ~= 1 - error('tiltError should be a single number in degrees.\n'); - else - fprintf('\nUsing %f degrees for beam tilt error.\n',tiltError) - end - - catch - fprintf('\nTiltErrorFile %s not found.\n', tiltErrorFile); - fprintf('\nUsing 0 degrees for beam tilt error.\n',tiltErrorFile) - tiltError = 0; - end - - - else - tiltError = 0; - end + fprintf('Using a avgZ of %3.3e nm\n',avgZ); + for iSection = 1:nSections - defFitFull = ''; preCombDefocus = 0; if (mapBackIter) @@ -659,13 +633,10 @@ end end - - if (PosControl2d) correctedStack = maskedStack; else - % I would have thought the global would be recognized, but it looks % like there is something odd about its use with a parfor loop % FIXME, when setting up the iterator, make clean copies for each @@ -690,8 +661,7 @@ end - outputStack = sprintf('%s/%s_ali%d_%d.fixed', ... - tmpCache,tiltList{iTilt},mapBackIter+1,iSection) + outputStack = sprintf('%s/%s_ali%d_%d.fixed', tmpCache,tiltList{iTilt},mapBackIter+1,iSection); SAVE_IMG(correctedStack,outputStack,emc.pixel_size_angstroms); correctedStack = []; @@ -740,7 +710,6 @@ end - fprintf('Local file %s\n',LOCAL); if exist(LOCAL,'file') flgLocal = 1; @@ -920,7 +889,7 @@ inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed',... STACK_PRFX,mapBackIter+1,suffix,samplingRate); if ~exist(inputStack, 'file') - BH_multi_loadOrBin(fullStack,-1.*samplingRate,2,flgMedianFilter); + BH_multi_loadOrBin(fullStack, samplingRate, 2, false); end @@ -930,31 +899,38 @@ function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, pixel_size_angstroms, nSections,tiltName, ctf3Depth) nTomos = length(tomoNumber); -sectionList = cell(nTomos,1); +sectionList = cell(nTomos,1) for iTomo = 1:nTomos % min and max in absolute pixels min and max from 1:nZrecon - sectionList{iTomo} = zeros(nSections,6); + sectionList{iTomo} = zeros(nSections,6) end % With rounding this could end up a bit short except the top and bottom are both % half a section larger than minimally needed. -nSec = floor(ctf3Depth*10^10/pixel_size_angstroms) ; -nSec = nSec + ~mod(nSec,2); -halfSec = (nSec-1)/2; +n_pixels_per_slab = floor(ctf3Depth*10^10/pixel_size_angstroms) +n_pixels_per_slab = n_pixels_per_slab + ~mod(n_pixels_per_slab,2) + +halfSec = (n_pixels_per_slab-1)/2 for iT = 1:length(tomoNumber) iTomo = tomoNumber(iT); % Origin + originshift - -1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6), - reconRange = floor([-1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6),0]); + oZ_in_tomo_frame = emc_get_origin_index(iCoords(iTomo,4)); + % iCoords(iTomo,6) is the origin of the specimen with respect to the origin of the tomogram + % We want the origin of the tomogram with respect to the origin of the specimen and then calculate the lower + % bound of the reconstruction range as half the volume lower than that + oZ_in_specimen_frame = -1*iCoords(iTomo,6); + reconRange = floor([oZ_in_specimen_frame - oZ_in_tomo_frame,0]); reconRange(2) = reconRange(1) + iCoords(iTomo,4) - 1; nZ = 1; flgFirstSec = 1; + % nSections is always ODD + % FIXME: The half section (slab) shift seems precarious, I don't remember why this is set up this way. for iSection = 1:nSections - sectionCenter = ((nSections-1)/-2+(iSection-1))*(nSec-1); + sectionCenter = ((nSections-1)/-2+(iSection-1))*(n_pixels_per_slab-1); % Check that sectionCenter is within range if sectionCenter + halfSec < reconRange(1) || ... @@ -1065,9 +1041,16 @@ % the center of mass of subtomograms in Z to the focal plane, rather than % the center of mass of the tomograms (specimen) -defocusOffset = (((nSections-1)/-2+(iSection-1))*ctf3dDepth); -fprintf('using offset %3.3e for section %d with COM offset %3.3e\n',defocusOffset,iSection,avgZ); -defocusOffset = (defocusOffset - avgZ)*(1-useSurfaceFit); % The average height of the particles is factored into the surface fit +if (useSurfaceFit) + defocusOffset = 0; +else + defocusOffset = (((nSections-1)/-2+(iSection-1))*ctf3dDepth); + fprintf('Not using surface fit, so using offset %3.3e for section %d with COM offset %3.3e\n', defocusOffset, iSection, avgZ); + % Assuming the majority of the fit defocus came from the subtomograms, then the estimated defocus value needs to be moved from + % the origin of the specimen to the origin of the subtomograms. + defocusOffset = (defocusOffset + avgZ); % The average height of the particles is factored into the surface fit +end + % The avg Z seems like it should be added? @@ -1117,8 +1100,8 @@ for iPrj = 1:nPrjs maxEval = cosd(TLT(iPrj,4)).*(d1/2) + maxZ./2*abs(sind(TLT(iPrj,4))); - oX = ceil((d1+1)./2); - oY = ceil((d2+1)./2); + oX = emc_get_origin_index(d1); + oY = emc_get_origin_index(d2); iEvalMask = floor(oX-maxEval):ceil(oX+maxEval); if ( applyExposureFilter ) @@ -1128,23 +1111,11 @@ end iExposureFilter = iExposureFilter .* bpFilter; - - - - - STRIPWIDTH = min(floor((0.5*ctf3dDepth/(pixel_size_angstroms*10^-10))/abs(tand(TLT(iPrj,4)))),512); - STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); - % take at least 1200 Ang & include the taper if equal to STRIPWIDTH - tileSize = floor(max(600./pixel_size_angstroms, STRIPWIDTH + 28)); - tileSize = tileSize + mod(tileSize,2); - %fprintf('stripwidth tilesize %d %d\n',STRIPWIDTH,tileSize); - incLow = ceil(tileSize./2); - incTop = tileSize - incLow; - border = ceil(incLow+apoSize)+1; + ddF = TLT(iPrj,12); dPhi = TLT(iPrj,13); - D0 = TLT(iPrj,15); + D0 = abs(TLT(iPrj,15)); %TLT(iPrj,16); @@ -1160,8 +1131,6 @@ [rX,rY,~] = BH_multi_gridCoordinates([d1,d2],'Cartesian','GPU',{'none'},0,1,0); % Assuming the plane fit is from the origin as it is. - - if (useSurfaceFit) %rZ = (surfaceFit.p00 + surfaceFit.p10.*(rX+oX)) + surfaceFit.p01.*(rY+oY); try @@ -1177,16 +1146,19 @@ else rZ = zeros([d1,d2],'single','gpuArray'); end + + defocus_adj = D0 - (defocusOffset.*cosd(TLT(iPrj,4))); - full_defocusOffset = ((defocusOffset.*cosd(TLT(iPrj,4))) + D0); - + % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) rA = BH_defineMatrix([0,TLT(iPrj,4),0],'SPIDER','inv'); + % Transform the specimen plane - tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ +oX); - tY = round(rA(2).*rX + rA(5).*rY + rA(8).*rZ +oY); - tZ = (pixel_size_angstroms*10^-10).*(rA(3).*rX + rA(6).*rY + rA(9).*rZ) + full_defocusOffset; - + tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ + oX); + tY = round(rA(2).*rX + rA(5).*rY + rA(8).*rZ + oY); + % undefocus is positive, but we have stored the negative value (so we just add this to the positional offset) + tZ = defocus_adj - (pixel_size_angstroms*10^-10).*(rA(3).*rX + rA(6).*rY + rA(9).*rZ); + % Some edge pixels can be out of bounds depending on the orientation of % the plan fit. Setting to zero will will ignore them (assuming defocus % is always < 0) @@ -1194,15 +1166,16 @@ minDefocus = min(tZ(:)); - maxDefocus = max(tZ(tZ<1)); + maxDefocus = max(tZ(tZ < 1)); % Spit out some info % fprintf('Found a min/max defocus of %3.3e/ %3.3e for tilt %d (%3.3f deg)\n',minDefocus,maxDefocus,iPrj,TLT(iPrj,4)); % To track sampling in case I put in overlap samplingMask = zeros([d1,d2],'single','gpuArray'); + % TODO: confirm the astigmatism is correct, check - and +/- 90 for iDefocus = minDefocus-ctf3dDepth/1:ctf3dDepth/1:maxDefocus+ctf3dDepth/1 - defVect = [iDefocus - ddF, iDefocus + ddF, dPhi]; + defVect = [iDefocus + ddF, iDefocus - ddF, dPhi]; if (phakePhasePlate(1) > 0) @@ -1231,7 +1204,7 @@ end end - + if (flgWhitenPS(3)) tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz./(abs(Hqz).^2+flgWhitenPS(3)))),trimVal(1,:),trimVal(2,:),'GPU','single'); else @@ -1275,41 +1248,32 @@ end function [avgZ, maxZ, tomoNumber,surfaceFit] = calcAvgZ(masterTM,iCoords, ... - tiltName,tomoList,... - nTomos, pixel_size_angstroms,... - samplingRate,cycleNumber,... - sectionList,calcMaxZ) + tiltName,tomoList,... + nTomos, pixel_size_angstroms,... + samplingRate,cycleNumber,... + sectionList,calcMaxZ) % Calculate the maximum extensions in Z and then how many separate sections % need to be corrected. surfaceFit = ''; avgZ = 0; -maxZ = 0; -tomoNumber = zeros(nTomos,1); -for iTomo = 1:nTomos - % The tomograms may not be listed monotonically so explicitly get their - % id number - - if isa(masterTM,'struct') - tomoNumber(iTomo) = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - nZdZ = iCoords(tomoNumber(iTomo),[4,6]); - else - tomoNumber(iTomo) = iTomo; - nZdZ = iCoords(iTomo,[4,6]); - end - - % half the size in z plus the shift back to the microscope coords. - sZneeded = 2.*ceil(nZdZ(1)/2+abs(nZdZ(2))+1); - if sZneeded > maxZ - maxZ = sZneeded; - end + + +if isa(masterTM,'struct') + val_to_pass = masterTM.mapBackGeometry.tomoName; +else + val_to_pass = 'dummy'; end -maxZ = maxZ + (samplingRate*2); +[ maxZ, tomoNumber ] = emc_get_max_specimen_NZ(val_to_pass, ... + iCoords * samplingRate, ... + tomoList, ... + nTomos, ... + samplingRate); -maxZ = maxZ.*pixel_size_angstroms./10; -fprintf('combining thickness and shift on tilt %s, found a maxZ %3.3f nm\n',tiltName,maxZ); +maxZ = maxZ .* pixel_size_angstroms ./ 10; +fprintf('combining the thickness and shift on tilt %s, found a maxZ %3.3f nm\n',tiltName,maxZ); if (calcMaxZ) return; @@ -1317,7 +1281,6 @@ % For now use cycle000, if adding a refinment focused on a specific set of % particles, then consider that later. - try initGeom = masterTM.(cycleNumber).RawAlign; fprintf('Loaded the geometry for RawAlign %s\n',cycleNumber); @@ -1349,94 +1312,72 @@ zFull{iSection} = []; end + for iT = 1:nTomos + iTomo = tomoNumber(iT); - micDimension = floor(masterTM.tiltGeometry.(tomoList{iT})(1,20:22) ./ samplingRate); - - % Already scaled to sampled pixels - tomoOrigin =[ ceil((iCoords(iTomo,1)+1)./2),... - ceil((iCoords(iTomo,3)-iCoords(iTomo,2))./2),... - ceil((iCoords(iTomo,4)+1)/2)]; - micOrigin = [-1*iCoords(iTomo,5), ... - (iCoords(iTomo,2) + tomoOrigin(2)) - ceil((micDimension(2)+1)/2),... - iCoords(iTomo,6)]; - + + % X in the Y frame means a vector from the Y lower left to the X origin + % X origin wrt Y origin is a vector from the origin of Y to the X origin + tomoReconCoords = masterTM.reconGeometry.(tomoList{iT}) ./ samplingRate; + tomo_origin_in_tomo_frame = emc_get_origin_index(tomoReconCoords(1,1:3)); + tomo_origin_wrt_specimen_origin = -1.*tomoReconCoords(2,1:3); + iTomoName = sprintf('%s_%d',tiltName,iTomo); + % Get the z-coordinates of the origin for all included subtomograms relative to the lower left of the tomogram % shouldn't be any removed particles at this stage but later there would be. - zList = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,13)./samplingRate; - + subtomo_origin_z_in_tomo_frame = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,13) ./ samplingRate; + + % We should not get to this point if all subtomograms have been removed. + if isempty(subtomo_origin_z_in_tomo_frame) + error('No subtomograms found for %s',iTomoName); + end + % shift from lower left to centered and include the tomos offset from the - % microscope frame - zList = zList - tomoOrigin(3) + micOrigin(3); - totalZ = totalZ + sum(zList); + subtomo_origin_wrt_specimen_origin = subtomo_origin_z_in_tomo_frame - tomo_origin_in_tomo_frame(3) + tomo_origin_wrt_specimen_origin(3); + + totalZ = totalZ + sum(subtomo_origin_wrt_specimen_origin); fprintf('%s tomo has %d subTomos with mean Z %3.3f nm\n', ... - iTomoName, length(zList), mean(zList)*pixel_size_angstroms./10); - nSubTomos = nSubTomos + length(zList); - + iTomoName, length(subtomo_origin_wrt_specimen_origin), ... + mean(subtomo_origin_wrt_specimen_origin) * pixel_size_angstroms ./ 10); + + nSubTomos = nSubTomos + length(subtomo_origin_wrt_specimen_origin); + for iSection = 1:nSections - iSecOrigin = sectionList{iT}(iSection,6); iSecRadius = sectionList{iT}(iSection,5)/2; - inSectionIDX = zList > iSecOrigin - iSecRadius & zList <= iSecOrigin + iSecRadius; - + inSectionIDX = subtomo_origin_wrt_specimen_origin > iSecOrigin - iSecRadius & subtomo_origin_wrt_specimen_origin <= iSecOrigin + iSecRadius; x = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,11)./samplingRate; - x = x - tomoOrigin(1) + micOrigin(1); + x = x - tomo_origin_in_tomo_frame(1) + tomo_origin_wrt_specimen_origin(1); y = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,12)./samplingRate; - y = y - tomoOrigin(2) + micOrigin(2); + y = y - tomo_origin_in_tomo_frame(2) + tomo_origin_wrt_specimen_origin(2); xFull{iSection} = [xFull{iSection} ; x(inSectionIDX)]; yFull{iSection} = [yFull{iSection} ; y(inSectionIDX)]; - zFull{iSection} = [zFull{iSection} ; zList(inSectionIDX)]; - - - + zFull{iSection} = [zFull{iSection} ; subtomo_origin_wrt_specimen_origin(inSectionIDX)]; end % loop over sections - clear zList + clear subtomo_origin_wrt_specimen_origin end % loop over tomos -avgZ = totalZ/nSubTomos*pixel_size_angstroms/10*10^-9; +avgZ = totalZ / nSubTomos*pixel_size_angstroms / 10*10^-9; + -% sf(x,y) = p00 + p10*x + p01*y; -% surfaceFit = fit([xFull, yFull],zFull,'poly11'); -%surfaceFit = fit([xFull, yFull],zFull,'lowess','Span',0.1); for iSection = 1:nSections if length(xFull{iSection}) >= 6 - % try - % try - % exclude = abs(mean(zFull{iSection})-zFull{iSection})>1.5.*std(zFull{iSection}); - % surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'lowess', 'Span', 0.05,'Normalize','on','Exclude',exclude); - % catch - % - % surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'lowess','Robust','on'); - % end - surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'poly22','Robust','on'); - % end - % - % figure('visible','off'), plot(surfaceFit{iSection},[xFull{iSection},yFull{iSection}],zFull{iSection}); - % saveas(gcf,sprintf('fitThis_%s_%d.pdf',tiltName,iSection)); - % close(gcf); else surfaceFit{iSection} = 0; end end -% save(sprintf('fitThis_%s_%d.mat',tiltName,iSection),'xFull','yFull','zFull','surfaceFit'); - - -fprintf('%s tilt-series has %d subTomos with mean Z %3.3f nm\n', ... - tiltName, nSubTomos,avgZ*10^9); - - - - +fprintf('%s tilt-series has %d subTomos with mean Z %3.3f nm\n', tiltName, nSubTomos,avgZ*10^9); end diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 59797657..76531128 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -126,7 +126,7 @@ % I still use the def for underfocus < 0 as this places the origin at the % focal plan in the microscope rather than on the specimen. Which makes % more sense to me. -defEST = -1.*emc.('defEstimate').*10^6 +defEST = emc.('defEstimate').*10^6 defWIN = emc.('defWindow').*10^6 tiltRange = [-1]; @@ -806,8 +806,8 @@ for iAng = coarseAngSearch for iDelDF = coarseDefSearch - df1 = maxDef*10^-6 - iDelDF*astigStep; - df2 = maxDef*10^-6 + iDelDF*astigStep; + df1 = maxDef*10^-6 + iDelDF*astigStep; + df2 = maxDef*10^-6 - iDelDF*astigStep; if (emc.pixel_size_si < 1*10^-10) @@ -846,8 +846,8 @@ for iAng = fineAngSearch for iDelDF = fineDefSearch - df1 = maxDef*10^-6 - (mDef + iDelDF); - df2 = maxDef*10^-6 + (mDef + iDelDF); + df1 = maxDef*10^-6 + (mDef + iDelDF); + df2 = maxDef*10^-6 - (mDef + iDelDF); % For values very close to zero, the search range may include % values |df1| < |df2| which is against convention. diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index 548a0b9e..43721261 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -264,7 +264,7 @@ ddZ, x1, y1, Xnew, Ynew, coordShift, ... reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine) -DFo = TLT(iPrj,15); +DFo = abs(TLT(iPrj,15)); padTileOver = 256; tmpTile = zeros(paddedSize.*[1,1]+2*padTileOver,'single','gpuArray'); @@ -391,12 +391,12 @@ function [minRes] = calcMinResolution(TLT, radialForCTF,Cs,WAVELENGTH,AMPCONT) -meanDef = mean(TLT(:,15)); +meanDef = abs(mean(TLT(:,15))); meanAst = mean(TLT(:,12)); meanAng = mean(TLT(:,13)); -df1 = meanDef - meanAst; -df2 = meanDef + meanAst; +df1 = meanDef + meanAst; +df2 = meanDef - meanAst; [ Hqz ] = BH_ctfCalc(radialForCTF,Cs,WAVELENGTH, ... [df1,df2,meanAng],size(radialForCTF{1}), ... diff --git a/masking/BH_weightMask3d.m b/masking/BH_weightMask3d.m deleted file mode 100755 index 71596467..00000000 --- a/masking/BH_weightMask3d.m +++ /dev/null @@ -1,475 +0,0 @@ -function [ WEDGE_MASK, padValue] = BH_weightMask3d(SIZE, ORIENTATION, METHOD, ... - particleRadius, flgIncCtf, ... - SYMMETRY, samplingRate) -%Create a missing wedge mask. -% -% Input variables: -% -% SIZE = size of the padded fft of the image -% -% ORIENTATION = missing wedge orientation -% [tiltangles, sample orientation], [-a, a, e1, e2, e3] -% As elsewhere the sample orientation are the ZXZ euler angles that -% transform coordinates from the microscope to the sample frame. -% -% METHOD = case sensitive 'GPU', cpu otherwise. -% -% particleRadius = Thickness of particle in pixels in beam direction, used to -% calculate a zone of influence for the mask. -% -% Output variables: -% -% WEDGEMASK = missing wedge mask ready to be applied to 3d fft -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% Goals & Limitations: -% -% As a test case, use an asymmetric wedge -50,70 in order to visualize -% any ambiguities in angles. -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% -% TODO: -% - test gpu option for function and return value. (template search requires -% return of gpu) -% -% -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -% Save the original size for binary wedge calc -binaryWedgeSize = abs(SIZE); - -if isvector(ORIENTATION) - flgCTF = 0; - tiltColumn=1; - tiltAzimuth=90; - -else - % Consider amplitude modulation by CTF (NOT CTF envelope though) - flgCTF = 1; - tiltColumn=4; - tiltAzimuth=ORIENTATION(1,6); - -end - -flgTiltWeight = 1; -flgRadial = 1; -flgSmooth = 1; - -if flgCTF && (flgIncCtf == 3) - % Don't include exposure weight, just CTF - flgNoExposure = 1; -elseif flgCTF && (flgIncCtf == 4) - % Calc full CTF - flgNoExposure = 0; -elseif flgCTF && (flgIncCtf == 2) - % Calc just the weights, which are still different than a binary wedge, but - % don't include CTF or exposure. - flgCTF = 0; -end - -if strcmp(METHOD, 'GPU') - useGPU = true; -else - useGPU = false; -end - -%SIZE = [512,512,512]; -if all(SIZE > 0) - outputScaling = SIZE(1)/512; - SIZE = [512,512,512] - % outputScaling = SIZE(1)/256; - % SIZE = [256,256,256] -else - %optional override for template matching which may not always be cubic, wich - %trades a little accuracy in the ctf mask for speed. - SIZE = abs(SIZE) - outputScaling = 1 -end - -if strcmpi(METHOD, 'applyMask') - useGPU = true; - filterMask = false; - METHOD = 'GPU'; -elseif strcmpi(METHOD, 'applyMaskcpu') - filterMask = true; - useGPU = false; - METHOD = 'cpu'; - -elseif strcmp(METHOD,'binaryWedgeGPU') || strcmp(METHOD,'binaryWedgeCpu') - if strcmp(METHOD,'binaryWedgeGPU') - METHOD = 'GPU'; - elseif strcmp(METHOD,'binaryWedgeCpu') - METHOD = 'cpu'; - else - error('binary wedge uncertain if cpu or gpu.') - end - - % Move binary wedge to a separate function - maxTilt = 90-abs(min(ORIENTATION(:,tiltColumn))); - minTilt = 90-abs(max(ORIENTATION(:,tiltColumn))); - tiltAxis= BH_defineMatrix(1.*[90-tiltAzimuth,0,0], 'Bah', 'forwardVector'); - [bX,~,bZ,~,~,~] = BH_multi_gridCoordinates(binaryWedgeSize,'Cartesian',METHOD,{'single',tiltAxis,[0,0,0]','forwardVector',1,1},0,1,0); - theta = atan2d(bX,bZ); clear bX bZ - - WEDGE_MASK = single(~( (180-minTilt<=theta | theta<=maxTilt-180) | ... - (-1.*minTilt<=theta & theta<=maxTilt) )); - - % Set the value at the origin = 0.2 - origMask = ceil((size(WEDGE_MASK)+1)./2); - WEDGE_MASK(origMask(1)-2:origMask(1)+2,:,origMask(3)-2:origMask(3)+2) = 1; - - - clear theta maxTilt minTilt tiltAxis - - [ gaussKernel ] = BH_multi_gaussian3d(9, 1.5 ); - if strcmpi(METHOD,'GPU') - gaussKernel = gpuArray(gaussKernel); - end - - - - WEDGE_MASK = convn(WEDGE_MASK, gaussKernel, 'same'); - % - % WEDGE_MASK = convn(WEDGE_MASK, gaussKernel, 'same'); - - % [ WEDGE_MASK ] = BH_multi_randomizeTaper(WEDGE_MASK); - - rad = fftshift(BH_bandpass3d(size(WEDGE_MASK),0,0,0,METHOD,'nyquist')); - WEDGE_MASK = WEDGE_MASK .* rad; - WEDGE_MASK = (WEDGE_MASK - min(WEDGE_MASK(:))); - WEDGE_MASK = WEDGE_MASK./(max(WEDGE_MASK(:))); clear rad - padValue = [0,0,0;0,0,0]; - return -else - filterMask = false; - -end - - - - - - - -padValue = [0,0,0;0,0,0]; -% Grid for the binary mask -% min / max tilt angle, inverted for reciprocal space - - - -% Minimum size to pad in each dimension so that rotation will not cause -% extrapolation. - -% % % padValue = ceil(0.5.*(sqrt(SIZE*SIZE')-SIZE)); -% % % paddedSize = SIZE + 2.*padValue - -% % % % 1/Diameter = thickness of projection in reciprocal space -% % % zoneOfInfluence = floor(SIZE(3)/(2.*THICKNESS)) -% % % [~,~,Z] = ndgrid(zeros(paddedSize(1),1), ... -% % % zeros(paddedSize(2),1), ... -% % % window(@hamming,2.*zoneOfInfluence+1)); - -% Use the shape transform to estimate the extent of the projections influence -% Even though we end up multiplying, because the projection is only a single -% pixel thick to start with, this is equivalent to convolution, st this is -% essentially accounting for creating a finite backprojection volume by -% multiplication in real space with a box, hence this sinc based convolution in -% reciprocal space.lt - -blob = BH_mask3d('sphere',SIZE, outputScaling.*[1,1,1].*particleRadius.*2,[0,0,0]); -if ~(useGPU) - blob = gather(blob); -end -BLOB = abs(fftn(blob)); clear blob -BLOB = fftshift(BLOB ./ max(BLOB(:))); -oB = ceil((SIZE+1)./2); -blobKernel = BLOB(oB(1)-6:oB(1)+6,... - oB(2)-6:oB(2)+6,... - oB(3)-6:oB(3)+6); - -BLOB = squeeze(gather(BLOB(oB(1), oB(2), :)))'; - -zWeight = repmat(BLOB, SIZE(2),1,SIZE(1)); -zWeight = (permute(zWeight,[3,1,2])); - - -% For re-weighting, calc the r-weighting applied in tomo reconstruction, note -% this goes from 1 --> nX/2 and not 0 --> 0.5 - -if (flgRadial) - - [ rWeight ] = calc_rWeight( SIZE, 'single', METHOD); - - % % % rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); - % % % rOrig = ceil((SIZE(1)+1)./2); - % % % % % % % imod tilt zero freq = 0.2 * first non zero component - % % % rWeight(rOrig ) = 1; - % % % - % % % - % % % % rWeight = rWeight + 1; - % % % - % % % - % % % % rWeight = (rWeight ./ max(rWeight)).^0.5; - % % % rWeight = rWeight + 1./rWeight.^2; - % % % % resample2d only handles scaling right now, so pad to z=3 - % % % rWeight = repmat((rWeight), 1, SIZE(2),3); - % % % - % % % rWeight = BH_resample3d( rWeight, [90-tiltAzimuth,0,0], ... - % % % [0,0,0],'Bah','GPU','forwardVector'); - % % % rWeight = rWeight(:,:,2); - -else - rWeight = 1; -end - -[mtf,~,~,~,~,~] = BH_multi_gridCoordinates(SIZE(1:2), ... - 'Cartesian','cpu',... - {'none'},1,1,1); - -if (useGPU) - [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(SIZE(1:2), ... - 'Cartesian','GPU',... - {'none'},1,0,1); -else - [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(SIZE(1:2), ... - 'Cartesian','cpu',... - {'none'},1,0,1); - -end - - - - -pixelSize = ORIENTATION(1,16).*ORIENTATION(1,14).*samplingRate; % scaled pixel size - -radialGrid = radialGrid./(pixelSize.*10.^10); - -% exposureFilter = ones([SIZE(1:2),size(ORIENTATION,1)],'single'); -if (flgCTF) - if (flgNoExposure) - exposureFilter = zeros(1,1,size(ORIENTATION,1)) + 1; - else - [ exposureFilter ] = BH_exposureFilter( SIZE(1:2), ORIENTATION, 'cpu',samplingRate,0 ); - end - - % Calc the downweighting due to CTF --> here assuming phases were flipped by - % multiplying by the CTF past the first zero - iPrj = 1; - defocus = [ORIENTATION(iPrj,15) - ORIENTATION(iPrj,12), ... - ORIENTATION(iPrj,15) + ORIENTATION(iPrj,12), ... - ORIENTATION(iPrj,13)]; - Cs = ORIENTATION(iPrj,17); - wavelength = ORIENTATION(iPrj,18); - ampContrast = ORIENTATION(iPrj,19); - - % assuming ampContrast = 0.1, using -0.15 results in a weight with - % (0.1^0.15)^2~ 0.5 at zero freqency. Allows some recovery of low freq without - % creating too severe a blur - [Hqz, HqzUnMod] = BH_ctfCalc(pixelSize,Cs,wavelength,defocus,SIZE(1:2),ampContrast,-1,-1); - - Hqz = single(abs(Hqz.*HqzUnMod)); - - % Hqz = conv2(Hqz,fspecial('gaussian',[5,5],1.0),'same'); - % SAVE_IMG(MRCImage(gather(single(Hqz))), 'tmp.mrc'); - -else - Hqz = zeros(size(radialGrid),'single')+1; - exposureFilter = zeros(1,1,size(ORIENTATION,1)) + 1; -end - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - - - - -% Make grid where 0.5 = Nyquist -if (useGPU) - %centralSection = zeros(SIZE,'single', 'gpuArray'); - %centralSection = gpuArray(centralSection); - wedgeMask = zeros(SIZE,'single', 'gpuArray'); -else - %centralSection = zeros(SIZE,'single'); - wedgeMask = zeros(SIZE,'single'); -end - -% centralSection(:,:,ceil((SIZE(3)+1)/2) - zoneOfInfluence: ... -% ceil((SIZE(3)+1)/2) + zoneOfInfluence) = Z; - -% % % centralSection = centralSection .* zWeight; -% % % clear zWeight - - -nPrjs = size(ORIENTATION,1); -if nPrjs == 1 - ORIENTATION = ORIENTATION'; - nPrjs = size(ORIENTATION,1); -end - -for iPrj = 1:nPrjs - - - %exposureFilter = exp(-1.*ORIENTATION(iPrj,11).*(fftshift(radialGrid)).^2); - - % Applying a re-weighting to the ctfTiles - remove this - % Optimal exposure filter from Grant,Grigorieff 2015 eLife - % Assuming either 200 or 300 KV - % % % WAVELENGTH = ORIENTATION(iPrj,18); - % % % if WAVELENGTH > 2.1*10^-12 - % % % kvScale = 0.8; - % % % else - % % % kvScale = 1.0; - % % % end - % % % CUMeDOSE = ORIENTATION(iPrj,11); - % % % exposureFilter = exp(-0.5*CUMeDOSE .* (kvScale.*0.245.*fftshift(radialGrid) .^ -1.665 + 2.81).^-1); - - %expF = 1; - if (flgCTF) - expF = exposureFilter(:,:,ORIENTATION(iPrj,1)); - else - expF = 1; - end - %expF = (expF./sqrt((sum(sum(sum(abs(expF).^2))))./numel(expF))); - if (flgRadial) - centralSection = repmat(rWeight.* fftshift(expF.* ... - Hqz ),1,1, SIZE(3)) .* ... - zWeight; - % centralSection = repmat(rWeight.* ... - % Hqz .* mtf,1,1, SIZE(3)) .* ... - % zWeight; - else - centralSection = repmat(fftshift(exposureFilter(:,:,ORIENTATION(iPrj,1)).*Hqz ),1,1, SIZE(3)) .* ... - zWeight; - end - - - - symInc = 360/SYMMETRY; - for iSym = 1:SYMMETRY - % Since the angles here are used bring the rotated projection back to standard - % basis, they will also rotate the x,y plane to the plane perpendicular to the - % projection direction in reciprocal space. - if any(size(ORIENTATION) == 1) - R = BH_defineMatrix([90, ORIENTATION(iPrj), -90], 'Bah', 'inv'); - else - % rSample = BH_defineMatrix((ORIENTATION(iPrj,8:10)), 'Bah', 'inv'); - % rElevation = BH_defineMatrix([90, ORIENTATION(iPrj, 7), -90], 'Bah', 'inv'); - rTilt = BH_defineMatrix([1.*ORIENTATION(iPrj,6),1.*ORIENTATION(iPrj,4),-1*ORIENTATION(iPrj,6)],'Bah','inv'); - - % Switch to imod means that the inPlane rotation has been applied to the - % projections already - %%%rInPlane = BH_defineMatrix(1.*[ORIENTATION(iPrj,5),0,0], 'Bah', 'inv'); - % rInPlane = eye(3); - - - % R = rElevation*rTilt*rSample*rInPlane; - R = rTilt; - end - - R = R*BH_defineMatrix([(1-iSym)*symInc,0,0],'Bah','forward'); - % - if (flgTiltWeight) - % tiltDiff = 1-cosd(ORIENTATION(iPrj,tiltColumn)).^1; - % tiltGrad = 1-(tiltDiff.*exp(-15.*fftshift(radialGrid).^2)); - % tiltWeight = repmat(tiltGrad,1,1,size(centralSection,3)); - iAng = ORIENTATION(iPrj,tiltColumn); - tiltWeight = ((exp(-10.*mtf.^(0.5+cosd(iAng).^2.5))+(0.6))./(1.6)).^sind(abs(iAng)); - tiltWeight = repmat(tiltWeight,1,1,size(centralSection,3)); - - % tiltWeight = cosd(ORIENTATION(iPrj,tiltColumn)); - else - tiltWeight = 1; - end - - - - wedgeMask = wedgeMask + ... - BH_resample3d(centralSection.*tiltWeight, R, ... - [0,0,0], 'Bah','GPU','inv'); - - - - % % % BH_resample3d(centralSection.*tiltWeight, R, [0,0,0], 'Bah','GPU','inv'); - end -end - - -% wedgeMask = wedgeMask .* mtf; -clear centralSection rWeight zWeight Hqz radialGrid mtf -if (flgSmooth) - - [ gaussKernel ] = gpuArray(BH_multi_gaussian3d(5, 0.75 )); - - wedgeMask = convn(wedgeMask, gaussKernel, 'same'); - % % % % % wedgeMask = convn(wedgeMask, gaussKernel, 'same'); -end - - - -% wedgeMask = wedgeMask(padValue(1)+1 : end - padValue(1), ... -% padValue(2)+1 : end - padValue(2), ... -% padValue(3)+1 : end - padValue(3)); - - -% wedgeMask = single(wedgeMask ./ max(wedgeMask(:))); -% wedgeMask = convn(wedgeMask, gaussKernel, 'same'); -% -% wedgeMask = wedgeMask ./ max(wedgeMask(:)); -% [rad,~,~,~,~,~] = BH_multi_gridCoordinates(SIZE,'Cylindrical', ... -% 'GPU',{BH_defineMatrix([0,90,90-tiltAzimuth],'Bah','forwardVector'),[0,0,0]','forward'},1,1,0); -% % % -% %rad = (rad < 0.5); -% wedgeMask = wedgeMask .* rad; - - - -WEDGE_MASK = BH_reScale3d( gather(single(wedgeMask ./ max(wedgeMask(:)))), ... - '', sprintf('%f',outputScaling), METHOD); - -rad = fftshift(BH_bandpass3d(size(WEDGE_MASK),0,0,0,'GPU','nyquist')); -WEDGE_MASK = WEDGE_MASK .* rad; -WEDGE_MASK = (WEDGE_MASK - min(WEDGE_MASK(:))); -WEDGE_MASK = WEDGE_MASK./(max(WEDGE_MASK(:))); - -% SAVE_IMG(MRCImage(gather(single(wedgeMask))),'wm.mrc');% .* (rad <0.5) ; -% error('sdfs') -% BINARY_WEDGE = BINARY_WEDGE .* (rad <0.5) ; -% BINARY_WEDGE = convn(single(BINARY_WEDGE), BH_multi_gaussian3d(5, 1.0 ),'same'); -% % % BINARY_WEDGE = BINARY_WEDGE ./ max(BINARY_WEDGE(:)) .*rad; -% clear wedgeMask centralSection radialGrid rad zWeight - - -clearvars -except WEDGE_MASK padValues - -end - -function [ rWeight ] = calc_rWeight( SIZE, PRECISION, METHOD) - -rWeight = ((abs([-1*floor((SIZE(1))/2):0,1:floor((SIZE(1)-1)/2)])')); -if strcmp(METHOD,'GPU') - rWeight = gpuArray(rWeight); -end -rOrig = ceil((SIZE(1)+1)./2); -% % % % imod tilt zero freq = 0.2 * first non zero component -rWeight(rOrig ) = 0.2; -[rCut] = find(rWeight == floor(0.45*SIZE(1))); -pixelFallOff = rCut(1) ; -taperLow = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); - -pixelFallOff = SIZE(1)-rCut(2)+1 ; -taperTop = 0.5+0.5.*cos((((1:pixelFallOff)).*pi)./(length((1:pixelFallOff+1)))); -rWeight(1:rCut(1)) = rWeight(1:rCut(1)).*flip(taperLow)'; -rWeight(rCut(2):end) = rWeight(rCut(2):end).*taperTop'; - - -%rWeight = rWeight + 1./rWeight.^2; -% resample2d only handles scaling right now, so pad to z=3 -rWeight = repmat((rWeight), 1, SIZE(2),1); -if strcmpi(PRECISION,'single') - rWeight = single(rWeight); -else - % This should be the default. - rWeight = double(rWeight); -end -end diff --git a/masking/BH_weightMaskMex.m b/masking/BH_weightMaskMex.m index 5a327a0b..f05afc27 100644 --- a/masking/BH_weightMaskMex.m +++ b/masking/BH_weightMaskMex.m @@ -22,7 +22,7 @@ iCs = single(TLT(:,17).*10^3); iWavelength = single(TLT(:,18).*10^10); iPhaseShift = TLT(:,19) ; -iDefocus = single(-TLT(:,15).*10^10) ; +iDefocus = single(abs(TLT(:,15)).*10^10) ; iddF = single(TLT(:,12).*10^10); idPHI = single(TLT(:,13).*180.0/pi); diff --git a/mexFiles/include/core_headers.cuh b/mexFiles/include/core_headers.cuh index 183b9301..6dc5f612 100644 --- a/mexFiles/include/core_headers.cuh +++ b/mexFiles/include/core_headers.cuh @@ -48,8 +48,8 @@ struct ctfParams { float waveLength; // Angstrom float CS; // millimeter float amplitudeContrast; - float defocus1; // Angstrom - float defocus2; // Angstrom + float mean_defocus; // Angstrom + float half_astigmatism; // Angstrom float astigmatism_angle; // from x-axis float cs_term; @@ -59,7 +59,7 @@ struct ctfParams { __host__ __device__ ctfParams() : doHalfGrid(true), doSqCTF(false), pixelSize(0.0f), waveLength(0.0f), CS(0.0f), amplitudeContrast(0.0f), - defocus1(0.0f), defocus2(0.0f), + mean_defocus(0.0f), half_astigmatism(0.0f), astigmatism_angle(0.0f), cs_term(0.0f), df_term(0.0f) {} __host__ __device__ ctfParams(bool doHalfGrid, bool doSqCTF, @@ -72,7 +72,7 @@ struct ctfParams { CS(CS * 1e7), amplitudeContrast(atanf(amplitudeContrast / sqrtf(1.0 - powf(amplitudeContrast, 2)))), // Convert ampContrast to phase shift TODO is this safe? - defocus1(0.5f*(df1+df2)), defocus2(0.5f*(df1-df2)), // Convert these to terms used in calc + mean_defocus(0.5f*(df1+df2)), half_astigmatism(0.5f*(df1-df2)), // Convert these to terms used in calc astigmatism_angle(astigmatism_angle*PI/180.0f - (PI * (float)lrintf(astigmatism_angle/180.0f))), cs_term(PI * 0.5f * CS* 1e7 * powf(waveLength,3)), df_term(PI * waveLength) {} // enforce -90 to 90 convention diff --git a/mexFiles/utils/ctf.cu b/mexFiles/utils/ctf.cu index 29dff9ef..0f0ecc82 100644 --- a/mexFiles/utils/ctf.cu +++ b/mexFiles/utils/ctf.cu @@ -54,12 +54,12 @@ __global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, flo if (b_ctf.doSqCTF ) { - tmp_coord = sinf(b_ctf.cs_term*powf(radius_sq,2) - b_ctf.df_term*radius_sq*(b_ctf.defocus1 + b_ctf.defocus2 * cosf(2.0f * (phi-b_ctf.astigmatism_angle))) - b_ctf.amplitudeContrast); + tmp_coord = sinf(b_ctf.cs_term*powf(radius_sq,2) - b_ctf.df_term*radius_sq*(b_ctf.mean_defocus + b_ctf.half_astigmatism * cosf(2.0f * (phi-b_ctf.astigmatism_angle))) - b_ctf.amplitudeContrast); // Is this any better (or worse) than pow? a[output_IDX] = tmp_coord*tmp_coord; } else{ - a[output_IDX] = sinf(b_ctf.cs_term*powf(radius_sq,2) - b_ctf.df_term*radius_sq*(b_ctf.defocus1 + b_ctf.defocus2 * cosf(2.0f * (phi-b_ctf.astigmatism_angle))) - b_ctf.amplitudeContrast); + a[output_IDX] = sinf(b_ctf.cs_term*powf(radius_sq,2) - b_ctf.df_term*radius_sq*(b_ctf.mean_defocus + b_ctf.half_astigmatism * cosf(2.0f * (phi-b_ctf.astigmatism_angle))) - b_ctf.amplitudeContrast); } } @@ -112,7 +112,7 @@ __global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, flo radius_sq = radius_sq*radius_sq + tmp_coord*tmp_coord; - tmp_coord = sinf(b_ctf.cs_term*powf(radius_sq,2) - b_ctf.df_term*radius_sq*(b_ctf.defocus1 + b_ctf.defocus2 * cosf(2.0f * (phi-b_ctf.astigmatism_angle))) - b_ctf.amplitudeContrast); + tmp_coord = sinf(b_ctf.cs_term*powf(radius_sq,2) - b_ctf.df_term*radius_sq*(b_ctf.mean_defocus + b_ctf.half_astigmatism * cosf(2.0f * (phi-b_ctf.astigmatism_angle))) - b_ctf.amplitudeContrast); // if you add the radial weighting you will need to fix this. diff --git a/statistics/BH_multi_loadAndMaskStack.m b/statistics/BH_multi_loadAndMaskStack.m index d5e74e63..0c6402ff 100644 --- a/statistics/BH_multi_loadAndMaskStack.m +++ b/statistics/BH_multi_loadAndMaskStack.m @@ -51,25 +51,21 @@ % calculate it based on subTomogram coordinates in 3d is not correct THICKNESS = 75; if ~isnumeric(STACK) - + error('This branch is deprecated and should not be reached.'); if samplingRate > 1 fullStack = sprintf('aliStacks/%s_ali%d.fixed', STACK,mapBackIter+1); inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed',STACK,mapBackIter+1,samplingRate); if ~exist(inputStack, 'file') - BH_multi_loadOrBin(fullStack,-1.*samplingRate,2); + BH_multi_loadOrBin(fullStack, samplingRate, 2, false); end else inputStack = sprintf('aliStacks/%s_ali%d%s.fixed',STACK,mapBackIter+1,suffix); end - - STACK = single(getVolume(MRCImage(inputStack))); - end - [d1,d2,d3] = size(STACK); if isa(STACK,'gpuArray') diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index ce8179dd..e4f52f3e 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -688,20 +688,7 @@ nSubTomos = size(positionList,1); - if (flgWMDs == 0) - % Make a wedge mask that can be interpolated with no extrapolation for - % calculating wedge weighting in class average alignment. - - % COMMMMMMENT - - % make a binary wedge - [ wedgeMask ]= BH_weightMask3d(sizeMask, tiltGeometry, ... - 'binaryWedgeGPU',2*maskRadius,1, 1, samplingRate); - - - error('do not do it man'); - end - + % reset for each tomogram wdgIDX = 0; diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index f5d55df7..87845586 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -226,15 +226,13 @@ particleMask = cell(nRefs,1); for iGold = 1:2 - if iGold == 1 halfSet = 'ODD'; else halfSet = 'EVE'; end - - imgNAME = sprintf('class_%d_Locations_Ref_%s', refName, halfSet) + imgNAME = sprintf('class_%d_Locations_Ref_%s', refName, halfSet); iHeader = getHeader(MRCImage(subTomoMeta.(cycleNumber).(imgNAME){1},0)); sizeWindow = iHeader.nZ.*[1,1,1]; @@ -288,22 +286,23 @@ skip_this_tilt_series_because_it_is_empty = false(nTomograms,1); - tiltList = cell(nTomograms,1); % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); tomoList = {}; tomoIDX = 1; + tilt_filename = sprintf('%saliStacks/%s_ali%d.fixed', CWD, tiltNameList{iTiltSeries}, mapBackIter + 1); + [~, tltName, tltExt] = fileparts(tilt_filename); + tilt_binned_filename = sprintf('%scache/%s_bin%d%s', CWD, tltName, samplingRate, tltExt); for iTomo = 1:size(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords,1) % This is dumb, fix it to be explicit. if any(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords(iTomo,:)) tomoList{tomoIDX} = sprintf('%s_%d',tiltNameList{iTiltSeries},iTomo); - - tiltList{tomoIDX} = sprintf('%saliStacks/%s_ali%d.fixed', CWD, tiltNameList{iTiltSeries}, mapBackIter + 1); + % Only increment if values found. tomoIDX = tomoIDX + 1; end end - [~,tiltBaseName,~] = fileparts(tiltList{1}); + [~,tiltBaseName,~] = fileparts(tilt_filename); mbOUT{3} = tiltBaseName; @@ -379,17 +378,15 @@ end end - + % TODO: these defaults should be re-examined nFiducialsPerPatch = ceil(100./sqrt(molMass)); targetPatchSize = max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch))); - + % The binned stacks should already exist, if not, this will re-create it in the cache dir. + % Note that this will also be checked when reconstructing the full 3d background tomo. if (samplingRate > 1) - for iTomo = 1:nTomograms - [~, tltName, tltExt] = fileparts(tiltList{iTomo}); - BH_multi_loadOrBin(tiltList{iTomo},-1.*samplingRate, 2); - tiltList{iTomo} = sprintf('%scache/%s_bin%d%s', CWD, tltName, samplingRate, tltExt); + BH_multi_loadOrBin(tilt_filename, samplingRate, 2, false); end end @@ -400,12 +397,9 @@ system(sprintf('mv mapBack%d mapBack%d_%d%0.2d%0.2d_%d_%d_%d',mapBackIter+1,mapBackIter+1,y,m,d,h,mi,floor(s))); clear y m d h mi s end + system(sprintf('mkdir -p %smapBack%d',tmpCache,mapBackIter+1)); - - - - % re-initialize the parpool for each tilt series to free up mem. if ~isempty(gcp('nocreate')) delete(gcp('nocreate')) @@ -415,47 +409,34 @@ end fprintf('init with %d workers\n',nWorkers); - outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); - % Get the thickest for recon maxZ = 0; - overSampleZforProjection = 1.0; - tiltHeader = getHeader(MRCImage(tiltList{1},0)); - - for iTomo = 1:nTomograms - - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; - tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - nZdZ = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,[4,6])./samplingRate - - % half the size in z plus the shift back to the microscope coords. - sZneeded = 2.*ceil(overSampleZforProjection * (nZdZ(1)/2+abs(nZdZ(2)))); - if sZneeded > maxZ - maxZ = sZneeded; - end - - clear tomoNumber nZdZ - end - maxZ = maxZ + (samplingRate*2); + + % The + tiltHeader = getHeader(MRCImage(tilt_binned_filename, 0)); + [ maxZ ] = emc_get_max_specimen_NZ(subTomoMeta.mapBackGeometry.tomoName, ... + subTomoMeta.mapBackGeometry.(tiltName).coords, ... + tomoList, ... + nTomograms, + samplingRate); + fprintf('combining thickness and shift, found a maxZ of %d\n',maxZ); % xyzproj assumes centered in Z, so add extra height for z offsets to create % the true "in microsope" dimension - reconstructionSize = [tiltHeader.nX,tiltHeader.nY,maxZ]; - originRec = ceil((reconstructionSize+1)./2); + reconstruction_size = [tiltHeader.nX, tiltHeader.nY, maxZ]; + originRec = emc_get_origin_index(reconstruction_size); avgTomo = cell(3,1); - - avgSampling = zeros(reconstructionSize,'uint8'); % These two are mutually exclusive for now, but not enforced. if (flgClassAvg) - avgColor = zeros(reconstructionSize, 'int16'); + avgColor = zeros(reconstruction_size, 'int16'); end if (emc.save_mapback_classes) - avgColor = zeros(reconstructionSize, 'int16'); + avgColor = zeros(reconstruction_size, 'int16'); end @@ -512,27 +493,11 @@ refVol{2}{iRef} = gpuArray(refVol{2}{iRef}); particleMask{iRef} = gpuArray(particleMask{iRef}); end - avgTomo{1} = avgTomo{1} ./ (overSampleZforProjection.*rmsScale*rms(avgTomo{1}(:))); - - % % % % Now reset the binned tilt to the non-ctf corrected. Could probably - % % % % just temporarily rename, but for testing do this. - % % % if (samplingRate > 1) - % % % rmTiltName = sprintf('%scache/%s_ali%d_bin%d.fixed', ... - % % % CWD,tiltNameList{iTiltSeries}, mapBackIter+1, samplingRate); - % % % % Force removal so that a binned version of the ctf stack will be - % % % % created - % % % system(sprintf('rm %s',rmTiltName)); - % % % - % % % % Resample the tilt if necessary, then modify the tilt list - % % % BH_multi_loadOrBin(sprintf('aliStacks/%s_ali%d.fixed',tiltNameList{iTiltSeries}, mapBackIter+1),-1.*samplingRate, 2); - % % % end - - % avgTomo{1} = zeros(reconstructionSize,'single'); - - + avgTomo{1} = avgTomo{1} ./ (rmsScale*rms(avgTomo{1}(:))); + if (emc.save_mapback_classes) - avgColor = zeros(reconstructionSize, 'int16'); + avgColor = zeros(reconstruction_size, 'int16'); end if (buildTomo) @@ -552,7 +517,6 @@ TLT = tiltGeometry.(tomoList{iTomo}); - doseList = TLT(:,[1,11]); postExposure = doseList(:,2)'; [sorted_doseList, doseIDX] = sortrows(doseList,2); @@ -599,7 +563,7 @@ nFidsTotal = nFidsTotal + size(positionList,1); % Need to store tilt name/path explicity in meta deta - tiltName = tiltList{iTomo}; + tiltName = tilt_binned_filename; tiltHeader = getHeader(MRCImage(tiltName,0)); @@ -617,7 +581,7 @@ % reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); tomoReconCoords = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); - tomo_origin_in_tomo_frame = ceil((tomoReconCoords(1,1:3)+1)./2); + tomo_origin_in_tomo_frame = emc_get_origin_index(tomoReconCoords(1,1:3)); tomo_origin_wrt_tilt_origin = tomoReconCoords(2,1:3); nPrjs = size(TLT,1); @@ -694,7 +658,7 @@ % The third entry is a dummy, normally used to make sure at least the % particle was being extracted even if the surrounding density (where % some delocalized values may be located) are not. - [ indVAL, padVAL, shiftVAL ] = BH_isWindowValid(reconstructionSize, sizeAvgVol, sizeAvgVol./5, subtomo_origin_in_sample); + [ indVAL, padVAL, shiftVAL ] = BH_isWindowValid(reconstruction_size, sizeAvgVol, sizeAvgVol./5, subtomo_origin_in_sample); if ischar(indVAL) @@ -947,7 +911,7 @@ '%s\n', ... '%s\n', ... '%s\n',... - 'EOF'],tiltList{1} ,outputStackName, maxZ, ... + 'EOF'],tilt_binned_filename ,outputStackName, maxZ, ... mbOUT{1:3},... taStr, mbOUT{1:3},iSave,... 0,sTY-1,... @@ -986,7 +950,7 @@ '%s\n', ... '%s\n', ... '%s\n',... - 'EOF'],tiltList{1} ,outputStackName, maxZ, ... + 'EOF'],tilt_binned_filename ,outputStackName, maxZ, ... mbOUT{1:3},... taStr, mbOUT{1:3},iSave,... 0,sTY-1,... @@ -1074,7 +1038,7 @@ '%s\n',... '%s\n',... '%s\n',... - 'EOF'],tiltList{1}, mbOUT{1:3}, maxZ, ... + 'EOF'],tilt_binned_filename, mbOUT{1:3}, maxZ, ... mbOUT{1:3},... mbOUT{1:3},... pixel_size./10, flgInvertTiltAngles,... % Ang --> nm @@ -1362,10 +1326,11 @@ evalMask(~samplingMask) = false; samplingMask = []; - mean_defocus = TLT(iPrj,15); + % FIXME: The old convention was negative. For now, assuming no support for overfocus, so if a negative value is encountered, invert it. + mean_defocus = abs(TLT(iPrj,15)); half_astigmatism = TLT(iPrj,12); angle_astigmatism = TLT(iPrj,13); - defVect = [mean_defocus - half_astigmatism, mean_defocus + half_astigmatism, angle_astigmatism]; + defVect = [mean_defocus + half_astigmatism, mean_defocus - half_astigmatism, angle_astigmatism]; [Hqz, HqzUnMod] = BH_ctfCalc(TLT(iPrj,16).*samplingRate,TLT(iPrj,17), ... TLT(iPrj,18),defVect,size(refPrj), ... diff --git a/testScripts/fourierCtfRec.m b/testScripts/fourierCtfRec.m deleted file mode 100644 index 4d4415f7..00000000 --- a/testScripts/fourierCtfRec.m +++ /dev/null @@ -1,249 +0,0 @@ -function [ reconstruction ] = fourierCtfRec(wantedSize, positionList, TLT, reconGeometry, originPrj, varargin) - - -% % % % -% For testing override inputs -% % % % -flgTesting = false; - - -% Insert a check forces a cubic dimension during reconstruction, followed -% by rescaling -d1 = wantedSize(1); -d2 = wantedSize(2); -d3 = wantedSize(3); - -samplingRate = 1; - -exposureWeight = 0; -if nargin == 6 - iThickness = 75; - samplingRate = varargin{1}; - fractionOfDose = TLT(:,14)/mean(TLT(:,14)); - fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(:,4))*400 )); - doHalfGrid = 1; - centerGrid = 1; - [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); - for iPrj = 1:size(exposureWeight,3) - exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)) .* ... - (fractionOfDose(iPrj).*(fractionOfElastics(iPrj))); - end -end - - - -if (d1 ~= d2 || d1 ~= d3) - error('fix the cubic check ya twit'); -end - - -if ( flgTesting ) - % These will just go in the object properties - rawtlt = -60:3:60; - nTilts = length(rawtlt); - TLT = zeros(nTilts,23,'single','gpuArray'); - TLT(:,4) = rawtlt; - TLT(:,1) = 1:length(rawtlt); - % 12,13 = astigmatism = 0 - % def,pix,cs,wl,ampcont - TLT(:,[15,16,17,18,19]) = repmat([-1.2e-6,2.0e-10,2.7e-3,1.969e-12,1e-1],nTilts,1); - reconGeometry = wantedSize; - reconShift = [0,0,0]; - originPrj = wantedSize; - originPrj = ceil((originPrj+1)./2); - originPrj(3) = 1; - % Position list is just for the wanted subTomo - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - - % This matrix would rotate a vector into the particles reference frame, - % we want to rotate back, so take the transpose to invert. - particleRotMat = BH_defineMatrix([0,0,0],'Bah','forward'); - prjVector = [0,0,0] - originVol + reconShift + [0.0,0.0,1.0]; -else - % These will just go in the object properties - reconShift = reconGeometry(2,:); - originPrj = ceil((originPrj+1)./2); - originPrj(3) = 1; - % Position list is just for the wanted subTomo - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - - - % This matrix would rotate a vector into the particles reference frame, - % we want to rotate back, so take the transpose to invert. - particleRotMat = reshape(positionList(17:25),3,3)'; - - % Taken from synthetic mapback, not sure why I'm adding the 1 here. - prjVector = positionList(11:13) - originVol + reconShift + [0.0,0.0,1.0]; - % We want the origin of the subTomogram in the tilted image to calculate - % a defocus offset. -end - - - -oX = floor(d1/2)+1; -oY = floor(d2/2)+1; -oZ = floor(d3/2)+1; - -spreadVal = [1]; -%spreadVal = [0.4904 0.3733 0.1363] -nOffCenter = length(spreadVal) - 1; - -% Replicate the taper -spreadVal = [flip(spreadVal(2:end)),spreadVal]; - -[X,Y] = BH_multi_gridCoordinates([d1,d2],'Cartesian',... - 'GPU',{'none'},0,1,0,{'halfgrid'}); - -[rad,phi] = BH_multi_gridCoordinates([d1,d2],'Cylindrical',... - 'GPU',{'none'},1,1,0,{'halfgrid'}); - -% Assuming the pixel size is constant. -rad = {rad(:) ./ (samplingRate.*TLT(1,16)),1,phi(:)}; -clear phi - -spatialToPixel = 0.5; - -X = X(:) + spatialToPixel; -Y = Y(:) + spatialToPixel; - -reconstruction = zeros([oX,d2,d3],'single','gpuArray'); - -weights = reconstruction; - -for iPrj = 1:size(TLT,1) - - - % Need a defocus offset based on XYZ position in the tomogram - rTilt = BH_defineMatrix([0,TLT(iPrj,4),0],'SPIDER','forwardVector'); - prjCoords = rTilt * prjVector'; - - iDefocusOffset = prjCoords(3).*(TLT(iPrj,16)); - - iDefocus = [iDefocusOffset + TLT(iPrj,15) - TLT(iPrj,12), ... - iDefocusOffset + TLT(iPrj,15) + TLT(iPrj,12), ... - TLT(iPrj,13)]; - - - - - iCTF = (BH_ctfCalc(rad,TLT(iPrj,17),TLT(iPrj,18),iDefocus,[oX,d2],TLT(iPrj,19),-1)).^2; - if (length(exposureWeight) > 1) - iCTF = iCTF(:).*reshape(exposureWeight(:,:,TLT(iPrj,1)),oX.*d2,1); - else - iCTF = iCTF(:); - end - - % The sample is rotated by tiltA, which is like rotating the beam by - % -tiltA. We want to insert in the plane perpendicular to the beam. - xform = particleRotMat *rTilt; - - for iOffset = 1:nOffCenter*2+1 - - iZ = iOffset - nOffCenter - 1 + spatialToPixel; - Xnew = X.*xform(1) + Y.*xform(4) + (iZ*xform(7)) ; - Ynew = X.*xform(2) + Y.*xform(5) + (iZ*xform(8)) ; - Znew = X.*xform(3) + Y.*xform(6) + (iZ*xform(9)) ; - - % For any points rotated into the unsampled half, generate Hermitian - % mate. The CTF is real, so no need to also negate that, however, for - % reconstruc - hermitianSym = Xnew < 0; - Xnew(hermitianSym) = -1.*Xnew(hermitianSym); - Ynew(hermitianSym) = -1.*Ynew(hermitianSym); - Znew(hermitianSym) = -1.*Znew(hermitianSym); - - % Need to use the conjugate!!! - - Xnew = Xnew + 1 ; - Ynew = Ynew + oY ; - Znew = Znew + oZ ; - - for dZ = [0:1] - zCoord = (floor(Znew)+dZ); - z_inBounds = (zCoord <= d3 & zCoord > 0); - weight_z = abs(1.0 - abs(Znew-zCoord)); - - for dY = [0:1] - yCoord = (floor(Ynew)+dY); - weight_yz = abs(1.0 - abs(Ynew-yCoord)) .* weight_z; - - y_inBounds = (z_inBounds & yCoord <= d2 & yCoord > 0); - - for dX = [0:1] - - xCoord = (floor(Xnew)+dX); - weight_xyz = weight_yz .* (spreadVal(iOffset) * abs(1.0 - abs(Xnew-xCoord))) ; - - x_inBounds = ( y_inBounds & xCoord <= oX & xCoord > 0 ); - - linearIDX = sub2ind([oX,d2,d3],xCoord(x_inBounds), ... - yCoord(x_inBounds), ... - zCoord(x_inBounds)); - - weight_xyz = weight_xyz(x_inBounds); - - xCTF = weight_xyz.*iCTF(x_inBounds); - - - toKeep = true(length(linearIDX),1,'gpuArray'); - - - % Currently ~ 2.7s for a 257^3 - half that time is in this block. - % Unique is the biggest killer, then assignments out. - % If all parts of loop could be assigned to a single output grid which - % is already unique, this could save most of this time. - while ~isempty(linearIDX) - - [~,ai] = unique(linearIDX); - ci = linearIDX(ai); - - reconstruction(ci) = reconstruction(ci) + xCTF(ai); - weights(ci) = weights(ci) + weight_xyz(ai); - - toKeep(ai) = false; - - linearIDX = linearIDX(toKeep); - weight_xyz = weight_xyz(toKeep); - xCTF = xCTF(toKeep); - toKeep = toKeep(toKeep); - - end - - - - end - end - end - - end % end of offset loop - - - -end - -% TODO these should be set up top, and possible outside. -gaussDiam = 5; -g = BH_multi_gaussian3d(gaussDiam.*[1,1,1],1.5); -% % TODO this should be worked in to avoid extra allogation. -% reconstruction = BH_padZeros3d(reconstruction,gaussDiam.*[1,1,1], ... -% gaussDiam.*[1,1,1], ... -% 'GPU','single'); -% weights = BH_padZeros3d(weights,gaussDiam.*[1,1,1], ... -% gaussDiam.*[1,1,1], ... -% 'GPU','single'); -% % Mirror along the origin -% reconstruction(1:gaussDiam,:,:) = reconstruction(2*gaussDiam:-1:gaussDiam+1,:,size(reconstruction,3):-1:1); -% weights(1:gaussDiam,:,:) = weights(2*gaussDiam:-1:gaussDiam+1,:,size(weights,3):-1:1); - -reconstruction = convn(reconstruction,g,'same')./convn((weights+.01),g,'same'); -% % Trim the fat -% reconstruction = reconstruction(gaussDiam+1:end-gaussDiam, ... -% gaussDiam+1:end-gaussDiam, ... -% gaussDiam+1:end-gaussDiam); - -clear weights - -end % end of function - diff --git a/testScripts/fourierCtfRecTex.m b/testScripts/fourierCtfRecTex.m deleted file mode 100644 index b113f003..00000000 --- a/testScripts/fourierCtfRecTex.m +++ /dev/null @@ -1,338 +0,0 @@ -function [ reconstruction ] = fourierCtfRecTex(wantedSize, positionList, TLT, reconGeometry, originPrj, varargin) - - -% % % % -% For testing override inputs -% % % % - -if length(varargin) > 1 - flgTesting = varargin{1,2}; -else - flgTesting = false; -end - - -% Insert a check forces a cubic dimension during reconstruction, followed -% by rescaling -d1 = wantedSize(1); -d2 = wantedSize(2); -d3 = wantedSize(3); - -samplingRate = 1; - -exposureWeight = 0; -if nargin == 6 && ~flgTesting - iThickness = 75; - samplingRate = varargin{1}; - fractionOfDose = TLT(:,14)./mean(TLT(:,14)); - fractionOfElastics = exp(-1.*iThickness./( cosd(TLT(:,4)).*400 )); - fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics); - doHalfGrid = 1; - centerGrid = 1; - [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); - for iPrj = 1:size(exposureWeight,3) - exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)).^2 .* ... - (fractionOfDose(iPrj).*(fractionOfElastics(iPrj))); - end -end - - - -if (d1 ~= d2 || d1 ~= d3) - error('fix the cubic check ya twit'); -end - - -if ( flgTesting ) - % These will just go in the object properties - rawtlt = single(-60:3:60); - nTilts = uint32(length(rawtlt)); - TLT = zeros(nTilts,23,'single','gpuArray'); - TLT(:,4) = rawtlt; - TLT(:,1) = 1:length(rawtlt); - % 12,13 = astigmatism = 0 - % def,pix,cs,wl,ampcont - pixelSize = single(2.0); - defocus = single(12000); - CS = single(2.7); - WL = single(0.01969); - AC = single(0.1); - TLT(:,[12,13,14,15,16,17,18,19]) = repmat([1e-7,pi/4,1,-defocus*10^-10,pixelSize*10^-10,CS*10^-3,WL*10^-10,AC],single(nTilts),1); - TLT(:,11) = TLT(:,1).*2;%%[[41:-1:20,1:19].*3]'; - iThickness = 75; - samplingRate = 1; - fractionOfDose =1; - fractionOfElastics = exp(-1.*iThickness/( cosd(TLT(:,4)).*400 )); - fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics(:)); - doHalfGrid = true; - centerGrid = 1; - reconGeometry = (wantedSize); - reconShift = [0,0,0]; - originPrj = wantedSize; - originPrj = ceil((originPrj+1)./2); - originPrj(3) = 1; - % Position list is just for the wanted subTomo - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - [ exposureWeight ] = BH_exposureFilter( wantedSize(1:2), TLT, 'GPU', samplingRate, centerGrid, doHalfGrid); - for iPrj = 1:size(exposureWeight,3) - exposureWeight(:,:,TLT(iPrj,1)) = exposureWeight(:,:,TLT(iPrj,1)).^2 .* ... - (fractionOfDose.*(fractionOfElastics(iPrj))); - end % This matrix would rotate a vector into the particles reference frame, - % we want to rotate back, so take the transpose to invert. - particleRotMat = BH_defineMatrix([0,0,0],'Bah','forward'); - prjVector = [0,0,0] - originVol + reconShift + [0.0,0.0,1.0]; - - defocusAst = single(0); - exposure = gather(single(TLT(:,11))); - [ SF3D] = BH_weightMaskMex(gather(uint32(wantedSize)), ... - gather(single(samplingRate)),... - gather(single(TLT)), ... - gather(single([0,0,0])),gather(single([reconGeometry;0,0,0]))); - - tic - for i = 1:30 - [ SF3D] = BH_weightMaskMex(gather(uint32(wantedSize)), ... - gather(single(samplingRate)),... - gather(single(TLT)), ... - gather(single([0,0,0])),gather(single([reconGeometry;0,0,0]))); - end - toc/30 - - - SAVE_IMG(SF3D,'test3d.mrc'); -else - % These will just go in the object properties - reconShift = reconGeometry(2,:); - originPrj = ceil((originPrj+1)./2); - originPrj(3) = 1; - % Position list is just for the wanted subTomo - originVol = ceil((reconGeometry(1,1:3)+1)./2); - lowerLeftVol = originPrj+reconShift-originVol; - - - % This matrix would rotate a vector into the particles reference frame, - % we want to rotate back, so take the transpose to invert. - particleRotMat = reshape(positionList(17:25),3,3)'; - - % Taken from synthetic mapback, not sure why I'm adding the 1 here. - prjVector = positionList(11:13) - originVol + reconShift + [0.0,0.0,1.0]; - % We want the origin of the subTomogram in the tilted image to calculate - % a defocus offset. - - -end - -nZpad = 9; -if mod(nZpad+1,2) - error('nZpad is assmued odd'); -end - -[ taper ] = BH_multi_calcTaper(floor(nZpad./2)); -taper = [ flip(taper), 1, taper ]; -taper = taper./sum(taper(:)); - - - -oX = floor(d1/2)+1; -oY = floor(d2/2)+1; -oZ = floor(d3/2)+1; - - -[X,Y,Z] = BH_multi_gridCoordinates([d1,d2,nZpad],'Cartesian',... - 'GPU',{'none'},0,1,0,{'halfgrid'}); - -[rad,phi] = BH_multi_gridCoordinates([d1,d2],'Cylindrical',... - 'GPU',{'none'},1,1,0,{'halfgrid'}); -% [X,Y,Z] = BH_multi_gridCoordinates([d1,d2,nZpad],'Cartesian',... -% 'GPU',{'none'},0,1,0); -% -% [rad,phi] = BH_multi_gridCoordinates([d1,d2],'Cylindrical',... -% 'GPU',{'none'},1,1,0); - -% Assuming the pixel size is constant. -rad = {(rad./ (samplingRate.*TLT(1,16).*10^10)),[1,1],1.*phi}; -clear phi - -spatialToPixel = 0.5; - -xIn = X + 1; -yIn = Y + oY; -zIn = Z + floor(nZpad/2); - -wIn = ones(size(X)); -for iTaper = 1:length(taper) - wIn(:,:,iTaper) = taper(iTaper); -end - - -X = X + spatialToPixel; -Y = Y + spatialToPixel; -Z = Z + spatialToPixel; - -reconstruction = zeros([oX,d2,d3],'single','gpuArray'); - -weights = reconstruction; - - -for iPrj = 1:size(TLT,1) - - - % Need a defocus offset based on XYZ position in the tomogram - rTilt = BH_defineMatrix(TLT(iPrj,4),'TILT','forwardVector') ; - prjCoords = rTilt * prjVector'; - iDefocusOffset = (prjCoords(3).*(TLT(iPrj,16))+TLT(iPrj,15)); - - iDefocus = [iDefocusOffset - TLT(iPrj,12), ... - iDefocusOffset + TLT(iPrj,12), ... - TLT(iPrj,13)]; - - - % TODO add an option to check and oversample the CTF, then crop in real space, to fix aliasing probs. - - iCTF = (BH_ctfCalc(rad,TLT(iPrj,17),TLT(iPrj,18),iDefocus,[oX,d2],TLT(iPrj,19),-1)).^2; - % SAVE_IMG(iCTF.*exposureWeight(:,:,TLT(iPrj,1)),'BHctf.mrc'); - % iCTF = mexCTF(false,true,gather(int16(d1)),gather(int16(d2)),... - % gather(single(samplingRate.*TLT(iPrj,16)*10^10)),gather(single(TLT(iPrj,18)*10^10)),... - % gather(single(TLT(iPrj,17)*10^3)),gather(single(iDefocus(1)*-1*10^10)),... - % gather(single(iDefocus(2)*-1*10^10)),gather(single(iDefocus(3)*180/pi)),gather(single(TLT(iPrj,19))),true,... - % single(1.0), gather(single(TLT(iPrj,11)))); - % SAVE_IMG(iCTF,'MExctf.mrc'); - % error('iCTF') - % In some cases we may be before the first zero. TODO fixme - if (mean(iCTF(:)) < 0.25 ) - iCTF = iCTF.*0 + 1; - end - % iCTF = mexCTF(true,true,int16(d1),int16(d2),single(TLT(iPrj,16)*10^10),single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),single(iDefocus(1)*-1*10^10),single(iDefocus(2)*-1*10^10),single(30),single(0.1)); - if (length(exposureWeight) > 1) - iWeight = wIn.*repmat(exposureWeight(:,:,TLT(iPrj,1)),1,1,nZpad); - iCTF = iWeight .* repmat(iCTF,1,1,nZpad); - else - iCTF = repmat(iCTF,1,1,nZpad).*wIn; - end - - - - - % The sample is rotated by tiltA, which is like rotating the beam by - % -tiltA. We want to insert in the plane perpendicular to the beam. - xform = particleRotMat *rTilt; - - Xnew = X.*xform(1) + (Z.*xform(7)) ; - Ynew = Y; - Znew = X.*xform(3) + (Z.*xform(9)) ; - - % iZ = iOffset - nOffCenter - 1 + spatialToPixel; - % Xnew = X.*xform(1) + Y.*xform(4) + (iZ*xform(7)) ; - % Ynew = X.*xform(2) + Y.*xform(5) + (iZ*xform(8)) ; - % Znew = X.*xform(3) + Y.*xform(6) + (iZ*xform(9)) ; - - % For any points rotated into the unsampled half, generate Hermitian - % mate. The CTF is real, so no need to also negate that, however, for - % reconstruc - hermitianSym = find(Xnew < 0); - Xnew(hermitianSym) = -1.*Xnew(hermitianSym); - Ynew(hermitianSym) = -1.*Ynew(hermitianSym); - Znew(hermitianSym) = -1.*Znew(hermitianSym); - - clear hermitianSym - - % Need to use the conjugate!!! - - Xnew = floor(Xnew + 1) ; - Ynew = floor(Ynew + oY); - Znew = floor(Znew + oZ) ; - - - xOutOfBounds = Xnew(:,1,:) <= 0 & Xnew(:,1,:) > size(reconstruction,1); - yOutOfBounds = Ynew(:,1,:) <= 0 & Ynew(:,1,:) > size(reconstruction,2); - zOutOfBounds = Znew(:,1,:) <= 0 & Znew(:,1,:) > size(reconstruction,3); - - - % Not sure what is the most efficient way to do this. All at once, better mem harder sort, or incrementally. - - % For rotation about a single axis, we only need to worry about one value - % of that axis to deterimine unique positions - [~, ia] = unique([reshape(Xnew(:,1,:),oX*nZpad,1),reshape(Znew(:,1,:),oX*nZpad,1)],'rows'); - k = false([oX*nZpad,1],'gpuArray'); - k(ia) = true; - k(xOutOfBounds | yOutOfBounds | zOutOfBounds) = false; - - k = repmat(reshape(k,oX,1,nZpad),1,d2,1); - k = find(k); - % A = [Xnew(k),Ynew(k),Znew(k)]; - - % A = unique([Xnew,Ynew,Znew],'rows'); - - Xnew = Xnew(k); - Ynew = Ynew(k); - Znew = Znew(k); - - - - % keepVal = Xnew(k) > 0 & Ynew(k) > 0 & Znew(k) > 0 &... - % Xnew(k) <= size(reconstruction,1) & ... - % Ynew(k) <= size(reconstruction,2) & ... - % Znew(k) <= size(reconstruction,3); - % A = A(keepVal,:); - linearIDX = sub2ind([oX,d2,d3],Xnew,Ynew,Znew); - % A = A(keepVal,:); - % linearIDX = sub2ind([oX,d2,d3],A(:,1),A(:,2),A(:,3)); - - - - % A now has a list of uniqe 3D coords in the reconstruction - % Those coords now need to be xformed back into the projection (transpose, should it also be negative?) - % Optimized for single-axis - Xnew = Xnew - 1; - Znew = Znew - oZ; - xInterp = Xnew.*xform(1) + Znew .* xform(3) + 1; - zInterp = Xnew.*xform(7) + Znew .* xform(9) + floor(nZpad/2); - % xInterp = (A(:,1)-1).*xform(1) + (A(:,2)-oY) .*xform(2) + (A(:,3)-oZ) .* xform(3) + 1; - % yInterp = (A(:,1)-1).*xform(4) + (A(:,2)-oY) .*xform(5) + (A(:,3)-oZ) .* xform(6) + oY; - % zInterp = (A(:,1)-1).*xform(7) + (A(:,2)-oY) .*xform(8) + (A(:,3)-oZ) .* xform(9) + floor(nZpad/2); - - - - - reconstruction(linearIDX) = reconstruction(linearIDX) + ... - interpn(xIn,yIn,zIn,iCTF,xInterp,Ynew,zInterp,'linear',0); - - weights(linearIDX) = weights(linearIDX) + ... - interpn(xIn,yIn,zIn,iWeight,xInterp,Ynew,zInterp,'linear',0); - - - % weights(linearIDX) = weights(linearIDX) + 1; - - - -end - -clear iCTF xIn yIn -clear xInterp yInterp iWeight - -doConv = false; - -if doConv - - % TODO these should be set up top, and possible outside. - gaussDiam = 5; - g = gpuArray(BH_multi_gaussian3d(gaussDiam.*[1,1,1],0.75)); - - reconstruction = BH_multi_makeHermitian(reconstruction,gaussDiam,1); - weights = BH_multi_makeHermitian(weights + 0.01,gaussDiam,1); - reconstruction = convn(reconstruction,g,'same')./convn(weights,g,'same'); - recosntruction = reconstruction(1+gaussDiam:end,:,:); - -else - reconstruction = reconstruction ./ (weights + 0.01); -end -% % Trim the fat -% reconstruction = reconstruction(gaussDiam+1:end-gaussDiam, ... -% gaussDiam+1:end-gaussDiam, ... -% gaussDiam+1:end-gaussDiam); - -clear weights g bp - -end % end of function - diff --git a/transformations/BH_multi_loadOrBin.m b/transformations/BH_multi_loadOrBin.m index b4140875..8c9d0a61 100755 --- a/transformations/BH_multi_loadOrBin.m +++ b/transformations/BH_multi_loadOrBin.m @@ -1,4 +1,4 @@ -function [ IMG_OUT, iPixelHeader, iOriginHeader, imgExt ] = BH_multi_loadOrBin( IMG, SAMPLING, DIMENSION, varargin ) +function [ IMG_OUT, iPixelHeader, iOriginHeader, imgExt ] = BH_multi_loadOrBin( input_tilt_series_filename, samplingRate, DIMENSION, LOAD_VOL ) %Check to see if a cached binned image exists, either load or bin and load. % Switched to using imod's newstack and binvol to create binning and % removed inline binning from my workflow. @@ -6,22 +6,17 @@ iPixelHeader = ''; iOriginHeader = ''; imgExt = ''; -flgLoad = 0; +flgLoad = LOAD_VOL; +IMG_OUT = ''; -if SAMPLING < 0 - samplingRate = abs(SAMPLING); - IMG_OUT = ''; -else - flgLoad = 1; - samplingRate = SAMPLING; -end try - [imgPath, imgName, imgExt] = fileparts(IMG); + [imgPath, imgName, imgExt] = fileparts(input_tilt_series_filename); catch - IMG + input_tilt_series_filename error('Trouble getting fileparts for this tilt-series'); end + if isempty(imgPath) imgPath = '.'; end @@ -36,44 +31,42 @@ fprintf('Using cached file %s_bin%d%s\n', imgName, samplingRate,imgExt); [checkHeader,~] = system(sprintf('header %s > /dev/null',nameOUT)); if (checkHeader) - fprintf('File exists but appears to be corrupt %s_bin%d%s\n', imgName, samplingRate,imgExt); + fprintf('File exists but appears to be corrupt %s_bin%d%s\n', imgName, samplingRate, imgExt); doCalc = 1; end else doCalc = 1; end - if (doCalc) + !mkdir -p cache - - switch DIMENSION case 3 system(sprintf('binvol -BinningFactor %d -antialias 6 %s cache/%s_bin%d%s > /dev/null', ... - samplingRate,IMG, imgName, samplingRate,imgExt)); + samplingRate, input_tilt_series_filename, imgName, samplingRate, imgExt)); case 2 try - tiltObj = MRCImage(IMG,0); + tiltObj = MRCImage(input_tilt_series_filename,0); catch fprintf('If you have restarted somewhere it is possible the aligned stack is not found?\n'); fprintf('Perhaps the value for CurrentTomoCpr is not correct?\n'); - error('Could not init an MRCImage for %s in loadOrBin',IMG); - + error('Could not init an MRCImage for %s in loadOrBin',input_tilt_series_filename); end iHeader = getHeader(tiltObj); - outputName = (sprintf('cache/%s_bin%d%s',imgName, samplingRate,imgExt)); + outputName = (sprintf('cache/%s_bin%d%s', imgName, samplingRate, imgExt)); + iPixelHeader = [iHeader.cellDimensionX/iHeader.nX .* samplingRate , ... - iHeader.cellDimensionY/iHeader.nY .* samplingRate, ... - iHeader.cellDimensionZ/iHeader.nZ .* samplingRate]; + iHeader.cellDimensionY/iHeader.nY .* samplingRate, ... + iHeader.cellDimensionZ/iHeader.nZ .* samplingRate]; iOriginHeader= [iHeader.xOrigin ./ samplingRate, ... - iHeader.yOrigin ./ samplingRate, ... - iHeader.zOrigin ./ samplingRate]; + iHeader.yOrigin ./ samplingRate, ... + iHeader.zOrigin ./ samplingRate]; pixelSize = iHeader.cellDimensionX/iHeader.nX; % Assuming X/Y the same and Z might be incorrect. - [binSize,binShift] = BH_multi_calcBinShift([iHeader.nX,iHeader.nY],1,samplingRate); + [binSize, binShift] = BH_multi_calcBinShift([iHeader.nX, iHeader.nY], samplingRate); % Gridding correction for the interpolation in the binning. Not % sure this is quite right, but it looks much better. TODO FIXME @@ -93,18 +86,13 @@ iProjection = BH_resample2d(iProjection,[0,0,0],binShift,'Bah','GPU','forward',1/samplingRate,binSize(1:2),bhF); - - % iProjection = real(ifftn(ifftshift(BH_padZeros3d(fftshift(fftn(iProjection)),'fwd',padVal,'GPU','singleTaper')))); - % - - newStack(:,:,iPrj) = gather(iProjection); end SAVE_IMG(newStack,outputName,iPixelHeader,iOriginHeader); clear newStack bpFilt iProjection % system(sprintf('newstack -shrink %d -antialias 6 %s cache/%s_bin%d%s > /dev/null', ... - % samplingRate,IMG, imgName, samplingRate,imgExt)); + % samplingRate,input_tilt_series_filename, imgName, samplingRate,imgExt)); otherwise error('DIMENSION should be 2 or 3\n.') end @@ -142,7 +130,7 @@ % but throw a warning. fprintf('\n\nYou requested a sampling of -1 Nonsense!! loading anyway.\n\n'); - IMG_OUT = single(getVolume(MRCImage(IMG))); + IMG_OUT = single(getVolume(MRCImage(input_tilt_series_filename))); end diff --git a/transformations/BH_multi_loadOrBuild.m b/transformations/BH_multi_loadOrBuild.m index e7d3f05c..f392f3aa 100755 --- a/transformations/BH_multi_loadOrBuild.m +++ b/transformations/BH_multi_loadOrBuild.m @@ -67,7 +67,7 @@ stack = sprintf('cache/%s_ali%d%s_bin%d.fixed',tomoName,mapBackIter+1,ctf,SAMPLING); if ~exist(stack, 'file') - BH_multi_loadOrBin(checkStack,SAMPLING, 2); %%%%% med filt flag + BH_multi_loadOrBin(checkStack, SAMPLING, 2, true); %%%%% med filt flag end else diff --git a/transformations/emc_get_max_specimen_NZ.m b/transformations/emc_get_max_specimen_NZ.m new file mode 100644 index 00000000..fca60af4 --- /dev/null +++ b/transformations/emc_get_max_specimen_NZ.m @@ -0,0 +1,40 @@ +function [ max_specimen_nz, tomoNumber ] = emc_get_max_specimen_NZ(subTomoMeta_tomoName, subTomoMeta_coords, tomo_name_list, n_tomograms, samplingRate) + + max_z_value = -inf; + min_z_value = inf; + + tomoNumber = zeros(n_tomograms,1); + + for iTomo = 1:n_tomograms + % subTomoMeta_tomoName = subTomoMeta.mapBackGeometry.tomoName + % subTomoMeta_coords = subTomoMeta.mapBackGeometry.(tiltName).coords + if isa(subTomoMeta_tomoName,'struct') + tomoNumber(iTomo) = subTomoMeta_tomoName.(tomo_name_list{iTomo}).tomoNumber; + else + tomoNumber(iTomo) = iTomo; + end + + % 4 is the unbinned size of the tomogram in Z + % 6 is location of the origin in Z relative to the origin of the specimen + nZ = subTomoMeta_coords(tomoNumber(iTomo),4) ./ samplingRate; + oZ = subTomoMeta_coords(tomoNumber(iTomo),6) ./ samplingRate; + + % We need to consider the shift of the tomogram relative to the specimen origin, + % which is -oZ + dZ = -oZ; + + if (dZ + nZ / 2 > max_z_value) + max_z_value = dZ + nZ / 2; + end + if (dZ - nZ / 2 < min_z_value) + min_z_value = dZ - nZ / 2; + end + end + + if (max_z_value < min_z_value) + error('The max z value is less than the min z value'); + end + + max_specimen_nz = ceil(max_z_value - min_z_value + (samplingRate * 2)); + +end \ No newline at end of file diff --git a/transformations/emc_get_origin_index.m b/transformations/emc_get_origin_index.m new file mode 100644 index 00000000..ca904dd8 --- /dev/null +++ b/transformations/emc_get_origin_index.m @@ -0,0 +1,12 @@ +function [ origin ] = emc_get_origin_index( obj ) + + if isempty(obj) + error('Input must not be empty'); + end + if ~isnumeric(obj) + error('Input must be a numeric object'); + end + + origin = floor(obj/2) + 1; + +end \ No newline at end of file From e6ed86bfd68a950d0c5643eb3091a69fc6ffe637 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 15 Feb 2024 14:26:34 -0500 Subject: [PATCH 025/151] wip --- ctf/BH_ctf_Correct.m | 206 ------------------------------- ctf/BH_ctf_Correct3d.m | 70 +++-------- ctf/BH_runCtfFind.m | 2 +- synthetic/BH_to_cisTEM_mapBack.m | 2 +- testScripts/emClarity.m | 4 +- transformations/BH_average3d.m | 2 +- 6 files changed, 24 insertions(+), 262 deletions(-) delete mode 100755 ctf/BH_ctf_Correct.m diff --git a/ctf/BH_ctf_Correct.m b/ctf/BH_ctf_Correct.m deleted file mode 100755 index bb23f598..00000000 --- a/ctf/BH_ctf_Correct.m +++ /dev/null @@ -1,206 +0,0 @@ -function [ ] = BH_ctf_Correct( PARAMETER_FILE, STACK_PRFX ) -%CTF correction for tilt series using general geometry. -% Correct for the CTF using a local approach, similar to strip based -% periodogram, but with tiles that are smaller allowing for arbitrary -% defocus gradients. -% -% The full stack is corrected, st if only a small region is to be used, -% it would be faster to have trimmed the stack. This should be done -% before ctf estimation though, st the correct origin is included in the -% tilt information. -% - -emc = BH_parseParameterFile(PARAMETER_FILE); - -try - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR; -catch - mapBackIter = 0; -end - -if isnan(str2double(STACK_PRFX)) - % It is a name, run here. - nGPUs = 1; - flgParallel = 0; - STACK_LIST = {STACK_PRFX}; - ITER_LIST = {STACK_LIST}; -else - flgParallel = 1; - nGPUs = emc.('nGPUs'); - - STACK_LIST_tmp = fieldnames(subTomoMeta.mapBackGeometry); - STACK_LIST_tmp = STACK_LIST_tmp(~ismember(STACK_LIST_tmp,'tomoName')); - ITER_LIST = cell(nGPUs,1); - nST = 1; STACK_LIST = {}; - for iStack = 1:length(STACK_LIST_tmp) - if subTomoMeta.mapBackGeometry.(STACK_LIST_tmp{iStack}).nTomos - STACK_LIST{nST} = STACK_LIST_tmp{iStack}; - nST = nST + 1; - end - end - clear STACK_LIST_tmp - for iGPU = 1:nGPUs - ITER_LIST{iGPU} = STACK_LIST(iGPU:nGPUs:length(STACK_LIST)); - end -end - -!mkdir -p ctfStacks - -try - EMC_parpool(nGPUs); -catch - delete(gcp('nocreate')); - EMC_parpool(nGPUs); -end - - -parfor iGPU = 1:nGPUs - - if ( flgParallel ) - useGPU = iGPU; - gpuDevice(useGPU); - else - useGPU = BH_multi_checkGPU(-1); - gpuDevice(useGPU); - end - - for iTilt = 1:length(ITER_LIST{iGPU}) - - - STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - - - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,mapBackIter+1); - TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - - inputStack = sprintf('aliStacks/%s_ali%d.fixed',STACK_PRFX,mapBackIter+1); - outputStack = sprintf('ctfStacks/%s_ali%d_ctf.fixed',STACK_PRFX,mapBackIter+1); - iMrcObj = MRCImage(inputStack,0); - - - - iHeader = getHeader(iMrcObj); - iPixelHeader = [iHeader.cellDimensionX/iHeader.nX, ... - iHeader.cellDimensionY/iHeader.nY, ... - iHeader.cellDimensionZ/iHeader.nZ]; - - d1 = iHeader.nX; - d2 = iHeader.nY; - nPrjs = iHeader.nZ; - - correctedStack = zeros(d1,d2,nPrjs,'single'); - - for iPrj = 1:nPrjs - - CS = TLT(iPrj,17); - WL = TLT(iPrj,18); - AMPCONT = TLT(iPrj,19); - ddF = TLT(iPrj,12); - dPhi = TLT(iPrj,13); - D0 = TLT(iPrj,15); - - - fastFTSize = BH_multi_iterator([d1,d2],'fourier2d'); - padVal = BH_multi_padVal([d1,d2],fastFTSize); - trimVal = BH_multi_padVal(fastFTSize,[d1,d2]); - - - initImg = randn(fastFTSize,'single','gpuArray'); - f = FFT(initImg); - - ctf = CTF(fastFTSize,emc.pixel_size_angstroms,'GPU'); - maxZ = 500; - maxEval = cosd(TLT(iPrj,4)).*(d1/2) + maxZ./2*abs(sind(TLT(iPrj,4))); - oX = ceil((d1+1)./2); - oY = ceil((d2+1)./2); - iEvalMask = floor(oX-maxEval):ceil(oX+maxEval); - - STRIPWIDTH = 512; - STRIPWIDTH = STRIPWIDTH + mod(STRIPWIDTH,2); - % take at least 1200 Ang & include the taper if equal to STRIPWIDTH - tileSize = floor(max(600./emc.pixel_size_si, STRIPWIDTH + 28)); - tileSize = tileSize + mod(tileSize,2); - %fprintf('stripwidth tilesize %d %d\n',STRIPWIDTH,tileSize); - incLow = ceil(tileSize./2); - incTop = tileSize - incLow; - - - iProjection = BH_padZeros3d(getVolume(iMrcObj,[-1],[-1],TLT(iPrj,23),'keep'), ... - padVal(1,:),padVal(2,:),'GPU','singleTaper'); - - correctedPrj = zeros([d1,d2],'single','gpuArray'); - iProjectionFT = f.fwdFFT(iProjection); - - - stripDefocusOffset = floor(STRIPWIDTH/2); - for i = 1: STRIPWIDTH : d1 - - if (i+tileSize-1) < d1 - endIDX = (i+tileSize-1); - endCUT = i + STRIPWIDTH - 1 + 7; - trimmedSIZE = STRIPWIDTH; - elseif any(ismember(i:d1,iEvalMask)) - endIDX = d1; - endCUT = d1; - trimmedSIZE = endCUT-i+1 -7; - end - - % The eval mask condition can be replaced once the per tomo condition - % is trusted. - if any(ismember(i:endIDX,iEvalMask)) - - DF = D0 + ( i + stripDefocusOffset - oX)*emc.pixel_size_si*-1.*tand(TLT(iPrj,4) ); - - if ~( isempty(DF) ) - - iDefocus = [DF - ddF, DF + ddF, dPhi]; - - if emc.pixel_size_si < 2.0e-10 - % use double precision - this is not enabled, but needs to be - - % requires changes to radial grid as well. - ctf.new_img(iDefocus,CS,WL,AMPCONT,-1,-1); - else - ctf.new_img(iDefocus,CS,WL,AMPCONT,-1); - end - - - tile = ctf.multiply(iProjectionFT); - - tile = BH_padZeros3d(real(f.invFFT(tile,2)), ... - trimVal(1,:),trimVal(2,:),'GPU','single'); - - % trim prior to pulling off gpu to minimize xfer - else - - % No particles in this strip, so just replace with simple inversion - % to keep the global image statistics ~ correct. - - tile = -1.*BH_padZeros3d(iProjection, trimVal(1,:),trimVal(2,:),'GPU','single'); - - end - - %correctedStack(i + 7 : endCUT,:,TLT(iPrj,1)) = ... - % gather(tile(8:trimmedSIZE+7,:)); - - correctedPrj(i:endIDX,:) = tile(i:endIDX,:); - - else - %fprintf('ignoring strip centered on %d for prj %d',i,TLT(iPrj,1)); - end - end % end loop over strips - correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj); - - end % end loop over prjs - - SAVE_IMG(MRCImage(correctedStack), outputStack,iPixelHeader); - - end % end loop over tilt-series -end % end parfor - -delete(gcp('nocreate')); - - -end - diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 104eee59..63747591 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -255,9 +255,6 @@ end -% ctf3dDepth=emc.('defocusErrorEst') -%mean in case cones. - %%%%% Take these from param file later. if (reconstructionParameters(1)) @@ -378,17 +375,15 @@ iterList{gpuList(iGPU)} = iGPU+(tiltStart-1):nGPUs:nTilts; iterList{gpuList(iGPU)}; end -% FIXME -% try -% EMC_parpool(nGPUs) -% catch -% delete(gcp('nocreate')) -% EMC_parpool(nGPUs) -% end +try + EMC_parpool(nGPUs) +catch + delete(gcp('nocreate')) + EMC_parpool(nGPUs) +end -% FIXME -% parfor iGPU = 1:nGPUs -for iGPU = 1:nGPUs +parfor iGPU = 1:nGPUs +% for iGPU = 1:nGPUs for iTilt = iterList{gpuList(iGPU)} @@ -447,8 +442,8 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. % FIXME -% parfor iGPU = 1:nGPUs - for iGPU = 1:nGPUs +parfor iGPU = 1:nGPUs + % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); % Loop over each tilt for iTilt = iterList{gpuList(iGPU)} @@ -615,21 +610,16 @@ end - fprintf('Using a avgZ of %3.3e nm\n',avgZ); - for iSection = 1:nSections defFitFull = ''; preCombDefocus = 0; if (mapBackIter) - defFitFull = sprintf('mapBack%d/%s_ali%d_ctf.defFidFull',mapBackIter, ... - tiltList{iTilt},mapBackIter); + defFitFull = sprintf('mapBack%d/%s_ali%d_ctf.defFidFull',mapBackIter, tiltList{iTilt},mapBackIter); if exist(defFitFull,'file') preCombDefocus = load(defFitFull); fprintf('3dCTF using pre calc combined per tilt defocus %s\n',defFitFull); - else - fprintf('Did not find %s\n!!',defFitFull); end end @@ -899,38 +889,31 @@ function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, pixel_size_angstroms, nSections,tiltName, ctf3Depth) nTomos = length(tomoNumber); -sectionList = cell(nTomos,1) +sectionList = cell(nTomos,1); for iTomo = 1:nTomos % min and max in absolute pixels min and max from 1:nZrecon - sectionList{iTomo} = zeros(nSections,6) + sectionList{iTomo} = zeros(nSections,6); end % With rounding this could end up a bit short except the top and bottom are both % half a section larger than minimally needed. -n_pixels_per_slab = floor(ctf3Depth*10^10/pixel_size_angstroms) -n_pixels_per_slab = n_pixels_per_slab + ~mod(n_pixels_per_slab,2) - -halfSec = (n_pixels_per_slab-1)/2 +nSec = floor(ctf3Depth*10^10/pixel_size_angstroms) ; +nSec = nSec + ~mod(nSec,2); +halfSec = (nSec-1)/2; for iT = 1:length(tomoNumber) iTomo = tomoNumber(iT); % Origin + originshift - oZ_in_tomo_frame = emc_get_origin_index(iCoords(iTomo,4)); - % iCoords(iTomo,6) is the origin of the specimen with respect to the origin of the tomogram - % We want the origin of the tomogram with respect to the origin of the specimen and then calculate the lower - % bound of the reconstruction range as half the volume lower than that - oZ_in_specimen_frame = -1*iCoords(iTomo,6); - reconRange = floor([oZ_in_specimen_frame - oZ_in_tomo_frame,0]); + -1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6), + reconRange = floor([-1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6),0]); reconRange(2) = reconRange(1) + iCoords(iTomo,4) - 1; nZ = 1; flgFirstSec = 1; - % nSections is always ODD - % FIXME: The half section (slab) shift seems precarious, I don't remember why this is set up this way. - for iSection = 1:nSections + for iSection = 1:nSections - sectionCenter = ((nSections-1)/-2+(iSection-1))*(n_pixels_per_slab-1); + sectionCenter = ((nSections-1)/-2+(iSection-1))*(nSec-1); % Check that sectionCenter is within range if sectionCenter + halfSec < reconRange(1) || ... @@ -1168,7 +1151,6 @@ minDefocus = min(tZ(:)); maxDefocus = max(tZ(tZ < 1)); % Spit out some info - % fprintf('Found a min/max defocus of %3.3e/ %3.3e for tilt %d (%3.3f deg)\n',minDefocus,maxDefocus,iPrj,TLT(iPrj,4)); % To track sampling in case I put in overlap samplingMask = zeros([d1,d2],'single','gpuArray'); @@ -1213,25 +1195,13 @@ tmpMask = (tZ > iDefocus - ctf3dDepth/2 & tZ <= iDefocus + ctf3dDepth/2); - % try - linearIDX = unique(sub2ind([d1,d2],tX(tmpMask),tY(tmpMask))); - % catch - % - % - % ferr=fopen('err.txt','w'); - % fprintf(ferr,'%f %f\n',[tX(tmpMask),tY(tmpMask)]); - % fclose(ferr); - % error('sdf') - % end correctedPrj(linearIDX) = correctedPrj(linearIDX) + tmpCorrection(linearIDX); samplingMask(linearIDX) = samplingMask(linearIDX) + 1; end % end loop over defocus values - - samplingMask(samplingMask == 0) = 1; if (flgWhitenPS(1)) diff --git a/ctf/BH_runCtfFind.m b/ctf/BH_runCtfFind.m index 369f6d4c..42e472ec 100644 --- a/ctf/BH_runCtfFind.m +++ b/ctf/BH_runCtfFind.m @@ -27,7 +27,7 @@ % end tmpTLT = load(sprintf('fixedStacks/ctf/%s_orig',tltName)); -meanDefocus = mean(tmpTLT(:,15))*-1.0*10^10; +meanDefocus = mean(abs(tmpTLT(:,15)))*10^10; fprintf('Searching around an estimated mean defocus of %3.6f Angstrom\n'); % write the run script, this should link to a distributed version with diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 2879998e..f816e32f 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -233,7 +233,7 @@ % Imod expects nanometers and underfocus positive (origin on specimen) % whereas I let the origin be the focal plane such that underfocus is % negative. - fprintf(iDefocusFile,'%f\n',defTLT(:,2)'.*(-1*10^9)); + fprintf(iDefocusFile,'%f\n',abs(defTLT(:,2)').*(10^9)); fclose(iDefocusFile); % We also need the transform from the microscope frame in order to diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index ee37ccf9..97127c80 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -533,8 +533,6 @@ error('\n\nYou now only need to specify %s parameter file.\n\n','the') end BH_ctf_Updatefft(varargin{3},'-1','full'); - case 'correct' - BH_ctf_Correct(varargin{3},varargin{4},varargin{5},varargin{6},varargin{7}); case '3d' if nArgs == 6 % last is a dummy, used for tomoCPR background @@ -549,7 +547,7 @@ end otherwise - error('ctf operations are estimate,refine,update,correct, or 3d.'); + error('ctf operations are estimate,refine,update, or 3d.'); end end case 'tomoCPR' diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 4646c837..d7abfc3b 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -899,7 +899,7 @@ if ( any(emc.filterDefocus)) iDef = abs(mean(tiltGeometry(:,15))*10^6); - iDef = -1.*(emc.filterDefocus(1)*max(iDef-1,0.5))^emc.filterDefocus(2); + iDef = (emc.filterDefocus(1)*max(iDef-1,0.5))^emc.filterDefocus(2); fprintf('Using iDef %f\n',iDef); % Frequency is already squared so adjust to match iDef scale % factor. From 1c408e620ffcdedacd9df671a55621789b224982 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 16 Feb 2024 13:02:32 -0500 Subject: [PATCH 026/151] Fix tiling in refine ctf and allow higher resolution at tilt in run ctffind. --- alignment/BH_runAutoAlign.m | 1 + ctf/BH_ctfCalc.m | 2 +- ctf/BH_ctf_Estimate.m | 49 +++++----- ctf/BH_ctf_Refine2.m | 150 +++++++++++++++---------------- ctf/BH_runCtfFind.m | 19 ++-- logicals/BH_isWindowValid.m | 24 ++--- metaData/BH_parseParameterFile.m | 7 +- synthetic/BH_synthetic_mapBack.m | 13 +-- testScripts/mCompile.sh | 2 +- 9 files changed, 140 insertions(+), 127 deletions(-) diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index f47bba91..1a1da359 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -98,6 +98,7 @@ tiltAngleOffset=0.0; TILT_OPTION = 0; +fprintf("Stack in is %s\n",stackIN); inputMRC = MRCImage(stackIN,0); inputStack = single(getVolume(inputMRC)); diff --git a/ctf/BH_ctfCalc.m b/ctf/BH_ctfCalc.m index dcd700cf..d9382207 100755 --- a/ctf/BH_ctfCalc.m +++ b/ctf/BH_ctfCalc.m @@ -133,7 +133,7 @@ error('df1 must be greater than df2') end -dfTerm = 0.5.*( (df1+df2) + (df1-df2)*cos(2.*(phi-phi0)) ); +dfTerm = 0.5.*( (df1+df2) + (df1-df2)*cos(2.*(phi0-phi)) ); phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 - WL.*(radialGrid).^2 .* dfTerm); % dPdQ = 2*pi*CS*WL^3.*radialGrid.^3 + 2*WL.*radialGrid.*dfTerm; if ( flgComplex ) diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 76531128..47caaa39 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -30,8 +30,10 @@ gpuIDX = BH_multi_checkGPU(-1); gDev = gpuDevice(gpuIDX); -flgResume = 0; -flgSkip = 0; +% for trouble shooting downstream, default falsw +flgSkip = false; +% For trouble shooting downstream, default true +resample_stack = true; % slightly dampen lower resolution information that may overwhelm the CCC calc, % but don't risk too much noise amplification. 1 = fit just the amplitude (not % PS) 0.5 = take sqrt prior to normalizing. @@ -171,6 +173,9 @@ tileSize = tileSize + mod(tileSize,2); % tileSize = max(tileSize, 384); +if (tileSize > 512) + tileOverlap = tileOverlap * 2; +end fprintf('Using a tile size of %d\n',tileSize); overlap = floor(tileSize ./ tileOverlap); @@ -232,9 +237,6 @@ % Make ctf directory to store diagnostic images system(sprintf('mkdir -p %s/ctf', pathName)); -% if ~(flgResume) - - iMrcObj = MRCImage(stackNameIN,0); @@ -317,7 +319,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - +if ( resample_stack) fprintf('Combining tranformations\n\n'); % Load in the mapBack alignment @@ -446,7 +448,8 @@ combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); + iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2), ... + 'Bah','GPU','forward',imodMAG,size(iProjection),bhF); else combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); combinedInverted = combinedInverted([1,2,4,5]); @@ -477,6 +480,7 @@ SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName),iPixelHeader,iOriginHeader); +end if ~(flgSkip) @@ -523,7 +527,7 @@ nT2 = 0; nT3 = 0; - halfX = floor(paddedSize/2) + 1; + halfX = emc_get_origin_index(paddedSize); % % % % psTile = zeros([(paddedSize).*[1,1],3],'single','gpuArray'); psTile = zeros([halfX,paddedSize,3],'single','gpuArray'); @@ -658,11 +662,11 @@ currentDefocusEst = defEST; currentDefocusWin = defWIN; measuredVsExpected = zeros(2,3); - end + end % end of not skip fitting for iTilt = 1:3 - if (skipFitting) + if (skipFitting && resample_stack) currentDefocusEst = defEST; % Add the determined defocus, and write out with mic paramters as well. @@ -823,7 +827,6 @@ initAstigCCC(n,:) = [iAng,iDelDF*astigStep,iCCC]; n = n + 1; - fprintf('%d / %d coarse astigmatism search\n',n,size(initAstigCCC,1)); end end @@ -947,18 +950,18 @@ else - [~, idx] = sortrows(abs(TLT(:,4)), -1); - TLT = TLT(idx,:); - % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt - % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY - % scaleFactor, defocus, emc.pixel_size_si, CS, Wavelength, Amplitude contrast - fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); - fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... - '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... - '%d\t%d\t%d\t%8.2f\n'], TLT'); - fclose(fileID); - - + if (resample_stack) + [~, idx] = sortrows(abs(TLT(:,4)), -1); + TLT = TLT(idx,:); + % number in stack, dx, dy, tilt angle, projection rotation, tilt azimuth, tilt + % elevation, e1,e2,e3, dose number (order in tilt collection), offsetX, offsetY + % scaleFactor, defocus, emc.pixel_size_si, CS, Wavelength, Amplitude contrast + fileID = fopen(sprintf('%s/ctf/%s_ctf.tlt',pathName,stackNameOUT), 'w'); + fprintf(fileID,['%d\t%08.2f\t%08.2f\t%07.3f\t%07.3f\t%07.3f\t%07.7f\t%07.7f\t',... + '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... + '%d\t%d\t%d\t%8.2f\n'], TLT'); + fclose(fileID); + end end % end flgSkip % TODO should I restart the parallel pool diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index 43721261..2b53705e 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -68,14 +68,18 @@ try paddedSize = emc.('paddedSize'); catch - paddedSize = 768; + paddedSize = 512; end % Tile size & overlap tileOverlap = 4; tileSize = floor(680e-10 / emc.pixel_size_si); -tileSize = tileSize + mod(tileSize,2); -fprintf('Using a tile size of %d',tileSize); +tileSize = BH_multi_iterator(tileSize.*[1,1],'fourier2d'); +tileSize = tileSize(1); +% if (tileSize > 512) +% tileOverlap = tileOverlap * 2; +% end +fprintf('Using a tile size of %d\n',tileSize); overlap = floor(tileSize ./ tileOverlap); @@ -119,7 +123,7 @@ % created in IMod alignment. [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); iPixelHeader = EMC_str2double(iPixelHeader); - [d1,d2,d3] = size(STACK) + [d1,d2,d3] = size(STACK); @@ -156,7 +160,7 @@ BH_movingRMS(iProjection,[tileSize,tileSize]); % Taking a cue from Alexis maxPixelSizeWanted = 2.0e-10; - if TLT(iPrj,16) < maxPixelSizeWanted + if TLT(iPrj,16) < maxPixelSizeWanted %fprintf(ftmp,'Resampling pixel size\n'); % Resample to 2Ang/pix padSq = BH_multi_padVal(size(iProjection),max(size(iProjection)).*[1,1]); @@ -167,13 +171,13 @@ iProjection = fftshift(fftn(iProjection)); trimVal = BH_multi_padVal(size(iProjection), floor(size(iProjection).*(TLT(iPrj,16)./maxPixelSizeWanted))); iProjection = real(ifftn(ifftshift(BH_padZeros3d(iProjection,trimVal(1,:),trimVal(2,:),'GPU','single')))); - sizeOUT = size(iProjection); + sizeOUT = size(iProjection); if iPrj == 1 flgReplaceStack = 1; newSTACK = zeros([sizeOUT,d3],'single'); end newSTACK(:,:,TLT(iPrj,1)) = gather(iProjection); - clear iProjection + iProjection = []; % Actual new pixel size pixelSize = sizeIN./sizeOUT(1).*TLT(iPrj,16); @@ -192,44 +196,50 @@ if ( flgReplaceStack ) STACK = newSTACK ; clear newSTACK; end - [d1,d2,d3] = size(STACK); - [Xnew, Ynew, ~, x1,y1, ~] = BH_multi_gridCoordinates([tileSize,d2], ... - 'Cartesian','GPU', ... - {'none'},0,1,0); - - [X, Y, ~,~,~, ~] = BH_multi_gridCoordinates([tileSize,tileSize], ... - 'Cartesian','GPU', ... - {'none'},0,1,0); - - coordShift = (-1).^(X+Y); - clear X Y + [d1,d2,d3] = size(STACK) - try - ppool = EMC_parpool(nWorkers); - catch - delete(gcp('nocreate')); - ppool = EMC_parpool(nWorkers); - end + - for iPrj = 1:d3 - - pFuture(iPrj) = parfeval(ppool,@runAvgTiles,2, TLT, paddedSize, tileSize, ... - d1,d2, iPrj, overlap, ... - STACK(:,:,TLT(iPrj,1)), ... - 1, ... - 1, ... - x1, y1, Xnew, Ynew,coordShift, ... - reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); - + debug_without_parallel = false; + if (debug_without_parallel) + for iPrj = 1:d3 + fprintf('Calculating stretched tiles on prj %d/ %d in serial debug mode\n',iPrj,d3); + + [psTile(:,:,TLT(iPrj,1)),pixelSize] = runAvgTiles(TLT, paddedSize, tileSize, ... + d1,d2, iPrj, overlap, ... + STACK(:,:,TLT(iPrj,1)), ... + 1, ... + 1, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); + + end + else + try + ppool = EMC_parpool(nWorkers); + catch + delete(gcp('nocreate')); + ppool = EMC_parpool(nWorkers); + end - end - - for iWorker = 1:d3 - fprintf('Refining defocus on prj %d/ %d\n',iWorker,d3); - [iPrj, ctfCorr,pixelSize] = fetchNext(pFuture); + for iPrj = 1:d3 + + pFuture(iPrj) = parfeval(ppool,@runAvgTiles,2, TLT, paddedSize, tileSize, ... + d1,d2, iPrj, overlap, ... + STACK(:,:,TLT(iPrj,1)), ... + 1, ... + 1, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); + + + end - psTile(:,:,TLT(iPrj,1)) = ctfCorr; - end + for iWorker = 1:d3 + fprintf('Calculating stretched tiles on prj %d/ %d\n',iWorker,d3); + [iPrj, ctfCorr,pixelSize] = fetchNext(pFuture); + + psTile(:,:,TLT(iPrj,1)) = ctfCorr; + end + end % debug without parallel pixelSize = pixelSize*10^10; SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS.mrc',fileName),pixelSize); @@ -261,13 +271,12 @@ function [psTile,pixelSize] = runAvgTiles(TLT, paddedSize, tileSize, d1,d2, iPrj, overlap, ... iProjection, evalMask, ... - ddZ, x1, y1, Xnew, Ynew, coordShift, ... + ddZ, ... reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine) DFo = abs(TLT(iPrj,15)); padTileOver = 256; -tmpTile = zeros(paddedSize.*[1,1]+2*padTileOver,'single','gpuArray'); tiltOrigin = ceil((size(iProjection,1)+1)./2); @@ -280,6 +289,7 @@ iEvalMask = floor(oXprj-maxEval):ceil(oXprj+maxEval); +psTile = zeros(paddedSize.*[1,1], 'single','gpuArray'); % Since I'm enforcing Y-tilt axis, then this could be dramatically sped up % by resampling strips along the sampling @@ -295,13 +305,16 @@ % Slightly randomize the step size to avoid a Moire like effect that % presents particulary strongly with a continuous carbon layer. - + iDeltaZ = (i - tiltOrigin)*pixelSize*-1.*tand(TLT(iPrj,4)); if any(ismember(i-tileSize/2+1:i+tileSize/2,iEvalMask)) %evalMask(i,paddedSize/2+1) mag = (1+iDeltaZ./DFo).^0.5; + if ~isfinite(mag) + error('mag is not finite'); + end - estSize = 2048; + estSize = tileSize(1); ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),iDeltaZ+DFo,estSize,TLT(iPrj,19),-1,1); ctf1 = ctf1(1:estSize/2); @@ -344,48 +357,35 @@ end scaledStrip = iProjection(i-tileSize/2+1:i+tileSize/2,:); - + for j = 1+tileSize/2:overlap:d2-tileSize/2 - iTile = scaledStrip(:,j-tileSize/2+1:j+tileSize/2);%.*coordShift; - if (reScaleRealSpace) - scaledSize = paddedSize; + iTile = fftshift(fftn(scaledStrip(:,j-tileSize/2+1:j+tileSize/2)));%.*coordShift; + + % Slightly randomize scaling + if (randi(2,1) == 2) + scaledSize = ceil(size(iTile) .* mag) + randi(2,1) -1; else - % Slightly randomize scaling - if (randi(2,1) == 2) - scaledSize = ceil(paddedSize .* mag) + randi(2,1) -1; - else - scaledSize = floor(paddedSize .* mag)+ randi(2,1) -1; - end - %scaledSize = floor(paddedSize ./ mag); + scaledSize = floor(size(iTile) .* mag)+ randi(2,1) -1; end + + tile_padVal = BH_multi_padVal(size(iTile),scaledSize); + iTile = real(ifftn(ifftshift(BH_padZeros3d(iTile,'fwd',tile_padVal,'GPU','singleTaper', 0)))); + + iPadVal = BH_multi_padVal(scaledSize,paddedSize.*[1,1]); + + iTile = fftshift(abs(fftn(BH_padZeros3d(iTile, 'fwd', iPadVal, ... + 'GPU','singleTaper', mean(iTile(:)))))); - [oX,oY] = size(tmpTile); - oX = ceil((oX+1)./2); - oY = ceil((oY+1)./2); - - iPadVal = BH_multi_padVal(size(iTile),[scaledSize,scaledSize]); - - - oupSize = [floor(scaledSize./2),ceil(scaledSize./2); ... - floor(scaledSize./2),ceil(scaledSize./2)]; - % Get rid of th fftshift - tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... - oY-oupSize(2,1):oY+oupSize(2,2)-1) = ... - tmpTile(oX-oupSize(1,1):oX+oupSize(1,2)-1, ... - oY-oupSize(2,1):oY+oupSize(2,2)-1) + ... - fftshift(abs(fftn(BH_padZeros3d(iTile,iPadVal(1,:),iPadVal(2,:), ... - 'GPU','singleTaper', mean(iTile(:)))))); + psTile = psTile + iTile; end % loop over j end % if over eval mask end % over tiles -psTile = gather(BH_padZeros3d(tmpTile, [-1,-1].* ... - padTileOver,[-1,-1].*padTileOver,... - 'GPU','single')); +psTile = gather(psTile); clear tmpTile iProjection ddZ evalMask Xnew Ynew x1 y1 end diff --git a/ctf/BH_runCtfFind.m b/ctf/BH_runCtfFind.m index 42e472ec..a9a35b10 100644 --- a/ctf/BH_runCtfFind.m +++ b/ctf/BH_runCtfFind.m @@ -36,18 +36,21 @@ fID = fopen(scriptName,'w'); fprintf(fID,'#!/bin/bash\n\n'); -for iPrj = 1:d3 % I want to fit to lower resolution at higher tilts +for iPrj = 1:d3 + % I want to fit to lower resolution at higher tilts tltIDX = find(tiltAngles(:,1) == iPrj); % put in a line to limit number of cores, or use the threaded version fprintf(fID,'\n%s --amplitude-spectrum-input << eof &',ctfFindPath); fprintf(fID,'\nfixedStacks/ctf/forCtfFind/%s_%d.mrc\n',randPrfx,iPrj); fprintf(fID,'fixedStacks/ctf/forCtfFind/%s_diagnostic_%d.mrc\n',randPrfx,iPrj); - fprintf(fID,'%f\n%f\n%f\n%f\n%d\n%f\n%f\n%d\n%d\n%d\n',ctfParams(1:4), ... - d1,30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)),... - 0.75*meanDefocus,... - 1.25*meanDefocus,... - 25.0); + fprintf(fID,'%f\n%f\n%f\n%f\n%d\n%f\n%f\n%d\n%d\n%d\n', ... + ctfParams(1:4), ... + d1, ... + 30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)).^0.4,... + 0.75*meanDefocus,... + 1.25*meanDefocus,... + 25.0); fprintf(fID,'no\nno\nyes\n500.0\nno\nno\nno\neof\n\n'); end fprintf(fID,'wait\n'); @@ -76,8 +79,8 @@ system(sprintf('rm -f %s',tmpName)); for iPrj = 1:d3 - - system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10,3.1415926535/180.0*$4,-1*(($2+$3)/2)*10^-10 }'' >> %s', baseName,iPrj,tmpName)); + % 2024 Jan, finally make switch to record positive for underfocus as is used internally. + system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10, 3.1415926535/180.0*$4, 1*(($2+$3)/2)*10^-10 }'' >> %s', baseName,iPrj,tmpName)); end diff --git a/logicals/BH_isWindowValid.m b/logicals/BH_isWindowValid.m index de725fd0..c44d3920 100755 --- a/logicals/BH_isWindowValid.m +++ b/logicals/BH_isWindowValid.m @@ -73,22 +73,22 @@ availableArea = WINDOW_SIZE - PADVALUES(1,:) - PADVALUES(2,:); if any(availableArea - minSizeMask < -2) - fprintf(['\nvs %d %d %d\nws %d %d %d\nmr %2.1f %2.1f %2.1f\n',... - 'minArea %d %d %d\navailArea %d %d %d\nc %2.1f %2.1f %2.1f\n'], ... - VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, minSizeMask, availableArea, CENTER); + % fprintf(['\nvs %d %d %d\nws %d %d %d\nmr %2.1f %2.1f %2.1f\n',... + % 'minArea %d %d %d\navailArea %d %d %d\nc %2.1f %2.1f %2.1f\n'], ... + % VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, minSizeMask, availableArea, CENTER); INDICES = 'noUse'; PADVALUES = [availableArea]; end if any(isnan(PADVALUES(:))) - fprintf('center %f %f %f\n',CENTER); - fprintf('min %f %f %f\n',minSizeMask); - fprintf('winLowCorner %f %f %f\n', winLowCorner); - fprintf('top %f %f %f\n',winTopCorner); - fprintf('%f %f %f\n',winCenter); - fprintf('del %f %f %f\n',deltaWinCenter); - fprintf('%f %f %f\n',LOW); - fprintf('%f %f %f\n',TOP); - error('\n\nFound a NaN in the pad values. But Why ben why?\n\n'); + % fprintf('center %f %f %f\n',CENTER); + % fprintf('min %f %f %f\n',minSizeMask); + % fprintf('winLowCorner %f %f %f\n', winLowCorner); + % fprintf('top %f %f %f\n',winTopCorner); + % fprintf('%f %f %f\n',winCenter); + % fprintf('del %f %f %f\n',deltaWinCenter); + % fprintf('%f %f %f\n',LOW); + % fprintf('%f %f %f\n',TOP); + % error('\n\nFound a NaN in the pad values. But Why ben why?\n\n'); INDICES='noUse'; PADVALUES = [availableArea]; end diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 29c6edfd..faf6d818 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -19,6 +19,7 @@ clear p emc = struct(); +last_parsed_parameter = 'none'; % Check that all paramters are name: value pairs stringValues = {'subTomoMeta'; ... 'Ali_mType';'Cls_mType';'Cls_mType';'Raw_mType';'Fsc_mType'; ... @@ -27,16 +28,18 @@ for i = 1:size(p2,1) pNameVal = strsplit(p2{i,1},'='); if length(pNameVal) == 1 + fprintf("Last successfully parsed parameter: %s\n", string(last_parsed_parameter)); error('Could not split Name=Value pair for\n\t %s',char(pNameVal)) elseif length(pNameVal) > 2 + fprintf("Last successfully parsed parameter: %s\n", string(last_parsed_parameter)); error('To many colons in\n\t %s',char(pNameVal)) - else - + else if any(strcmp(stringValues, pNameVal{1})) emc.(pNameVal{1}) = pNameVal{2}; else emc.(pNameVal{1}) = EMC_str2double(pNameVal{2}); end + last_parsed_parameter = pNameVal{1}; end end diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 87845586..525a8d6a 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -415,10 +415,11 @@ % The tiltHeader = getHeader(MRCImage(tilt_binned_filename, 0)); + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{1}).tiltName; [ maxZ ] = emc_get_max_specimen_NZ(subTomoMeta.mapBackGeometry.tomoName, ... subTomoMeta.mapBackGeometry.(tiltName).coords, ... tomoList, ... - nTomograms, + nTomograms, ... samplingRate); fprintf('combining thickness and shift, found a maxZ of %d\n',maxZ); @@ -473,7 +474,7 @@ % if (emc.save_mapback_classes) % BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3); % end - + % FIXME: calling like this does not use the surface fit for the background send_phakePhasePlateOption = [0,0]; BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution); @@ -623,7 +624,7 @@ % We need to rotate the model 90 degrees around X to match the "natural" reconstruction reference frame of imod % that is [x,z,-y] - modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector'); + modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector') for iSubTomo = 1:nSubTomos @@ -1666,11 +1667,13 @@ fprintf(fidBin,'%d %4.4f %4.4f %d\n',fFull'); fclose(fidBin); - % % % % % fFull(:,2:3) = fFull(:,2:3).*samplingRate; + fFull(:,2:3) = fFull(:,2:3) .* samplingRate; + % FIXME: I am reverting to sampling rate here, which I think should be correct. + % The model ends up seeing the pixel size as 1, so even though it loads % properly on the full aligned stack, these coords need to be scaled by % the pixel size since this is the input to tiltalign. - fFull(:,2:3) = fFull(:,2:3).*pixel_size; + % fFull(:,2:3) = fFull(:,2:3).*pixel_size; fprintf(fidCombine,'%d %4.4f %4.4f %d\n',fCombine'); fclose(fidCombine); diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index fb25a0c2..5cc0935e 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=0 -nightly=2 +nightly=3 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 98f2259147143324ddfc1b0f3537ddf67f0da3f7 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 16 Feb 2024 15:08:51 -0500 Subject: [PATCH 027/151] Fix date time and doHelical errors from parameter conversion in template search --- alignment/BH_templateSearch3d_2.m | 5 ++--- ctf/BH_ctfCalc.m | 3 ++- ctf/BH_ctf_Estimate.m | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 0f43cbc2..d2d6db6c 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -283,7 +283,7 @@ % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search -if ( doHelical ) +if ( emc.doHelical ) rotConvention = 'Helical'; end @@ -1359,8 +1359,7 @@ fclose(fileID); - -fprintf('Total execution time : %f seconds\n', datetime("now") - startTime); +fprintf('Total execution time : %f seconds\n', seconds(datetime("now")-startTime)); diff --git a/ctf/BH_ctfCalc.m b/ctf/BH_ctfCalc.m index d9382207..74559f8d 100755 --- a/ctf/BH_ctfCalc.m +++ b/ctf/BH_ctfCalc.m @@ -111,7 +111,7 @@ radialGrid = double(radialGrid); phi = double(phi); end - + radialGrid = radialGrid ./ PIXEL_SIZE; end @@ -134,6 +134,7 @@ end dfTerm = 0.5.*( (df1+df2) + (df1-df2)*cos(2.*(phi0-phi)) ); + phasePerturbation = pi.*(0.5.*CS.*WL^3.*(radialGrid).^4 - WL.*(radialGrid).^2 .* dfTerm); % dPdQ = 2*pi*CS*WL^3.*radialGrid.^3 + 2*WL.*radialGrid.*dfTerm; if ( flgComplex ) diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 47caaa39..86b3f744 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -101,7 +101,7 @@ WAVELENGTH = 10^-12*1226.39/sqrt(emc.VOLTAGE + 0.97845*10^-6*emc.VOLTAGE^2) ; -CUM_e_DOSE = emc.('CUM_e_DOSE'); +CUM_e_DOSE = 0; % test astigmatism vals flgAstigmatism = 1; if (flgAstigmatism ~=1 && flgAstigmatism ~= 0) From fb8b20f1d99d839182c1e7c92d4e27882f4afc5a Mon Sep 17 00:00:00 2001 From: himesb Date: Sun, 18 Feb 2024 11:18:54 -0500 Subject: [PATCH 028/151] mapBack WIP --- synthetic/BH_synthetic_mapBack.m | 2 +- synthetic/BH_to_cisTEM_mapBack.m | 752 ++++++++++++++++++------------- 2 files changed, 443 insertions(+), 311 deletions(-) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 525a8d6a..e1b49560 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -1042,7 +1042,7 @@ 'EOF'],tilt_binned_filename, mbOUT{1:3}, maxZ, ... mbOUT{1:3},... mbOUT{1:3},... - pixel_size./10, flgInvertTiltAngles,... % Ang --> nm + pixel_size./10, ...flgInvertTiltAngles,... % Ang --> nm mbOUT{1:3},... mbOUT{1:3},... mbOUT{1:3},... diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index f816e32f..3e7bbab7 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -1,4 +1,4 @@ -function [ ] = BH_to_cisTEM_mapBack(PARAMETER_FILE, CYCLE,outputName, symmetry, MAX_EXPOSURE, varargin) +function [ ] = BH_to_cisTEM_mapBack(PARAMETER_FILE, CYCLE, output_prefix, symmetry, MAX_EXPOSURE, varargin) % Map back and align using the subtomograms as fiducial markers. @@ -10,9 +10,9 @@ % Otherwise, pass a string that points at a single volume to use. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -% Some flags that are worth keeping as options, but not accessible directly - - +% Some flags that are worth keeping as options, but not accessible +% directlyCT +% by the users (private methods-ish) global bh_global_imodProjectionShifts; if isempty(bh_global_imodProjectionShifts) % bh_global_imodProjectionShifts = [ -0.5, -0.5, 0.5 ; -0.5, -0.5, 0; 0.5,0.5,1.0 ]; @@ -20,76 +20,120 @@ bh_global_imodProjectionShifts = [ 0.5, -0.5, 0.5 ; 0.0, -0.5, 0; 0.5,0.5,1.0 ]; end +preShift = bh_global_imodProjectionShifts(1,:); +postShift = bh_global_imodProjectionShifts(2,1:2); - -tiltStart=1; +emc = BH_parseParameterFile(PARAMETER_FILE); MAX_EXPOSURE = EMC_str2double(MAX_EXPOSURE) if isnan(MAX_EXPOSURE) error('MAX_EXPOSURE is nan - if running from an interactive matlab session, did you enter as a string?'); end + +% Ideally, we would transform fully and go back to the non-rotated stack. I think with the apoferritin test set, +% The resolution will be high-enough to sort this out. +useFixedNotAliStack = false; + CYCLE = EMC_str2double(CYCLE); +cycle_numerator = ''; +cycle_denominator =''; -if CYCLE < 0 - CYCLE = abs(CYCLE); - doFullXform = true; -else - doFullXform = false; -end +% Copied over from tomoCPR. I think I can use this to write out all the partial stacks, +% Then rather than "runAlignments" have a command that combines all the stacks. +flgRunAlignments = true; +multi_node_run = false; +skip_to_the_end_and_run = false; -pixelShift = -1; -pixelMultiplier = 0; - -cacheAdd = ''; -if nargin > 5 - preShift = varargin{1}; - postShift = varargin{2}; - prjVectorShift = varargin{3}'; - if length(varargin) > 3 - cacheAdd = varargin{4}; + +if numel(CYCLE) == 3 + multi_node_run = true; + % After splitting, run the alignments while skipping everything else + if CYCLE(2) == 0 && CYCLE(3) == 0 + skip_to_the_end_and_run = true; else - cacheAdd = ''; + flgRunAlignments = false; end -else - - % in tilt the coords are transformed from the model coordinate system to - % the tomogram by [0.5,0.5,1.0]. After transformation, these are then - % subtracted back off. - preShift = bh_global_imodProjectionShifts(1,:); - postShift = bh_global_imodProjectionShifts(2,1:2); - prjVectorShift = bh_global_imodProjectionShifts(3,:)'; - % prjVectorShift = [0,0,-1]; - - + cycle_numerator = CYCLE(2); + cycle_denominator = CYCLE(3); + CYCLE = CYCLE(1); +end + +EMC_assert_numeric(CYCLE, 1, [0, inf]); + +% skip_to_the_end_and_run is only relevant when running on multiple nodes +if (skip_to_the_end_and_run && ~multi_node_run) + error('You are trying to skip to the end and run, but you are not running on multiple nodes'); end -% baseFile = sprintf('%s_%d_%2.2f_preShift_%2.2f_%2.2f_%2.2f_postShift_%2.2f_%2.2f_prjVect_%2.2f_%2.2f_%2.2f','microShiftsFollowup',MAX_EXPOSURE, preShift, postShift, prjVectorShift); -baseFile = outputName; %sprintf('%s_%d_%2.2f','withZeroedXF_IPFirst',MAX_EXPOSURE); -useFixedNotAliStack = false; cycleNumber = sprintf('cycle%0.3u', CYCLE); -emc = BH_parseParameterFile(PARAMETER_FILE); +% Always working at full binning, not emc.('Ali_samplingRate'); reconScaling = 1; -samplingRate = 1; % Always working at full binning. emc.('Ali_samplingRate'); -load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); -resForFitting = 1.3*mean(subTomoMeta.currentResForDefocusError); +% TODO: use this and add a block to calculate the FSC of the output reconstruction prior to refinement +MOL_MASS = emc.('particleMass'); + + -% % Add error check onrange for reasonable values. -% ctfRange = emc.('tomoCprDefocusRange')*10^10; -% ctfInc = emc.('tomoCprDefocusStep')*10^10; +% Used to calc defocus values using tilt instead of manually. Convention +% diff. +flgInvertTiltAngles = 0; -% calcCTF = emc.('tomoCprDefocusRefine'); +if (skip_to_the_end_and_run) + % The fractional runs have already copied everything to cache/mapback%d, + % so override the tmpCache. + tmpCache = ''; +else + tmpCache= emc.('fastScratchDisk'); +end + +if strcmpi(tmpCache, 'ram') + if isempty(getenv('EMC_CACHE_MEM')) + fprintf('Did not find a variable for EMC_CACHE_MEM\nSkipping ram\n'); + tmpCache= ''; + else + % I have no ideah how much is needed + if EMC_str2double(getenv('EMC_CACHE_MEM')) < 64 + fprintf('There is only 64 Gb of cache on ramdisk, not using'); + tmpCache = ''; + else + tmpCache=getenv('MCR_CACHE_ROOT'); + fprintf('Using the tmp EMC cache in ram at %s\n',tmpCache); + end + end +end + +% % % nWorkers = EMC_str2double(nWORKERS) nGPUs = emc.('nGPUs'); pInfo = parcluster(); -gpuScale=3*samplingRate +gpuScale=3; nWorkers = min(nGPUs*gpuScale,emc.('nCpuCores')); % 18 fprintf('Using %d workers as max of %d %d*nGPUs and %d nWorkers visible\n', ... nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); -tmpCache = sprintf('cache/to_cisTEM%s/',cacheAdd); +% Check to make sure it even exists +if isempty(dir(tmpCache)) + fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); + tmpCache = ''; +end +if isempty(tmpCache) + tmpCache='cache/to_cisTEM'; + flgCleanCache = 0; CWD = ''; +else + flgCleanCache = 1; + CWD = sprintf('%s/',pwd); + % Check for a trailing slash + slashCheck = strsplit(tmpCache,'/'); + if isempty(slashCheck{end}) + tmpCache = sprintf('%scache/to_cisTEM',tmpCache); % prefix for mapBack + else + tmpCache = sprintf('%s/cache/to_cisTEM',tmpCache); % prefix for mapBack + end +end + + system(sprintf('mkdir -p %s',tmpCache)); @@ -97,54 +141,88 @@ load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; +% TODO: use these to add an optional defocus fitting step +% So translational, optional defocus, angles +ctfRange = emc.('tomo_cpr_defocus_range')*10^10; +ctfInc = emc.('tomo_cpr_defocus_step')*10^10; +calcCTF = emc.('tomo_cpr_defocus_refine'); -system(sprintf('mkdir -p %smapBack%d',tmpCache, mapBackIter+1)); +[tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); -tiltNameList = fieldnames(subTomoMeta.mapBackGeometry); -tiltNameList = tiltNameList(~ismember(tiltNameList,{'tomoName','viewGroups'})); -nTiltSeries = length(tiltNameList); +if (multi_node_run && ~skip_to_the_end_and_run) + nParts = ceil(nTiltSeries ./ cycle_denominator); + tiltStart = 1+(cycle_numerator - 1)*nParts; + nTotal = nTiltSeries; + nTiltSeries = min(cycle_numerator*nParts,nTiltSeries); + fprintf('Running a subset of your tiltSeries %d - %d (of %d total)\n',tiltStart,nTiltSeries,nTotal); +end % Cycle 0 is named differently - I'll be deleting this in an overhaul of the way % the subTomoMeta is written. if (CYCLE) try +fprintf('Using Alignment geometry %s\n',cycleNumber); geometry = subTomoMeta.(cycleNumber).RawAlign; - fprintf('Using Alignment geometry %s\n',cycleNumber); - catch + catch +fprintf('Using Average geometry %s\n',cycleNumber); geometry = subTomoMeta.(cycleNumber).Avg_geometry; - fprintf('Using Average geometry %s\n',cycleNumber); - end + end else try +fprintf('Using Alignment geometry %s\n',cycleNumber); geometry = subTomoMeta.(cycleNumber).RawAlign; - fprintf('Using Alignment geometry %s\n',cycleNumber); - - catch + catch +fprintf('Using Average geometry %s\n',cycleNumber); geometry = subTomoMeta.(cycleNumber).geometry; - fprintf('Using Average geometry %s\n',cycleNumber); - end + end end + +load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); +resForFitting = 1.3*mean(subTomoMeta.currentResForDefocusError); tiltGeometry = subTomoMeta.tiltGeometry; -% Assume No 2d CTF until proven otherwise +% TODO: this is a bit of an aritfact, can be removed. +outCTF = '_ctf'; + +is_first_run = true; + +mbOUT = {[tmpCache],'dummy'}; +tiltStart=1; +firstTilt = true; + +pixelShift = -1; -firstTilt = 1; -nFidsTotalDataSet= 0; -iCell = 1; +iCell = 0; output_cell = {}; -% TODO split this up into chunks +newstack_file = sprintf('%s/temp_particle_stack.newstack',mbOUT{1}); for iTiltSeries = tiltStart:nTiltSeries + n_particles_added_to_stack = 0; + if (skip_to_the_end_and_run) + continue; + end - nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos; + if (useFixedNotAliStack) + tilt_filename = sprintf('%sfixedStacks/%s.fixed',CWD,tiltNameList{iTiltSeries}); + else + tilt_filename = sprintf('%saliStacks/%s_ali%d.fixed',CWD,tiltNameList{iTiltSeries},mapBackIter+1); + end + tilt_filename = sprintf('%saliStacks/%s_ali%d.fixed', CWD, tiltNameList{iTiltSeries}, mapBackIter + 1); + + mapBackRePrjSize = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize'); + % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); + nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos if nTomograms == 0 - % No points were saved after template matching so skip this tilt seoarries + % No points were saved after template matching so skip this tilt series % altogether. continue end + skip_this_tilt_series_because_it_is_empty = false(nTomograms,1); + + % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); tomoList = {}; tomoIDX = 1; for iTomo = 1:size(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords,1) @@ -152,17 +230,16 @@ if any(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords(iTomo,:)) tomoList{tomoIDX} = sprintf('%s_%d',tiltNameList{iTiltSeries},iTomo); - tiltList{tomoIDX} = sprintf('%saliStacks/%s_ali%d.fixed',... - CWD,tiltNameList{iTiltSeries},mapBackIter+1); % Only increment if values found. tomoIDX = tomoIDX + 1; - end - + end end + + [~,tiltBaseName,~] = fileparts(tilt_filename); + mbOUT{2} = tiltBaseName; if (mapBackIter) - localFile = sprintf('%smapBack%d/%s_ali%d_ctf.local', ... - CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter); + localFile = sprintf('%s/%s_ali%d_ctf.local', CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter); else localFile = sprintf('%sfixedStacks/%s.local',CWD,tiltNameList{iTiltSeries}); end @@ -170,47 +247,107 @@ if exist(localFile,'file') fprintf('Found local file %s\n.', localFile); else - fprintf('No local transforms found.\n'); + fprintf('No local transforms requested.\n'); localFile = 0; end % The model is scaled to full sampling prior to passing to tiltalign, % make sure the header in the synthetic stack is set appropriately. - fullPixelSize = emc.pixel_size_angstroms; - pixelSize = fullPixelSize.*samplingRate; + pixel_size = emc.pixel_size_angstroms; + + try + eraseMaskType = emc.('Peak_mType'); + eraseMaskRadius = emc.('Peak_mRadius') ./ pixel_size; + fprintf('Further restricting peak search to radius of [%f %f %f] pixels\n', eraseMaskRadius); + eraseMask = 1; + catch + eraseMask = 0; + fprintf('\n'); + end - PARTICLE_RADIUS = floor(max(emc.('particleRadius')./pixelSize)); - [~,tiltBaseName,~] = fileparts(tiltList{1}); - mbOUT = {[tmpCache],[mapBackIter+1],[tiltBaseName]}; - fprintf('\nmBOUT name is %smapBack%d/%s\n',mbOUT{1:3}); + particle_radius = floor(max(emc.('particleRadius')./pixel_size)); + + % TODO, is this too restricted? + % current default peak_mask_fraction = 0.4 + peak_search_radius = floor(emc.peak_mask_fraction .* particle_radius .* [1,1]); + + % FIXME: this should be in parseParameterFile + try + lowPassCutoff = emc.('tomoCprLowPass'); + fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n.', lowPassCutoff); + catch + % TODO are these range limits okay? + lowPassCutoff = 1.5.*mean(subTomoMeta.currentResForDefocusError); + if (lowPassCutoff < 10) + lowPassCutoff = 10; + elseif (lowPassCutoff > 24) + lowPassCutoff = 24; + end + fprintf('Using an internatlly determined lowpass cutoff of %3.3f Ang\n.',... + lowPassCutoff); + end + + % FIXME: this can also be in parseParameterFile + if lowPassCutoff < 2* pixel_size + fprintf('Psych, the cutoff is being set to Nyquist'); + lowPassCutoff = 2*pixel_size; + end + % FIXME: this should be in parseParameterFile + min_res_for_ctf_fitting = 10.0; + if (calcCTF) + try + min_res_for_ctf_fitting = emc.('min_res_for_ctf_fitting'); + catch + end + + if sqrt(2)*pixel_size > min_res_for_ctf_fitting + fprintf('Warning the current resolution is too low to refine the defocus. Turning off this feature'); + calcCTF = false; + end + end - tiltHeader = getHeader(MRCImage(tiltList{1},0)); - % This is only needed in the re-projection of the model. I don't think it - % should affect anything, but double check. FIXME - maxZ = 100; + + + % Get the thickest for recon + maxZ = 0; + + % The + tiltHeader = getHeader(MRCImage(tilt_filename, 0)); + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{1}).tiltName; + [ maxZ ] = emc_get_max_specimen_NZ(subTomoMeta.mapBackGeometry.tomoName, ... + subTomoMeta.mapBackGeometry.(tiltName).coords, ... + tomoList, ... + nTomograms, ... + 1); + + fprintf('combining thickness and shift, found a maxZ of %d\n',maxZ); + % xyzproj assumes centered in Z, so add extra height for z offsets to create + % the true "in microsope" dimension - % The tilt angles are the same for each tomo, so it is okay to just use - % number 1 here. + reconstruction_size = [tiltHeader.nX, tiltHeader.nY, maxZ]; + originRec = emc_get_origin_index(reconstruction_size); + TLT = tiltGeometry.(tomoList{1}); - iRawTltName = sprintf('%smapBack%d/%s_align.rawtlt',mbOUT{1:3}) + iRawTltName = sprintf('%s/%s_align.rawtlt',mbOUT{1:2}) iTiltFile = fopen(iRawTltName, 'w'); rawTLT = sortrows(TLT(:,[1,4]),1); fprintf(iTiltFile,'%f\n',rawTLT(:,2)'); fclose(iTiltFile); - coordOUT = fopen(sprintf('%smapBack%d/%s.coord',mbOUT{1:3}),'w'); - coordSTART = fopen(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3}),'w'); + coordOUT = fopen(sprintf('%s/%s.coord',mbOUT{1:2}),'w'); + coordSTART = fopen(sprintf('%s/%s.coord_start',mbOUT{1:2}),'w'); - defOUT = fopen(sprintf('%smapBack%d/%s.defAng',mbOUT{1:3}),'w'); + defOUT = fopen(sprintf('%s/%s.defAng',mbOUT{1:2}),'w'); % Track the number of fiducials in order to scale the K-factor to more or less % aggressivley downweight outliers in the alignment nFidsTotal = 0; + fidIDX = 0; for iTomo = 1:nTomograms TLT = tiltGeometry.(tomoList{iTomo}); @@ -221,34 +358,30 @@ preExposure = diff(sorted_doseList(:,2)); preExposure = [preExposure; preExposure(end)]; preExposure = postExposure - preExposure(doseIDX)'; - - - + + % Extract a "defocus file" for tilt to calculate the defocus for each + % Extract a "defocus file" for tilt to calculate the defocus for each % fiducial also considering the local alignment. If this works, I can % get rid of defAng - iDefocusFileName = sprintf('%smapBack%d/%s_align.defocus',mbOUT{1:3}); + iDefocusFileName = sprintf('%s/%s_align.defocus',mbOUT{1:2}); iDefocusFile = fopen(iDefocusFileName,'w'); defTLT = sortrows(TLT(:,[1,15]),1); - % Imod expects nanometers and underfocus positive (origin on specimen) - % whereas I let the origin be the focal plane such that underfocus is - % negative. - fprintf(iDefocusFile,'%f\n',abs(defTLT(:,2)').*(10^9)); + fprintf(iDefocusFile,'%f\n',abs(defTLT(:,2)').*10^9); fclose(iDefocusFile); % We also need the transform from the microscope frame in order to % get an accurate defocus value. Not sure if I should be binning? % Additionally, we do NOT want the model for alignment in the % microscope frame, - iXFName = sprintf('%smapBack%d/%s_align.XF',mbOUT{1:3}); + iXFName = sprintf('%s/%s_align.XF',mbOUT{1:2}); iXF = fopen(iXFName,'w'); - - - if (useFixedNotAliStack || doFullXform) + if (useFixedNotAliStack) % 20190509 - I think this is royally screwing things up FIXME % Commenting this out invalidates the defocus vals + % positionn in stack, imod rotation matrix (2x2), x,y shift (unbinned) xfTLT = sortrows(TLT(:,[1,7:10,2,3],1)); fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT(:,2:7)'); fclose(iXF); @@ -257,7 +390,7 @@ if (useFixedNotAliStack) isEven = 1; - iXFBase = sprintf('%smapBack%d/%s_align_base.XF',mbOUT{1:3}); + iXFBase = sprintf('%s/%s_align_base.XF',mbOUT{1:2}); iXFB = fopen(iXFBase,'w'); for ix = 1:size(xfTLT,1) fprintf(iXFB,'%f %f %f %f %f %f\n',[1,0,0,1,-isEven,-isEven]); @@ -268,10 +401,11 @@ % We need to invert this transform to map from the aligned stack to the % fixed stack - iXFName_inv = sprintf('%smapBack%d/%s_align_inv.XF',mbOUT{1:3}); + iXFName_inv = sprintf('%s/%s_align_inv.XF',mbOUT{1:2}); system(sprintf('xfinverse %s %s', iXFName, iXFName_inv)); end else + % Create an identity transform for the model % 20190509 - I think this is royally screwing things up FIXME % Commenting this out invalidates the defocus vals xfTLT = zeros(size(TLT,1),6); @@ -283,60 +417,55 @@ % fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT(:,2:7)'); % fclose(iXF); end - - + positionList = geometry.(tomoList{iTomo}); + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); + + tiltHeader = getHeader(MRCImage(tilt_filename,0)); - sTX = floor(tiltHeader.nX ); - sTY = floor(tiltHeader.nY ); - - originPrj = floor([sTX,sTY,1]./2) + 1; + fullTiltSizeXandY = [tiltHeader.nX,tiltHeader.nY]; + sTX = floor(tiltHeader.nX); + sTY = floor(tiltHeader.nY); + tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; % reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - tomoReconCoords = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); - + tomoReconCoords = (subTomoMeta.reconGeometry.(tomoList{iTomo})); - originVol = floor(tomoReconCoords(1,1:3)./2) + 1; - reconShift = tomoReconCoords(2,1:3); - - maxZ = 1000; % Does not seem to affect anything. - reconstructionSize = [tiltHeader.nX,tiltHeader.nY,maxZ]; - originRec = floor(reconstructionSize./2) + 1; - - + tomo_origin_in_tomo_frame = emc_get_origin_index(tomoReconCoords(1,1:3)); + tomo_origin_wrt_tilt_origin = tomoReconCoords(2,1:3); + nPrjs = size(TLT,1); nSubTomos = size(positionList,1); - - if (iTomo == 1) - fidIDX = 0; + if (nSubTomos == 0) + % No points were saved after template matching so skip this tilt series + % altogether. + skip_this_tilt_series_because_it_is_empty(iTomo) = true; + continue; end - + modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector'); for iSubTomo = 1:nSubTomos - - rSubTomo = reshape(positionList(iSubTomo,17:25),3,3); - prjVector = (positionList(iSubTomo,11:13)./samplingRate) - originVol + reconShift; - - - % % % nRefs = 1; - % % % if (nRefs > 1) - % % % iClassIDX = positionList(iSubTomo,26); - % % % else - % % % iClassIDX = 1; - % % % end - - % prjVector = prjVector - [0.5,0.5,1.0]; %prjVectorShift; - prjVector = prjVector - preShift; + subtomo_rot_matrix = reshape(positionList(iSubTomo,17:25),3,3); + subtomo_origin_in_tomo_frame = (positionList(iSubTomo,11:13)); + subtomo_origin_wrt_tilt_origin = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_tilt_origin; + + % This extra shift came from experiments with real data but is both annoying and not understood. + subtomo_origin_wrt_tilt_origin = subtomo_origin_wrt_tilt_origin - emc.flgPreShift; + + % subTomo origin relative to reconLowerLeft + subtomo_origin_in_sample = originRec + subtomo_origin_wrt_tilt_origin; % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot*prjVector' + [originRec(1),originRec(3),originRec(2)]' - prjVectorShift([1,3,2]), fidIDX); + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_tilt_origin' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2]), fidIDX); nPrjsIncluded = 0; for iPrj = 1:nPrjs @@ -345,62 +474,51 @@ if (abs(TLT(iPrj_nat,11)) <= MAX_EXPOSURE) nPrjsIncluded = nPrjsIncluded + 1; - % imod is indexing from zero zCoord = iPrj_nat; + % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) + rTilt = BH_defineMatrix([0,TLT(iPrj_nat,4),0],'SPIDER','inv'); + prjCoords = rTilt*subtomo_origin_wrt_tilt_origin'; + % I think this is for comparison with the values obtained from projecting using IMOD: FIXME + fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, abs(TLT(iPrj_nat,15)) - prjCoords(3).*pixel_size.*10^-10); - rTilt = BH_defineMatrix([90,1.*TLT(iPrj_nat,4),-90],'Bah','forwardVector'); - - - prjCoords = rTilt*prjVector'; - - fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)); - % d1 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; - % d2 = -1.*((samplingRate.*prjCoords(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; - - d1 = -1.*(samplingRate.*prjVector(3).*fullPixelSize.*10^-10+TLT(iPrj_nat,15))*10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm + % Defocus value adjusted for Z coordinate in the tomogram. nm + d1 = (abs(TLT(iPrj_nat,15)) - subtomo_origin_wrt_tilt_origin(3).*pixel_size.*10^-10) * 10^9; d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value - fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',fidIDX, tomoNumber,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(rSubTomo,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); - - nFidsTotalDataSet = nFidsTotalDataSet + 1; - + fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n', ... + fidIDX, tomoNumber, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_nat,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_nat), postExposure(iPrj_nat), positionList(iSubTomo,7)); else fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',-9999, -9999,-9999,1.0,1.0,1.0,1,1,1,1,1,1,1,1,1,0,0,1); - - nFidsTotalDataSet = nFidsTotalDataSet + 1; end + % nFidsTotalDataSet = nFidsTotalDataSet + 1; end % loop over tilt projections - fidIDX = fidIDX + 1; - end % loop over subtomos - - - end + end % loop over subtomos + end % end of loop over tomograms on this tilt-series + % No subtomos remain + if all( skip_this_tilt_series_because_it_is_empty ) + continue; + end fclose(coordOUT); fclose(coordSTART); - p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 ',... - '%smapBack%d/%s.coord %smapBack%d/%s.3dfid > /dev/null'], ... - mbOUT{1:3},mbOUT{1:3}); + '%s/%s.coord %s/%s.3dfid'], ... + mbOUT{1:2},mbOUT{1:2}); system(p2m); - - taStr = [sprintf('%f',rawTLT(1,2))]; for iTa = 2:length(rawTLT(:,2)) taStr = [taStr sprintf(',%f',rawTLT(iTa,2))]; end - - if (localFile) lastLine1 = sprintf('LOCALFILE %s', localFile) % Used if GPU fails @@ -427,31 +545,32 @@ % refPrj = zeros(sTX,sTY,iTLT, 'single'); iSave = 1; - reModFileName = sprintf('%smapBack%d/%s_%d_reMod.sh',mbOUT{1:3},iSave); + reModFileName = sprintf('%s/%s_%d_reMod.sh',mbOUT{1:2},iSave); reModFile = fopen(reModFileName,'w'); invertTiltAngles = 0; fprintf(reModFile,['#!/bin/bash\n\n',... 'tilt -StandardInput << EOF\n',... 'input %s\n', ... - 'output %smapBack%d/%s.fid\n', ... + 'output %s/%s.fid\n', ... 'COSINTERP 0\n', ... 'THICKNESS %d\n', ... - 'TILTFILE %smapBack%d/%s_align.rawtlt \n', ... - 'DefocusFile %smapBack%d/%s_align.defocus \n', ... + 'TILTFILE %s/%s_align.rawtlt\n', ... + 'DefocusFile %s/%s_align.defocus\n', ... 'PixelForDefocus %f,%f\n', ... - 'AngleOutputFile %smapBack%d/%s.defAngTilt\n', ... - 'AlignTransformFile %smapBack%d/%s_align.XF\n', ... - 'ProjectModel %smapBack%d/%s.3dfid\n', ... + 'AngleOutputFile %s/%s.defAngTilt\n', ... + 'AlignTransformFile %s/%s_align.XF\n', ... + 'ProjectModel %s/%s.3dfid\n', ... '%s\n',... '%s\n',... '%s\n',... - 'EOF'],tiltList{1}, mbOUT{1:3}, maxZ, ... - mbOUT{1:3},... - mbOUT{1:3},... - pixelSize/10, invertTiltAngles,... % Ang --> nm - mbOUT{1:3},... - mbOUT{1:3},... - mbOUT{1:3},... + 'EOF'],tilt_filename, mbOUT{1:2}, maxZ, ... + mbOUT{1:2},... + mbOUT{1:2},... + pixel_size./10, ... % Ang --> nm + 0, ... % do not invert the tilt angles + mbOUT{1:2},... + mbOUT{1:2},... + mbOUT{1:2},... lastLine1,lastLine2,... lastLine3); @@ -472,30 +591,35 @@ % transform the projected model back to the fixed stack frame, and % then convert to text. - system(sprintf('imodtrans -2 %s %smapBack%d/%s.fid %smapBack%d/%s.invfid', iXFName_inv, mbOUT{1:3},mbOUT{1:3})); + system(sprintf('imodtrans -2 %s %s/%s.fid %s/%s.invfid > /dev/null', iXFName_inv, mbOUT{1:2},mbOUT{1:2})); system(sprintf(['model2point -contour -zero ',... - '%smapBack%d/%s.invfid %smapBack%d/%s.coordPrj'],... - mbOUT{1:3}, mbOUT{1:3})) + '%s/%s.invfid %s/%s.coordPrj > /dev/null'],... + mbOUT{1:2}, mbOUT{1:2})) else system(sprintf(['model2point -contour -zero ',... - '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'],... - mbOUT{1:3}, mbOUT{1:3})) + '%s/%s.fid %s/%s.coordPrj > /dev/null'],... + mbOUT{1:2}, mbOUT{1:2})) end try - fidList = load(sprintf('%smapBack%d/%s.coordPrj',mbOUT{1:3})); + fidList = load(sprintf('%s/%s.coordPrj',mbOUT{1:2})); catch - fprintf('\nWarning, did not load the projected coords\nSkipping along'); - continue; + error('\nWarning, did not load the projected coords\nSkipping along'); end - parList = load(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3})); - defList = load(sprintf('%smapBack%d/%s.defAngTilt',mbOUT{1:3})); - + % unused + parList = load(sprintf('%s/%s.coord_start',mbOUT{1:2})); + defList = load(sprintf('%s/%s.defAngTilt',mbOUT{1:2})); + % Need to shift again from the model coordinate system - fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(prjVectorShift(1:2)', size(fidList,1),1); + % Columns are + % particle idx, from 0 + % x + % y + % projection idx, from 0 + fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(emc.prjVectorShift(1:2)', size(fidList,1),1); foundNans = sum(isnan(fidList(:,3))); if (foundNans) fprintf('\n\t\tThere are %d NaNs in the projected fiducial list %3.3f\n\n',foundNans, foundNans/size(fidList,1)*100); @@ -503,34 +627,27 @@ error("Exiting"); end + % Give every instance of each fiducial a unique identifier. fidList = [1:size(fidList,1);fidList']'; - - particlePad = 2.0; - tileRadius = floor(particlePad.*PARTICLE_RADIUS); + tileRadius = floor(particlePad.*particle_radius); tileSize = (2.*tileRadius).*[1,1]; tileOrigin = floor(tileSize./2) + 1; nFidsTotal = numel(unique(fidList(parList(:,1)~=-9999,2))); - - - if (useFixedNotAliStack) - tiltSeries = sprintf('%sfixedStacks/%s.fixed',CWD,tiltName); - else - tiltSeries = sprintf('%saliStacks/%s_ali%d.fixed',CWD,tiltName,mapBackIter+1); - end + % This will need to be changed to aggregate output_particle_stack = zeros([tileSize,nFidsTotal*nPrjsIncluded],'single'); iGpuDataCounter = 1; + if (firstTilt) - iDataCounter = 1; - starFile = fopen(sprintf('%s.star',baseFile),'w'); + starFile = fopen(sprintf('%s.star',output_prefix),'w'); fprintf(starFile, [ ... '# Written by emClarity Version 2.0.0-alpha on %s\n\n' ... 'data_\n\n' ... @@ -565,16 +682,15 @@ '_cisTEMTotalExposure #28\n' ... '# POS PSI THETA PHI SHX SHY DF1 DF2 ANGAST PSHIFT OCC LogP SIGMA SCORE CHANGE PSIZE VOLT Cs AmpC BTILTX BTILTY ISHFTX ISHFTY 2DCLS TGRP PARGRP PREEXP TOTEXP\n' ... ], datetime); - - - firstTilt = 0; + + firstTilt = false; end if (useFixedNotAliStack) fullXform = load(iXFName_inv); end - STACK = single(getVolume(MRCImage(tiltSeries))); + STACK = single(getVolume(MRCImage(tilt_filename))); for iPrj = 1:nPrjs @@ -582,8 +698,7 @@ continue; end - dataPrj = STACK(:,:,TLT(iPrj,1)); - + % Both are ordered by fiducial (imod contour number) but are not % explicitly checked to correspond. Should this be done? @@ -599,119 +714,134 @@ if (wrkPar(iFid,1) == -9999) continue; end - - + pixelX = wrkFid(iFid,3) - pixelShift + postShift(1); pixelY = wrkFid(iFid,4) - pixelShift + postShift(2); ox = floor(pixelX) - tileRadius; oy = floor(pixelY) - tileRadius; - sx = pixelX - floor(pixelX); sy = pixelY - floor(pixelY); particle_was_skipped = false; if ( ox > 0 && oy > 0 && ox + 2*tileRadius < sTX && oy +2*tileRadius < sTY ) - output_particle_stack(:,:,iGpuDataCounter) = dataPrj(ox:ox+2.*tileRadius-1,oy:oy+2.*tileRadius-1); - else - particle_was_skipped = true; - output_particle_stack(:,:,iGpuDataCounter) = randn(tileSize,'single').*0.1; % Why am I not just skipping these? - end + output_particle_stack(:,:,iGpuDataCounter) = STACK(ox:ox+2.*tileRadius-1,oy:oy+2.*tileRadius-1,TLT(iPrj,1)); + - if (useFixedNotAliStack) - rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); - RF = fullXform(TLT(iPrj,1),1:4); - rotFull = rTilt*[RF(1), RF(2), 0; RF(3), RF(4), 0; 0, 0, 1]*reshape(wrkPar(iFid,7:15),3,3); - else + if (useFixedNotAliStack) + rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); + RF = fullXform(TLT(iPrj,1),1:4); + rotFull = rTilt*[RF(1), RF(2), 0; RF(3), RF(4), 0; 0, 0, 1]*reshape(wrkPar(iFid,7:15),3,3); + else + % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); + rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','forwardVector'); + + rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); + end + eul = rotm2eul(rotFull,'ZYZ'); + e1 = 180./pi.*eul(1); - % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); - rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','forwardVector'); + e2 = 180./pi.*eul(2); + e3 = 180./pi.*eul(3); - rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); - end - - eul = rotm2eul(rotFull,'ZYZ'); - e1 = 180./pi.*eul(1); - - e2 = 180./pi.*eul(2); - e3 = 180./pi.*eul(3); - - - phaseShift = 0.0; - occupancy = 100.0; % TODO test replacement with CCC score? - logp = -1000; - sigma = 10.0; - score = 10.0; % TODO test with scaled CCC score? - scoreChange = 0.0; - pixelSize = emc.pixel_size_angstroms; - micVoltage = emc.('VOLTAGE') * 10^-3; - micCS = emc.('Cs') * 10^3; - ampContrast = emc.('AMPCONT') * 10^0; - beamTiltX = 0.0; - beamTiltY = 0.0; - beamTiltShiftX = 0.0; - beamTiltShiftY = 0.0; - best2dClass = 0.0; - if (particle_was_skipped) - beamTiltGroup = 0; % FSC half set, coopting this param for now. - else - beamTiltGroup = wrkPar(iFid,18); % FSC half set, coopting this param for now. - end - particleGroup = wrkPar(iFid,3); - preExposure = wrkPar(iFid,16); - totalExposure = wrkPar(iFid,17); - - xShift = pixelMultiplier*sx*pixelSize; - yShift = pixelMultiplier*sy*pixelSize; - - % df1 = wrkPar(iFid,4); - % df2 = wrkPar(iFid,5); - % dfA = wrkPar(iFid,6); - df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; - df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; - dfA = wrkPar(iFid,6); - - fprintf(starFile, '%8u %7.2f %7.2f %7.2f %9.2f %9.2f %8.1f %8.1f %7.2f %7.2f %5i %7.2f %9i %10.4f %7.2f %8.5f %7.2f %7.2f %7.4f %7.3f %7.3f %7.3f %7.3f %5i %5i %8u %7.2f %7.2f\n', ... - iDataCounter,-e1,-e2,-e3,xShift,yShift, ... - df1,df2,dfA, ... - phaseShift, occupancy, logp, sigma, score, scoreChange, ... - pixelSize, micVoltage, micCS, ampContrast, ... - beamTiltX, beamTiltY, beamTiltShiftX, beamTiltShiftY, ... - best2dClass, beamTiltGroup, particleGroup, preExposure, totalExposure); - - - iDataCounter = iDataCounter + 1; - iGpuDataCounter = iGpuDataCounter + 1; - - end + + phaseShift = 0.0; + occupancy = 100.0; % TODO test replacement with CCC score? + logp = -1000; + sigma = 10.0; + score = 10.0; % TODO test with scaled CCC score? + scoreChange = 0.0; + pixelSize = emc.pixel_size_angstroms; + micVoltage = emc.('VOLTAGE') * 10^-3; + micCS = emc.('Cs') * 10^3; + ampContrast = emc.('AMPCONT') * 10^0; + beamTiltX = 0.0; + beamTiltY = 0.0; + beamTiltShiftX = 0.0; + beamTiltShiftY = 0.0; + best2dClass = 0.0; + if (particle_was_skipped) + beamTiltGroup = 0; % FSC half set, coopting this param for now. + else + beamTiltGroup = wrkPar(iFid,18); % FSC half set, coopting this param for now. + end + particleGroup = wrkPar(iFid,3); + preExposure = wrkPar(iFid,16); + totalExposure = wrkPar(iFid,17); + + xShift = sx*pixelSize; + yShift = sy*pixelSize; + + + % df1 = ( wrkPar(iFid,4) + wrkPar(iFid,5)) * 10; + % df2 = ( wrkPar(iFid,4) - wrkPar(iFid,5)) * 10; + % dfA = wrkPar(iFid,6) + % fidIDX, tomoNumber, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_nat,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_nat), postExposure(iPrj_nat), positionList(iSubTomo,7)); + + df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; + df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; + dfA = wrkPar(iFid,6); + + fprintf(starFile, '%8u %7.2f %7.2f %7.2f %9.2f %9.2f %8.1f %8.1f %7.2f %7.2f %5i %7.2f %9i %10.4f %7.2f %8.5f %7.2f %7.2f %7.4f %7.3f %7.3f %7.3f %7.3f %5i %5i %8u %7.2f %7.2f\n', ... + iDataCounter,-e1,-e2,-e3,xShift,yShift, ... + df1,df2,dfA, ... + phaseShift, occupancy, logp, sigma, score, scoreChange, ... + pixelSize, micVoltage, micCS, ampContrast, ... + beamTiltX, beamTiltY, beamTiltShiftX, beamTiltShiftY, ... + best2dClass, beamTiltGroup, particleGroup, preExposure, totalExposure); + + + iDataCounter = iDataCounter + 1; + iGpuDataCounter = iGpuDataCounter + 1; + n_particles_added_to_stack = n_particles_added_to_stack + 1; + end % if on windowing + end % end of fiducial loop end % end of prj loop - - output_cell{iCell}= gather(output_particle_stack); + % Trim the stack to account for windowing skips + output_particle_stack = gather(output_particle_stack(:,:,1:n_particles_added_to_stack)); + tmp_stack_filename = sprintf('%s/%s_%d.mrc',mbOUT{1:2},iCell); + SAVE_IMG(output_particle_stack, tmp_stack_filename, pixelSize); + + + newstack_file_handle = fopen(newstack_file,'a'); + fprintf(newstack_file_handle, '%s\n',tmp_stack_filename); + fprintf(newstack_file_handle, '0-%d\n',n_particles_added_to_stack-1); + fclose(newstack_file_handle); + + % output_cell{iCell}= gather(output_particle_stack); iCell = iCell + 1; - -end + +end % end of the loop over tilt series + fclose(starFile); +newstack_file_with_n_stacks = sprintf('%s/%s.newstack_full',mbOUT{1:2}); +fh = fopen(newstack_file_with_n_stacks,'w'); +fprintf(fh,'%d\n', iCell); +fclose(fh); + +system(sprintf('cat %s >> %s',newstack_file,newstack_file_with_n_stacks)); +system(sprintf('newstack -FileOfInputs %s %s.mrc',newstack_file_with_n_stacks,output_prefix)); -SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',baseFile),pixelSize); +% SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',output_prefix),pixelSize); maxThreads = emc.('nCpuCores'); %%%%%%%%%%%%%%%%%%%%%%%%%% % Initial reconstruction %%%%%%%%%%%%%%%%%%%%%%%%%%% -system(sprintf('rm -f %s_rec.sh',baseFile)); -recScript = fopen(sprintf('%s_rec.sh',baseFile), 'w'); +system(sprintf('rm -f %s_rec.sh',output_prefix)); +recScript = fopen(sprintf('%s_rec.sh',output_prefix), 'w'); fprintf(recScript,[ ... '#!/bin/bash\n\n', ... '%s << eof\n', ... - '%s.mrc\n', ... sprintf('%s.mrc',baseFile) - '%s.star\n', ... sprintf('%s.star',baseFile) + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s.star\n', ... sprintf('%s.star',output_prefix) 'none.mrc\n', ... '%s_rec1.mrc\n',... '%s_rec2.mrc\n',... @@ -743,28 +873,30 @@ 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : '%2.2d\n', ...Max. threads to use for calculation [36] : - ], getenv('EMC_RECONSTRUCT3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... + ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); fprintf(recScript, '\neof\n'); fclose(recScript); -system(sprintf('chmod a=wrx %s_rec.sh',baseFile)); +system(sprintf('chmod a=wrx %s_rec.sh',output_prefix)); pause(3) -system(sprintf('./%s_rec.sh',baseFile)); +system(sprintf('./%s_rec.sh',output_prefix)); +% FIXME: add a parameter to control this as we will probably want to consider just moving into cisTEM from the start +return; %%%%%%%%%%%%%%%%%%%%%%%%% % Refine %%%%%%%%%%%%%%%%%%%%%%%%%%%% -system(sprintf('rm -f %s_ref.sh',baseFile)); -refineScript = fopen(sprintf('%s_ref.sh',baseFile), 'w'); +system(sprintf('rm -f %s_ref.sh',output_prefix)); +refineScript = fopen(sprintf('%s_ref.sh',output_prefix), 'w'); fprintf(refineScript,[ ... '#!/bin/bash\n\n', ... '%s << eof\n', ... - '%s.mrc\n', ... sprintf('%s.mrc',baseFile) - '%s.star\n', ... sprintf('%s.star',baseFile) + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s.star\n', ... sprintf('%s.star',output_prefix) '%s_recFilt.mrc\n',... '%s_stats.txt\n',... 'yes\n',... Use statistics [Yes] : @@ -812,7 +944,7 @@ 'yes\n',...Normalize input reconstruction [Yes] : 'no\n',...Threshold input reconstruction [No] : '%2.2d\n', ...Max. threads to use for calculation [36] : - ], getenv('EMC_REFINE3D'),baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... + ], getenv('EMC_REFINE3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... resForFitting,resForFitting,maxThreads); @@ -820,20 +952,20 @@ fprintf(refineScript, '\neof\n'); fclose(refineScript); pause(3); -system(sprintf('chmod a=wrx %s_ref.sh',baseFile)); -system(sprintf('./%s_ref.sh',baseFile)); +system(sprintf('chmod a=wrx %s_ref.sh',output_prefix)); +system(sprintf('./%s_ref.sh',output_prefix)); %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Reconstruct refined %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -system(sprintf('rm -f %s_rec2.sh',baseFile)); -recScript = fopen(sprintf('%s_rec2.sh',baseFile), 'w'); +system(sprintf('rm -f %s_rec2.sh',output_prefix)); +recScript = fopen(sprintf('%s_rec2.sh',output_prefix), 'w'); fprintf(recScript,[ ... '#!/bin/bash\n\n', ... '%s << eof\n', ... - '%s.mrc\n', ... sprintf('%s.mrc',baseFile) - '%s_refined.star\n', ... sprintf('%s.star',baseFile) + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s_refined.star\n', ... sprintf('%s.star',output_prefix) 'none.mrc\n', ... '%s_rec1.mrc\n',... '%s_rec2.mrc\n',... @@ -865,7 +997,7 @@ 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : '%2.2d\n', ...Max. threads to use for calculation [36] : - ], getenv('EMC_RECONSTRUCT3D'), baseFile, baseFile, baseFile, baseFile, baseFile, baseFile, ... + ], getenv('EMC_RECONSTRUCT3D'), output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); @@ -873,9 +1005,9 @@ fclose(recScript); pause(2) -system(sprintf('chmod a=wrx %s_rec2.sh',baseFile)); +system(sprintf('chmod a=wrx %s_rec2.sh',output_prefix)); pause(2) -system(sprintf('./%s_rec2.sh',baseFile)); +system(sprintf('./%s_rec2.sh',output_prefix)); end From 7ccd09cf9f13f1e81ff7a695d35b531d24e0aa8b Mon Sep 17 00:00:00 2001 From: himesb Date: Sun, 18 Feb 2024 12:20:47 -0500 Subject: [PATCH 029/151] Fix indexing problem in substack saving for toCistem --- synthetic/BH_to_cisTEM_mapBack.m | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 3e7bbab7..060c92e7 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -198,6 +198,8 @@ iCell = 0; output_cell = {}; newstack_file = sprintf('%s/temp_particle_stack.newstack',mbOUT{1}); +newstack_file_handle = fopen(newstack_file,'w'); + for iTiltSeries = tiltStart:nTiltSeries n_particles_added_to_stack = 0; if (skip_to_the_end_and_run) @@ -727,13 +729,18 @@ particle_was_skipped = false; if ( ox > 0 && oy > 0 && ox + 2*tileRadius < sTX && oy +2*tileRadius < sTY ) output_particle_stack(:,:,iGpuDataCounter) = STACK(ox:ox+2.*tileRadius-1,oy:oy+2.*tileRadius-1,TLT(iPrj,1)); - + % else + % particle_was_skipped = true; + % output_particle_stack(:,:,iGpuDataCounter) = randn(tileSize,'single').*0.1; % Why am I not just skipping these? + % end if (useFixedNotAliStack) rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); RF = fullXform(TLT(iPrj,1),1:4); rotFull = rTilt*[RF(1), RF(2), 0; RF(3), RF(4), 0; 0, 0, 1]*reshape(wrkPar(iFid,7:15),3,3); else + + % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','forwardVector'); @@ -770,10 +777,9 @@ particleGroup = wrkPar(iFid,3); preExposure = wrkPar(iFid,16); totalExposure = wrkPar(iFid,17); - - xShift = sx*pixelSize; - yShift = sy*pixelSize; - + pixelMultiplier = 0; + xShift = pixelMultiplier*sx*pixelSize; + yShift = pixelMultiplier*sy*pixelSize; % df1 = ( wrkPar(iFid,4) + wrkPar(iFid,5)) * 10; % df2 = ( wrkPar(iFid,4) - wrkPar(iFid,5)) * 10; @@ -795,29 +801,25 @@ iDataCounter = iDataCounter + 1; iGpuDataCounter = iGpuDataCounter + 1; - n_particles_added_to_stack = n_particles_added_to_stack + 1; end % if on windowing end % end of fiducial loop end % end of prj loop % Trim the stack to account for windowing skips - output_particle_stack = gather(output_particle_stack(:,:,1:n_particles_added_to_stack)); + output_particle_stack = gather(output_particle_stack(:,:,1:iGpuDataCounter - 1)); tmp_stack_filename = sprintf('%s/%s_%d.mrc',mbOUT{1:2},iCell); SAVE_IMG(output_particle_stack, tmp_stack_filename, pixelSize); - - newstack_file_handle = fopen(newstack_file,'a'); fprintf(newstack_file_handle, '%s\n',tmp_stack_filename); - fprintf(newstack_file_handle, '0-%d\n',n_particles_added_to_stack-1); - fclose(newstack_file_handle); + fprintf(newstack_file_handle, '0-%d\n',iGpuDataCounter-2); % output_cell{iCell}= gather(output_particle_stack); iCell = iCell + 1; end % end of the loop over tilt series - +fclose(newstack_file_handle); fclose(starFile); newstack_file_with_n_stacks = sprintf('%s/%s.newstack_full',mbOUT{1:2}); @@ -826,7 +828,7 @@ fclose(fh); system(sprintf('cat %s >> %s',newstack_file,newstack_file_with_n_stacks)); -system(sprintf('newstack -FileOfInputs %s %s.mrc',newstack_file_with_n_stacks,output_prefix)); +system(sprintf('newstack -FileOfInputs %s %s.mrc > /dev/null',newstack_file_with_n_stacks,output_prefix)); % SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',output_prefix),pixelSize); From 6408c1575a7cd6e7dc55c45e599012bd490434ef Mon Sep 17 00:00:00 2001 From: himesb Date: Sun, 18 Feb 2024 23:11:59 -0500 Subject: [PATCH 030/151] Sweep transformation fudge factors --- metaData/BH_parseParameterFile.m | 4 +-- synthetic/BH_to_cisTEM_mapBack.m | 52 ++++++++++++-------------------- 2 files changed, 21 insertions(+), 35 deletions(-) diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index faf6d818..f16c5794 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -380,13 +380,13 @@ if isfield(emc, 'flgPostShift') EMC_assert_numeric(emc.flgPostShift, 2); else - emc.flgPostShift = [-0.5,-0.5]; + emc.flgPostShift = [-1.5,1.0]; end if isfield(emc, 'prjVectorShift') EMC_assert_numeric(emc.prjVectorShift, 3); else - emc.prjVectorShift = [0.5,0.5,1.0]'; + emc.prjVectorShift = [0.5,0.5,0.5]; end if isfield(emc,'pixelShift') diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 060c92e7..07e43e4e 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -13,15 +13,7 @@ % Some flags that are worth keeping as options, but not accessible % directlyCT % by the users (private methods-ish) -global bh_global_imodProjectionShifts; -if isempty(bh_global_imodProjectionShifts) - % bh_global_imodProjectionShifts = [ -0.5, -0.5, 0.5 ; -0.5, -0.5, 0; 0.5,0.5,1.0 ]; - % From a more thorough sweep - bh_global_imodProjectionShifts = [ 0.5, -0.5, 0.5 ; 0.0, -0.5, 0; 0.5,0.5,1.0 ]; - -end -preShift = bh_global_imodProjectionShifts(1,:); -postShift = bh_global_imodProjectionShifts(2,1:2); + emc = BH_parseParameterFile(PARAMETER_FILE); MAX_EXPOSURE = EMC_str2double(MAX_EXPOSURE) @@ -193,13 +185,14 @@ tiltStart=1; firstTilt = true; -pixelShift = -1; + iCell = 0; output_cell = {}; newstack_file = sprintf('%s/temp_particle_stack.newstack',mbOUT{1}); newstack_file_handle = fopen(newstack_file,'w'); + for iTiltSeries = tiltStart:nTiltSeries n_particles_added_to_stack = 0; if (skip_to_the_end_and_run) @@ -381,7 +374,7 @@ if (useFixedNotAliStack) - % 20190509 - I think this is royally screwing things up FIXME + % 20190509 - I think this is ry_startally screwing things up FIXME % Commenting this out invalidates the defocus vals % positionn in stack, imod rotation matrix (2x2), x,y shift (unbinned) xfTLT = sortrows(TLT(:,[1,7:10,2,3],1)); @@ -408,7 +401,7 @@ end else % Create an identity transform for the model - % 20190509 - I think this is royally screwing things up FIXME + % 20190509 - I think this is ry_startally screwing things up FIXME % Commenting this out invalidates the defocus vals xfTLT = zeros(size(TLT,1),6); xfTLT(:,[1,4]) = 1.0; @@ -461,13 +454,13 @@ subtomo_origin_in_tomo_frame = (positionList(iSubTomo,11:13)); subtomo_origin_wrt_tilt_origin = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_tilt_origin; - % This extra shift came from experiments with real data but is both annoying and not understood. + % This extra shift came from experiments with real data but is both anny_starting and not understood. subtomo_origin_wrt_tilt_origin = subtomo_origin_wrt_tilt_origin - emc.flgPreShift; % subTomo origin relative to reconLowerLeft subtomo_origin_in_sample = originRec + subtomo_origin_wrt_tilt_origin; % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_tilt_origin' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2]), fidIDX); + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_tilt_origin' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2])', fidIDX); nPrjsIncluded = 0; for iPrj = 1:nPrjs @@ -621,7 +614,8 @@ % x % y % projection idx, from 0 - fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(emc.prjVectorShift(1:2)', size(fidList,1),1); + + fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(emc.prjVectorShift(1:2), size(fidList,1),1); foundNans = sum(isnan(fidList(:,3))); if (foundNans) fprintf('\n\t\tThere are %d NaNs in the projected fiducial list %3.3f\n\n',foundNans, foundNans/size(fidList,1)*100); @@ -634,9 +628,8 @@ particlePad = 2.0; tileRadius = floor(particlePad.*particle_radius); - tileSize = (2.*tileRadius).*[1,1]; - - tileOrigin = floor(tileSize./2) + 1; + tileSize = BH_multi_iterator((2.*tileRadius).*[1,1],'fourier2d'); + tileOrigin = emc_get_origin_index(tileSize); nFidsTotal = numel(unique(fidList(parList(:,1)~=-9999,2))); @@ -717,22 +710,18 @@ continue; end - pixelX = wrkFid(iFid,3) - pixelShift + postShift(1); - pixelY = wrkFid(iFid,4) - pixelShift + postShift(2); + pixelX = wrkFid(iFid,3) - emc.pixelShift + emc.flgPostShift(1); + pixelY = wrkFid(iFid,4) - emc.pixelShift + emc.flgPostShift(2); - ox = floor(pixelX) - tileRadius; - oy = floor(pixelY) - tileRadius; + x_start = floor(pixelX) - tileOrigin(1) + 1; + y_start = floor(pixelY) - tileOrigin(2) + 1; sx = pixelX - floor(pixelX); sy = pixelY - floor(pixelY); particle_was_skipped = false; - if ( ox > 0 && oy > 0 && ox + 2*tileRadius < sTX && oy +2*tileRadius < sTY ) - output_particle_stack(:,:,iGpuDataCounter) = STACK(ox:ox+2.*tileRadius-1,oy:oy+2.*tileRadius-1,TLT(iPrj,1)); - % else - % particle_was_skipped = true; - % output_particle_stack(:,:,iGpuDataCounter) = randn(tileSize,'single').*0.1; % Why am I not just skipping these? - % end + if ( x_start > 0 && y_start > 0 && x_start + tileSize(1) - 1 < sTX && y_start + tileSize(2) - 1 < sTY ) + output_particle_stack(:,:,iGpuDataCounter) = STACK(x_start:x_start+tileSize(1)-1,y_start:y_start+tileSize(2)-1,TLT(iPrj,1)); if (useFixedNotAliStack) rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); @@ -740,7 +729,6 @@ rotFull = rTilt*[RF(1), RF(2), 0; RF(3), RF(4), 0; 0, 0, 1]*reshape(wrkPar(iFid,7:15),3,3); else - % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','forwardVector'); @@ -756,8 +744,8 @@ phaseShift = 0.0; occupancy = 100.0; % TODO test replacement with CCC score? - logp = -1000; - sigma = 10.0; + logp = -1000; % Tru -40000 + sigma = 10.0; % try 20 score = 10.0; % TODO test with scaled CCC score? scoreChange = 0.0; pixelSize = emc.pixel_size_angstroms; @@ -886,8 +874,6 @@ pause(3) system(sprintf('./%s_rec.sh',output_prefix)); -% FIXME: add a parameter to control this as we will probably want to consider just moving into cisTEM from the start -return; %%%%%%%%%%%%%%%%%%%%%%%%% % Refine From ac840c35797c0b27d19afddc9dd3a4aa25cbebff Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 19 Feb 2024 08:56:35 -0500 Subject: [PATCH 031/151] Change defaults for fudge factors, add emc.print_alignment_stats to default false to make logs more manageable (angles from alignRaw. Update tomoCPR to handle transpose of emc.projectionVector --- alignment/BH_alignRaw3d_v2.m | 21 +++++++++------------ metaData/BH_parseParameterFile.m | 5 +++++ synthetic/BH_synthetic_mapBack.m | 6 +++--- testScripts/mCompile.sh | 2 +- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index e729458a..0f55ef32 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -1430,7 +1430,7 @@ deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); - if (deltaCCC < 0) && (abs(deltaCCC) > 0.15*cccInitial(1,6)) + if (emc.print_alignment_stats && deltaCCC < 0 && abs(deltaCCC) > 0.15*cccInitial(1,6)) fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... @@ -1464,7 +1464,7 @@ cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); - if (flgRefine) + if (emc.print_alignment_stats && flgRefine) cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... @@ -1474,12 +1474,13 @@ cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - else - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - + else + if (emc.print_alignment_stats) + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + end end @@ -1528,10 +1529,6 @@ [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); cccSortedandUnique = sortCCC(a,:); - % % % save('cccSortedandUnique.mat','cccSortedandUnique'); - % % % g = gather(geometry); - % % % save('TBL_geom.mat','g'); - bestAngles_tmp.(tomoList{iTomo}) = gather(cccSortedandUnique); % save doesn't work in a parfor, so write out the results for each tomogram so that a diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index f16c5794..dd2cda5d 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -528,5 +528,10 @@ emc = EMC_assert_deprecated_substitution(emc, false, 'tomo_cpr_defocus_refine', 'calcCTF'); EMC_assert_boolean(emc.tomo_cpr_defocus_refine); +if isfield(emc, 'print_alignment_stats') + EMC_assert_boolean(emc.print_alignment_stats); +else + emc.print_alignment_stats = false; +end end diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index e1b49560..c2e6b257 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -719,11 +719,11 @@ % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_tilt_origin' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2]), fidIDX); + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_tilt_origin' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2])', fidIDX); % Save a non-rotated model with each class on its own object for visualization if (emc.save_mapback_classes) - fprintf(coordCLASS,'%d 1 %0.4f %0.4f %0.4f\n', iClassIDX, subtomo_origin_wrt_tilt_origin' + originRec'- emc.prjVectorShift); + fprintf(coordCLASS,'%d 1 %0.4f %0.4f %0.4f\n', iClassIDX, subtomo_origin_wrt_tilt_origin' + originRec'- emc.prjVectorShift'); end for iPrj = 1:nPrjs @@ -1086,7 +1086,7 @@ % Need to shift again from the model coordinate system - fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(emc.prjVectorShift(1:2)', size(fidList,1),1); + fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(emc.prjVectorShift(1:2), size(fidList,1),1); foundNans = sum(isnan(fidList(:,3))); if (foundNans) fprintf('\n\t\tThere are %d NaNs in the projected fiducial list %3.3f\n\n',foundNans, foundNans/size(fidList,1)*100); diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 5cc0935e..65d59928 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=0 -nightly=3 +nightly=4 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 44a293f6ee82fdd1df26905a277f44bfb5797282 Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 19 Feb 2024 09:18:11 -0500 Subject: [PATCH 032/151] Fix a few neg -> pos defocus oversights in tomoCPR. (Really nead to de-duplicate some of the code between this and to_cistem --- synthetic/BH_synthetic_mapBack.m | 29 ++++++++++++----------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index c2e6b257..e0f4fb66 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -534,7 +534,7 @@ % Imod expects nanometers and underfocus positive (origin on specimen) % whereas I let the origin be the focal plane such that underfocus is % negative. - fprintf(iDefocusFile,'%f\n',defTLT(:,2)'.*(-1*10^9)); + fprintf(iDefocusFile,'%f\n',abs(defTLT(:,2)') .* 10^9); fclose(iDefocusFile); % We also need the transform from the microscope frame in order to @@ -624,7 +624,7 @@ % We need to rotate the model 90 degrees around X to match the "natural" reconstruction reference frame of imod % that is [x,z,-y] - modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector') + modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector'); for iSubTomo = 1:nSubTomos @@ -740,7 +740,7 @@ % d1 = -1.*((samplingRate.*prjCoords(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; % d2 = -1.*((samplingRate.*prjCoords(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; - d1 = -1.*(samplingRate.*subtomo_origin_wrt_tilt_origin(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15))*10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm + d1 = (abs(TLT(iPrj_nat,15)) - samplingRate.*subtomo_origin_wrt_tilt_origin(3).*unsampled_pixel_size.*10^-10) * 10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n', ... @@ -1039,10 +1039,14 @@ '%s\n',... '%s\n',... '%s\n',... - 'EOF'],tilt_binned_filename, mbOUT{1:3}, maxZ, ... - mbOUT{1:3},... - mbOUT{1:3},... - pixel_size./10, ...flgInvertTiltAngles,... % Ang --> nm + 'EOF'], ... + tilt_binned_filename, ... % input + mbOUT{1:3}, ... % output fiducial model + maxZ, ... % thickness + mbOUT{1:3},... % tilt angle file + mbOUT{1:3},... % defocus file + pixel_size./10, ... + 0, ... % flgInvertTiltAngles,... % Ang --> nm mbOUT{1:3},... mbOUT{1:3},... mbOUT{1:3},... @@ -1063,28 +1067,19 @@ end % re-write the projected coords - system(sprintf(['model2point -float -contour -zero ',... - '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'],... - mbOUT{1:3}, mbOUT{1:3})) - - - - + system(sprintf(['model2point -float -contour -zero ', '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'], mbOUT{1:3}, mbOUT{1:3})) end - for iSave = 1 % Remove the full size tomo system(sprintf('rm %smapBack%d/%s.tmpRot%d',mbOUT{1:3},iSave)); end - fidList = load(sprintf('%smapBack%d/%s.coordPrj',mbOUT{1:3})); parList = load(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3})); defList = load(sprintf('%smapBack%d/%s.defAngTilt',mbOUT{1:3})); - % Need to shift again from the model coordinate system fidList(:,[2,3]) = fidList(:,[2,3]) + repmat(emc.prjVectorShift(1:2), size(fidList,1),1); foundNans = sum(isnan(fidList(:,3))); From 8f77c251ebc20e3db076bf0b06bc69c86fe868a2 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 21 Feb 2024 06:46:29 -0500 Subject: [PATCH 033/151] wip --- alignment/BH_alignRaw3d_v2.m | 39 ++--- ctf/BH_ctf_Correct3d.m | 86 +++-------- masking/BH_multi_projectionMask.m | 4 +- metaData/BH_parseParameterFile.m | 46 ++++++ metaData/EMC_setup_tmp_cache.m | 86 +++++++++++ statistics/BH_fscGold_class.m | 7 +- synthetic/BH_synthetic_mapBack.m | 124 +++++----------- synthetic/BH_to_cisTEM_mapBack.m | 49 +------ testScripts/emClarity.m | 142 +++++-------------- transformations/BH_average3d.m | 11 +- transformations/BH_multi_combineLowResInfo.m | 8 +- transformations/BH_multi_loadOrBin.m | 2 +- 12 files changed, 243 insertions(+), 361 deletions(-) create mode 100644 metaData/EMC_setup_tmp_cache.m diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 0f55ef32..aceee036 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -13,15 +13,6 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -global bh_global_print_shifts_in_particle_basis; -if isempty(bh_global_print_shifts_in_particle_basis) - bh_global_print_shifts_in_particle_basis = true; -end - -global bh_global_zero_lag_score; -if isempty(bh_global_zero_lag_score) - bh_global_zero_lag_score = false; -end if (nargin ~= 2 && nargin ~= 3) error('args = PARAMETER_FILE, CYCLE, [1,abs(ccc),2,weighted,3,abs(weighted)]') @@ -148,9 +139,6 @@ % % % extList = subTomoMeta.mapExt; masterTM = subTomoMeta; clear subTomoMeta - - - refVector = cell(2,1); refGroup = cell(2,1); refSym = cell(2,1); @@ -159,26 +147,23 @@ % Sort low to high, because order is rearranged as such unstack refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) + refVector{iGold} = refVectorFull{iGold}(1,:); % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) + refGroup{iGold} = refVectorFull{iGold}(3,:); % axial symmetry to apply, negative value indicates creating a mirrored ref % accros the corresponding axis - refSym{iGold} = refVectorFull{iGold}(2,:) + refSym{iGold} = refVectorFull{iGold}(2,:); end % make sure the number of references match the unique groups in the classVector % and also that the class/group pairs match the class/ref pairs. nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; -nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]'' +nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]; nRefOut(1:2) = [length(unique(refGroup{1})) + sum(( refSym{1} < 0 )),... length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; - -%%%%%%%%%%%%%%%%%%%%%%% - % Get the number of tomograms to process. tomoList = fieldnames(geometry); nTomograms = length(tomoList); @@ -195,10 +180,10 @@ [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms) + BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms); [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); @@ -698,7 +683,6 @@ % Load the tomo into gpu tomoName = tomoList{iTomo}; - %fprintf('gpu %d working on tomoName %s\n', iGPU, tomoName); tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); % Load in the geometry for the tomogram, and get number of subTomos. @@ -1414,7 +1398,7 @@ % It is probably more useful see the shifts in the particle % reference frame vs. the avg which was the original - if (bh_global_print_shifts_in_particle_basis) + if (emc.printShiftsInParticleBasis) printShifts = zeros(3,3); printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); @@ -1490,8 +1474,7 @@ if ~(rem(iSubTomo,100)) timeClass = toc; - fprintf('\nworking on %d/%d subTomo from %s...%fs\n',... - iSubTomo,nSubTomos,tomoName,timeClass); + fprintf('Refining %d/%d subTomo from %s...%fs\n', iSubTomo, nSubTomos, tomoName, timeClass); tic; end @@ -1556,7 +1539,7 @@ if ( flgReverseOrder || flgStartThird ) - fprintf('This reverse run will not write the metaData\n'); + fprintf('This multi-node run will not write the metaData\n'); else save('bestAnglesResults.mat', 'bestAnglesResults'); @@ -1578,9 +1561,7 @@ clear bestAngles rawAlign subTomoMeta = masterTM; save(emc.('subTomoMeta'), 'subTomoMeta'); - - - + end delete(gcp('nocreate')) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 63747591..2eb2e6e7 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -70,6 +70,7 @@ filterProjectionsForTomoCPRBackground=0; loadSubTomoMeta = true; flgWhitenPS = [0,0,0.0]; +use_existing_tmpCache=''; if nargin > 2 if isempty(EMC_str2double(varargin{1})) error('Extra argument to ctf 3d should be a vector [THICKNESS, BINNING] tiltN, or a string templateSearch'); @@ -78,8 +79,11 @@ recWithoutMat = true; if length(varargin) > 2 % Full recon for tomoCPR - bh_global_turn_on_phase_plate = varargin{3} + bh_global_turn_on_phase_plate = varargin{3}; filterProjectionsForTomoCPRBackground = varargin{4}; + if length(varargin) > 4 + use_existing_tmpCache = varargin{5}; + end else loadSubTomoMeta = false; % Default to on for subregion picking @@ -118,7 +122,7 @@ end if (bh_global_turn_on_phase_plate(1) && any(emc.whitenPS)) - fprintf('WARNING: phakePhasePlate and whitening are conflicting preocesses. Turning off whitening.\n') + fprintf('WARNING: phakePhasePlate and whitening are conflicting preocesses. Turning off whitening.\n'); emc.whitenPS = [0,0,0]; end @@ -134,7 +138,7 @@ try useSurfaceFit = emc.('useSurfaceFit') catch - useSurfaceFit = 1 + useSurfaceFit = 1; end try @@ -152,55 +156,11 @@ - -tmpCache= emc.('fastScratchDisk'); - -if strcmpi(tmpCache, 'ram') - if isempty(getenv('EMC_CACHE_MEM')) - fprintf('Did not find a variable for EMC_CACHE_MEM\nSkipping ram\n'); - tmpCache= ''; - else - % I have no ideah how much is needed - if EMC_str2double(getenv('EMC_CACHE_MEM')) < 64 - fprintf('There is only 64 Gb of cache on ramdisk, not using'); - tmpCache = ''; - else - tmpCache=getenv('MCR_CACHE_ROOT'); - fprintf('Using the tmp EMC cache in ram at %s\n',tmpCache); - end - end -end - -% Check to make sure it even exists -if isempty(dir(tmpCache)) - fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); - tmpCache = ''; -end - reconScaling = 1; -if isempty(tmpCache) - tmpCache='cache'; - flgCleanCache = 0; - CWD=''; -else - flgCleanCache = 1; - CWD = sprintf('%s/',pwd); - % Check for a trailing slash - slashCheck = strsplit(tmpCache,'/'); - if isempty(slashCheck{end}) - % This means the final character was a slash, strip it - tmpCache = sprintf('%scache',tmpCache); %strjoin(slashCheck(1:end-1),'/'); - else - tmpCache = sprintf('%s/cache',tmpCache); - end -end +[tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(emc.fastScratchDisk, use_existing_tmpCache, 'ctf3d', false); + -% Incase this is launched form another process (synthetic mapback for example, make one level lower in the cache -tmpCache=sprintf('%s/ctf3d',tmpCache); -fprintf('tmpCache is %s\n',tmpCache); -system(sprintf('mkdir -p %s',tmpCache)); -system(sprintf('mkdir -p %s','cache')); % This should exist, but to be safe. if (recWithoutMat) useSurfaceFit = false; if (loadSubTomoMeta) @@ -234,9 +194,9 @@ try - flgDampenAliasedFrequencies = emc.('flgDampenAliasedFrequencies') + flgDampenAliasedFrequencies = emc.('flgDampenAliasedFrequencies'); catch - flgDampenAliasedFrequencies = 0 + flgDampenAliasedFrequencies = 0; end try @@ -382,6 +342,7 @@ EMC_parpool(nGPUs) end +% FIXME parfor iGPU = 1:nGPUs % for iGPU = 1:nGPUs @@ -394,8 +355,8 @@ % not present. TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - + fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); + % Get all the tomogram names that belong to a given tilt-series. if (~recWithoutMat) @@ -441,14 +402,12 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. -% FIXME parfor iGPU = 1:nGPUs % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); % Loop over each tilt for iTilt = iterList{gpuList(iGPU)} - - + if (recWithoutMat) if (loadSubTomoMeta) @@ -476,9 +435,9 @@ TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); TLT = load(TLTNAME); - fprintf('using TLT %s\n', TLTNAME); - - + fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); + + if (~recWithoutMat) % Get all the tomogram names that belong to a given tilt-series. @@ -590,7 +549,7 @@ [ sectionList ] = calcTomoSections(iCoords, tomoNumber, emc.pixel_size_angstroms, nSections, tiltList{iTilt}, ctf3dDepth); - + if (recWithoutMat) avgZ = 0; surfaceFit = 0; @@ -800,13 +759,13 @@ reconNameFull = sprintf('cache/%s_%d_bin%d_filtered.rec', ... tiltList{iTilt},thisTomo,samplingRate); elseif reconstructionParameters(1) - - reconNameFull = sprintf('cache/%s_%d_bin%d_backgroundEst.rec', ... - tiltList{iTilt},thisTomo,samplingRate); + reconNameFull = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec', ... + CWD,tiltList{iTilt},thisTomo,samplingRate); else reconNameFull = sprintf('cache/%s_%d_bin%d.rec', ... tiltList{iTilt},thisTomo,samplingRate); end + fprintf('in ctf3d reconNameFull is %s\n\n',reconNameFull); recCMD = 'newstack -fromone'; for iSection = 1:nSections @@ -1035,7 +994,6 @@ end -% The avg Z seems like it should be added? if ( flgDampenAliasedFrequencies ) % Experiment with dampning higher frequencies where aliasing is going to diff --git a/masking/BH_multi_projectionMask.m b/masking/BH_multi_projectionMask.m index c48557df..7503e09e 100755 --- a/masking/BH_multi_projectionMask.m +++ b/masking/BH_multi_projectionMask.m @@ -33,11 +33,9 @@ evalMask = false([d1,d2,d3]); Z1 = zeros([r1,r2],'single'); else - error('METHOD must be GPU or cpu\n.'); + error('METHOD must be GPU or cpu\n'); end - - for iPrj = 1:d3 R = BH_defineMatrix([TLT(iPrj,6),TLT(iPrj,4),TLT(iPrj,6)], ... diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index dd2cda5d..c90b2fc5 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -47,6 +47,25 @@ % Asserts on required parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +if isfield(emc, 'fastScratchDisk') + if strcmpi(emc.fastScratchDisk, 'ram') + if isempty(getenv('EMC_CACHE_MEM')) + fprintf('Did not find a variable for EMC_CACHE_MEM\nSkipping ram\n'); + emc.fastScratchDisk = ''; + else + % I have no ideah how much is needed + if EMC_str2double(getenv('EMC_CACHE_MEM')) < 32 + fprintf('There is only 64 Gb of cache on ramdisk, not using\n'); + emc.fastScratchDisk = ''; + else + emc.fastScratchDisk=getenv('MCR_CACHE_ROOT'); + end + end % if EMC_CACHE_MEM + end % if ram +else + emc.fastScratchDisk = ''; +end + if isfield(emc, 'nGPUs') EMC_assert_numeric(emc.nGPUs, 1, [1, 1000]); else @@ -533,5 +552,32 @@ else emc.print_alignment_stats = false; end + +if isfield(emc, 'printShiftsInParticleBasis') + EMC_assert_boolean(emc.printShiftsInParticleBasis); +else + emc.printShiftsInParticleBasis = true; +end + +if isfield(emc, 'ML_compressByFactor') + EMC_assert_numeric(emc.ML_compressByFactor, 1); +else + emc.ML_compressByFactor = 2.0; +end + +if isfield(emc, 'ML_angleTolerance') + EMC_assert_numeric(emc.ML_angleTolerance, 1); +else + emc.ML_angleTolerance = 2.0; +end + +if isfield(emc, 'mtf_value') + EMC_assert_numeric(emc.mtf_value, 1); +else + emc.mtf_value = 2.0; +end + + + end diff --git a/metaData/EMC_setup_tmp_cache.m b/metaData/EMC_setup_tmp_cache.m new file mode 100644 index 00000000..077f4b09 --- /dev/null +++ b/metaData/EMC_setup_tmp_cache.m @@ -0,0 +1,86 @@ +function [tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(emc_fastScratchDisk, existing_tmpCache, caller_name, add_trailing_slash) + + % After parseParameter file + % emc.fastScratchDisk= '' || getenv('MCR_CACHE_ROOT'); + + tmpCache = emc_fastScratchDisk; + flgCleanCache = false; + if isempty(existing_tmpCache) + % Default condition we need to setup the cache which is now coming + % from the parser and can be empty or a path to the local MCR on ramdisk + if isempty(tmpCache) + % Local project directory, we don't wan to clean the cache as this will conflict + % with other processes + tmpCache='cache'; + end + else + % We are using an existing cache directory, we need to make sure it exists + if isdir(existing_tmpCache) + tmpCache = existing_tmpCache; + else + error('The existing tmpCache %s does not exist',existing_tmpCache); + end + end + + [ tmpCache ] = strip_trailing_slash(tmpCache); + [filepath,name,ext] = fileparts(tmpCache); + % Check if the path is local or remote, this logic only works if the trailing slash is gone, + % eg. /to/path, '', '' = fileparts(/to/path/) + if (isempty(name) && isempty(filepath)) + error('The tmpCache (%s) should not be empty at this point in the code.', tmpCache) + end + + if isempty(name) + % We must be in the local project directory, add this little check + if ~isdir('fixedStacks') + % Should work for soft links too + error('The fixedStacks directory does not exist in the current directory, %s',pwd); + end + tmpCache = 'cache'; + CWD = ''; + else + if ~strcmp(name,'cache') + tmpCache = fullfile(filepath,name, 'cache'); + end + CWD = sprintf('%s/',pwd); + end + + system(sprintf('mkdir -p %s', tmpCache)); + + % Check for legal names + switch caller_name + case 'ctf3d' + tmpCache = fullfile(tmpCache, 'ctf3d'); + system(sprintf('mkdir -p %s', tmpCache)); + if ~isdir(tmpCache) + error('The tmpCache %s does not exist',tmpCache); + end + case 'tomoCPR' + % Nothing to do here + + case 'cisTEM' + tmpCache = fullfile(tmpCache, 'to_cisTEM'); + system(sprintf('mkdir -p %s', tmpCache)); + otherwise + error('This function is not allowed to be called by %s',caller_name); + end + + % Finally check if the program asks for a trailing slash. + if (add_trailing_slash) + tmpCache = sprintf('%s/',tmpCache); + end + fprintf('tmpCache is %s\n',tmpCache); + +end + +function [ cleaned ] = strip_trailing_slash(input) + + % Check for a trailing slash + slashCheck = strsplit(input,'/'); + if isempty(slashCheck{end}) + cleaned = strjoin(slashCheck(1:end-1),'/'); + else + cleaned = input; + end + +end \ No newline at end of file diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index c370df9e..dda0c541 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -79,11 +79,6 @@ peakSearch = floor(emc.('particleRadius')./emc.pixel_size_angstroms); peakCOM =3; -global bh_global_MTF -if isempty(bh_global_MTF) - bh_global_MTF = 2; -end - % The default is fsc-Gold Standard so the two images should need some degree of % alignment prior to calculating the fsc. flgAlignImages = 1; @@ -770,7 +765,7 @@ % to work? if ~(flgJustFSC) masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRef)) = ... - {shellsFreq,shellsFSC,{cRef,cRefAli,bh_global_MTF},osX,forceMaskAlign,forceMask,nCones,coneList,halfAngle,samplingRate}; + {shellsFreq,shellsFSC,{cRef,cRefAli,emc.mtf_value},osX,forceMaskAlign,forceMask,nCones,coneList,halfAngle,samplingRate}; sprintf('Resample_%s%d',savePrefix,iRef) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index e0f4fb66..f0636e3e 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -94,11 +94,9 @@ % diff. flgInvertTiltAngles = 0; -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -%%% Playing around with the model -n_surfaces=2; - - +% We expect our particles to be distributed throught the thickness. +% Setting to 2 can cause the alignment to silently die +n_surfaces=1; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% try @@ -121,23 +119,8 @@ tmpCache= emc.('fastScratchDisk'); end -if strcmpi(tmpCache, 'ram') - if isempty(getenv('EMC_CACHE_MEM')) - fprintf('Did not find a variable for EMC_CACHE_MEM\nSkipping ram\n'); - tmpCache= ''; - else - % I have no ideah how much is needed - if EMC_str2double(getenv('EMC_CACHE_MEM')) < 64 - fprintf('There is only 64 Gb of cache on ramdisk, not using'); - tmpCache = ''; - else - tmpCache=getenv('MCR_CACHE_ROOT'); - fprintf('Using the tmp EMC cache in ram at %s\n',tmpCache); - end - end -end +[tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(tmpCache, '', 'tomoCPR', true); -% % % nWorkers = EMC_str2double(nWORKERS) nGPUs = emc.('nGPUs'); pInfo = parcluster(); gpuScale=3*samplingRate; @@ -145,30 +128,6 @@ fprintf('Using %d workers as max of %d %d*nGPUs and %d nWorkers visible\n', ... nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); -% Check to make sure it even exists -if isempty(dir(tmpCache)) - fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); - tmpCache = ''; -end -if isempty(tmpCache) - tmpCache='cache/'; - flgCleanCache = 0; - CWD = ''; -else - flgCleanCache = 1; - CWD = sprintf('%s/',pwd); - % Check for a trailing slash - slashCheck = strsplit(tmpCache,'/'); - if isempty(slashCheck{end}) - tmpCache = sprintf('%scache/',tmpCache); % prefix for mapBack - else - tmpCache = sprintf('%s/cache/',tmpCache); % prefix for mapBack - end - -end - -system(sprintf('mkdir -p %s',tmpCache)); - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; @@ -268,7 +227,7 @@ is_first_run = true; mbOUT = {[tmpCache],[mapBackIter+1],'dummy'}; -tiltStart=1; + for iTiltSeries = tiltStart:nTiltSeries if (skip_to_the_end_and_run) continue; @@ -313,7 +272,7 @@ end if exist(localFile,'file') - fprintf('Found local file %s\n.', localFile); + fprintf('Found local file %s\n', localFile); else fprintf('No local transforms requested.\n'); localFile = 0; @@ -328,7 +287,7 @@ try eraseMaskType = emc.('Peak_mType'); eraseMaskRadius = emc.('Peak_mRadius') ./ pixel_size; - fprintf('Further restricting peak search to radius of [%f %f %f] pixels\n', eraseMaskRadius); + % fprintf('Further restricting peak search to radius of [%f %f %f] pixels\n', eraseMaskRadius); eraseMask = 1; catch eraseMask = 0; @@ -345,7 +304,7 @@ % FIXME: this should be in parseParameterFile try lowPassCutoff = emc.('tomoCprLowPass'); - fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n.', lowPassCutoff); + fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n', lowPassCutoff); catch % TODO are these range limits okay? lowPassCutoff = 1.5.*mean(subTomoMeta.currentResForDefocusError); @@ -354,13 +313,13 @@ elseif (lowPassCutoff > 24) lowPassCutoff = 24; end - fprintf('Using an internatlly determined lowpass cutoff of %3.3f Ang\n.',... + fprintf('Using an internatlly determined lowpass cutoff of %3.3f Ang\n',... lowPassCutoff); end % FIXME: this can also be in parseParameterFile if lowPassCutoff < 2* pixel_size - fprintf('Psych, the cutoff is being set to Nyquist'); + fprintf('Psych, the cutoff is being set to Nyquist\n'); lowPassCutoff = 2*pixel_size; end @@ -373,13 +332,12 @@ end if sqrt(2)*pixel_size > min_res_for_ctf_fitting - fprintf('Warning the current resolution is too low to refine the defocus. Turning off this feature'); + fprintf('Warning the current resolution is too low to refine the defocus. Turning off this feature\n'); calcCTF = false; end end - % TODO: these defaults should be re-examined - nFiducialsPerPatch = ceil(100./sqrt(molMass)); +% TODO: these defaults should be re-examined targetPatchSize = max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch))); % The binned stacks should already exist, if not, this will re-create it in the cache dir. @@ -461,22 +419,23 @@ particleMask{iRef} = gather(particleMask{iRef}); end - sprintf('[%d,%d]',maxZ,samplingRate) - tiltNameList{iTiltSeries} + sprintf('[%d,%d]',maxZ,samplingRate); + tiltNameList{iTiltSeries}; backgroundName = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec',CWD,tiltNameList{iTiltSeries},1, samplingRate); + fprintf('In tomocpr, using background estimate %s\n\n',backgroundName); send_backgroundLowPassResolution = 28; % TODO: investigate deviations from the default, which is to shut off the phakePhasePlate and to use a backgroundLowPassResolution of 28 % Default false, we don't apply this filter % if enabled, it currently only saves the filtered background estimate for visualization in addition to the normal version % if (emc.save_mapback_classes) - % BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3); + % BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3, tmpCache); % end % FIXME: calling like this does not use the surface fit for the background send_phakePhasePlateOption = [0,0]; - BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution); + BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution, tmpCache); % re-initialize the parpool for each tilt series to free up mem. delete(gcp('nocreate')) @@ -486,7 +445,7 @@ avgTomo{1} = getVolume(MRCImage(backgroundName)); if (delete_background_estimate) - system(sprintf('rm %s',backgroundName)); + system(sprintf('rm -f %s',backgroundName)); end for iRef = 1:nRefs @@ -731,8 +690,8 @@ iPrj_nat = find(TLT(:,1) == iPrj); % imod is indexing from zero zCoord = iPrj_nat; - - rTilt = BH_defineMatrix([90,1.*TLT(iPrj_nat,4),-90],'Bah','forwardVector'); + % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus)% For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) + rTilt = BH_defineMatrix([0,TLT(iPrj_nat,4),0],'SPIDER','inv'); prjCoords = rTilt*subtomo_origin_wrt_tilt_origin'; @@ -1197,7 +1156,9 @@ emc.k_factor_scaling = 10 / sqrt(nFidsTotal); end - parfor iPrj = 1:nPrjs + % FIXME revert + for iPrj = 1:nPrjs + % parfor iPrj = 1:nPrjs % % For some reason if these mrc objects are created before the parfor % loop begins, they fail to load. It is fine as a regular for loop @@ -1223,7 +1184,7 @@ end end if toc == 300 - error('failed to load dataPrj %s at %d after %d tries\n.', ... + error('failed to load dataPrj %s at %d after %d tries\n', ... tiltSeries,iPrj,3000); else % fprintf('loaded dataPrj %d on try %d\n',iPrj,floor(toc./0.1)); @@ -1241,7 +1202,7 @@ end end if toc == 300 - error('failed to load refPrj %s at %d after %d tries\n.', ... + error('failed to load refPrj %s at %d after %d tries\n', ... tiltSeries,iPrj,3000); else % fprintf('loaded refPrj %d on try %d\n',iPrj,floor(toc./0.1)); @@ -1259,7 +1220,7 @@ end end if toc == 300 - error('failed to load samplingMask %s at %d after %d tries\n.', ... + error('failed to load samplingMask %s at %d after %d tries\n', ... tiltSeries,iPrj,3000); else % fprintf('loaded refPrj %d on try %d\n',iPrj,floor(toc./0.1)); @@ -1470,14 +1431,14 @@ refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixel_size])); end - bestScore = -1000000; + bestScore = -inf; bestCTF = 1; for deltaCTF = 1:nDefTotal iRefCTF = refFT .* ... mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... - single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... - single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18))); + single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... + single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18))); % try iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')); @@ -1485,9 +1446,6 @@ cccMap = dataFT .* iRefCTF; if (use_PCF) cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); - else - % % % % % cccMap = peakMask.*real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(dataTile,1,0,[0,300,lowPassCutoff,pixel_size]).*conj(bhF.fwdFFT(refTile,1,0) .* iCTF),'fwd'))); - end cccMap = peakMask.*real(bhF.invFFT(cccMap)); @@ -1573,13 +1531,7 @@ dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estPeak - [sx,sy]; end - - fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), dXY, wrkFid(iFid,5)); - - - - end % end of loop over fiducials if (calcCTF) @@ -1605,17 +1557,13 @@ clear diagnosticCell evalMaskCell - % if ~(conserveDiskSpace) && bh_global_save_tomoCPR_diagnostics - % SAVE_IMG(MRCImage(gather(diagnosticStack)),sprintf('%smapBack%d/%s_diagnostic.mrc',mbOUT{1:3})); - % SAVE_IMG(MRCImage(gather(evalMaskStack)),sprintf('%smapBack%d/%s_evalMask.mrc',mbOUT{1:3})); - % end + system(sprintf('cat %smapBack%d/%s_???.coordFIT | sort -k 1 -g > %smapBack%d/%s.coordFIT',mbOUT{1:3},mbOUT{1:3})); system(sprintf('rm %smapBack%d/%s_???.coordFIT',mbOUT{1:3})); system(sprintf('cat %smapBack%d/%s_???.global | sort -k 1 -g > %smapBack%d/%s.global',mbOUT{1:3},mbOUT{1:3})); system(sprintf('rm %smapBack%d/%s_???.global ',mbOUT{1:3})); - % create model tomogram for cross correlation fidShifts = load(sprintf('%smapBack%d/%s.coordFIT',mbOUT{1:3})); fidShifts = fidShifts(:,2:end); @@ -1628,7 +1576,6 @@ fidBin = fopen(sprintf('%smapBack%d/%s.coordBin%d',mbOUT{1:3},samplingRate),'w'); fidList = fidList(:,2:end); - % shifts/List col 1/4 should match - maybe add a check to be safe size(fidShifts) size(fidList) @@ -1636,8 +1583,6 @@ fCombine = [fidShifts(:,1),fidList(:,2:3)+fidShifts(:,2:3),fidShifts(:,4)]; fprintf('\n\n%d/%d pts ignored\n\n',sum(fCombine(:,4)==-9999),size(fCombine,1)); - - fFull = fCombine; fDefFull = [fCombine,zeros(size(fCombine,1),1)]; % % % % % fDefFull(:,2:3) = fDefFull(:,2:3).*pixel_size; @@ -1649,7 +1594,6 @@ fDefFull(wrkFidIDX,5) = defList(wrkFidIDX,7) + defocusShifts{iPrj}; end - fDefFull = fDefFull(fDefFull(:,4)~=-9999,:); fFull = fFull(fFull(:,4)~=-9999,:); fCombine = fCombine(fCombine(:,4)~=-9999,:); @@ -1662,13 +1606,11 @@ fprintf(fidBin,'%d %4.4f %4.4f %d\n',fFull'); fclose(fidBin); - fFull(:,2:3) = fFull(:,2:3) .* samplingRate; - % FIXME: I am reverting to sampling rate here, which I think should be correct. - - % The model ends up seeing the pixel size as 1, so even though it loads + % % % % % fFull(:,2:3) = fFull(:,2:3).*samplingRate; + % The model ends up seeing the pixel size as 1, so even though it loads % properly on the full aligned stack, these coords need to be scaled by % the pixel size since this is the input to tiltalign. - % fFull(:,2:3) = fFull(:,2:3).*pixel_size; + fFull(:,2:3) = fFull(:,2:3).*pixel_size; fprintf(fidCombine,'%d %4.4f %4.4f %d\n',fCombine'); fclose(fidCombine); diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 07e43e4e..1f90cd99 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -80,23 +80,8 @@ tmpCache= emc.('fastScratchDisk'); end -if strcmpi(tmpCache, 'ram') - if isempty(getenv('EMC_CACHE_MEM')) - fprintf('Did not find a variable for EMC_CACHE_MEM\nSkipping ram\n'); - tmpCache= ''; - else - % I have no ideah how much is needed - if EMC_str2double(getenv('EMC_CACHE_MEM')) < 64 - fprintf('There is only 64 Gb of cache on ramdisk, not using'); - tmpCache = ''; - else - tmpCache=getenv('MCR_CACHE_ROOT'); - fprintf('Using the tmp EMC cache in ram at %s\n',tmpCache); - end - end -end +[tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(tmpCache, '', 'cisTEM', true); -% % % nWorkers = EMC_str2double(nWORKERS) nGPUs = emc.('nGPUs'); pInfo = parcluster(); gpuScale=3; @@ -104,31 +89,6 @@ fprintf('Using %d workers as max of %d %d*nGPUs and %d nWorkers visible\n', ... nWorkers,gpuScale,nGPUs*gpuScale,pInfo.NumWorkers); -% Check to make sure it even exists -if isempty(dir(tmpCache)) - fprintf('\n\nIt appears your fastScratchDisk\n\t%s\ndoes not exist!\n\n',tmpCache); - tmpCache = ''; -end -if isempty(tmpCache) - tmpCache='cache/to_cisTEM'; - flgCleanCache = 0; -CWD = ''; -else - flgCleanCache = 1; - CWD = sprintf('%s/',pwd); - % Check for a trailing slash - slashCheck = strsplit(tmpCache,'/'); - if isempty(slashCheck{end}) - tmpCache = sprintf('%scache/to_cisTEM',tmpCache); % prefix for mapBack - else - tmpCache = sprintf('%s/cache/to_cisTEM',tmpCache); % prefix for mapBack - end -end - - - -system(sprintf('mkdir -p %s',tmpCache)); - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; @@ -240,7 +200,7 @@ end if exist(localFile,'file') - fprintf('Found local file %s\n.', localFile); + fprintf('Found local file %s\n', localFile); else fprintf('No local transforms requested.\n'); localFile = 0; @@ -271,7 +231,7 @@ % FIXME: this should be in parseParameterFile try lowPassCutoff = emc.('tomoCprLowPass'); - fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n.', lowPassCutoff); + fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n', lowPassCutoff); catch % TODO are these range limits okay? lowPassCutoff = 1.5.*mean(subTomoMeta.currentResForDefocusError); @@ -280,7 +240,7 @@ elseif (lowPassCutoff > 24) lowPassCutoff = 24; end - fprintf('Using an internatlly determined lowpass cutoff of %3.3f Ang\n.',... + fprintf('Using an internatlly determined lowpass cutoff of %3.3f Ang\n',... lowPassCutoff); end @@ -871,7 +831,6 @@ fclose(recScript); system(sprintf('chmod a=wrx %s_rec.sh',output_prefix)); -pause(3) system(sprintf('./%s_rec.sh',output_prefix)); diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 97127c80..ecc1d38c 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -41,7 +41,7 @@ add_slash = '/'; end -fprintf('ctfroot is %s\n',ctfroot); +% fprintf('ctfroot is %s\n',ctfroot); emC_PATH = strsplit(pathWithDir, slashCheck{end-shift_end}); emC_PATH = sprintf('%s%semClarity',emC_PATH{1},add_slash); @@ -50,7 +50,7 @@ % This will find the m-file, which is used to grab the shell script which is what we want to define here. [BH_checkInstallPath, fname, fext] = fileparts(which('BH_checkInstall')); BH_checkInstallPath = fullfile(BH_checkInstallPath,fname); - fprintf('BH_checkInstall is %s\n',BH_checkInstallPath); + % fprintf('BH_checkInstall is %s\n',BH_checkInstallPath); %BH_checkInstallPath = sprintf('%s%s/metaData/BH_checkInstall',ctfroot,compiled_PATH); else BH_checkInstallPath=sprintf('%s/metaData/BH_checkInstall',emC_PATH); @@ -61,8 +61,8 @@ emC_autoAliPath = sprintf('%s/alignment/emC_autoAlign',fileparts(fileparts(BH_checkInstallPath))); emC_findBeadsPath = sprintf('%s/alignment/emC_findBeads',fileparts(fileparts(BH_checkInstallPath))); -fprintf('emC_autoAlign is %s\n',emC_autoAliPath); -fprintf('emC_findBeads is %s\n',emC_findBeadsPath); +% fprintf('emC_autoAlign is %s\n',emC_autoAliPath); +% fprintf('emC_findBeads is %s\n',emC_findBeadsPath); setenv('EMC_AUTOALIGN',emC_autoAliPath); @@ -229,16 +229,16 @@ length(varargin) < 2 && length(varargin)> 5 fprintf(['\nUsage: emClarity init param.m [tomoCpr iter, for continuing after second globalsearch]\n']); elseif length(varargin) == 5 - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_geometryInitialize(varargin{2},varargin{3},varargin{4},varargin{5}); elseif length(varargin) == 4 - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_geometryInitialize(varargin{2},varargin{3},varargin{4}); elseif length(varargin) == 3 - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_geometryInitialize(varargin{2},varargin{3}); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_geometryInitialize(varargin{2}); end case 'removeNeighbors' @@ -246,7 +246,7 @@ length(varargin) ~= 6 fprintf(['\nUsage: emCLarity removeNeighbors pixelSize CYCLE distanceCutoff (Ang) angleCutoff (Deg) N-neighbors\n']); else - %emC_testParse(varargin{2}) + %emC_testParse(varargin{2}); if length(varargin) == 6 BH_geometry_Constraints(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}); else @@ -259,7 +259,7 @@ (length(varargin) ~= 5 && length(varargin) ~= 6) fprintf(['\nUsage: emClarity autoAlign param.m stackName tiltFile tilt-axis Rotation\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); if ~exist(varargin{4}, 'file') fprintf('Did not find your .rawtlt file %s\n',varargin{3}); error('Expecting tiltName.st tiltName.rawtlt pixelSize (Ang) imageRotation (degrees)'); @@ -282,7 +282,7 @@ length(varargin) ~= 3 fprintf(['\nUsage: emClarity skip param.m iter\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_skipClassAlignment(varargin{2},varargin{3},'RawAlignment','1'); end case 'rescale' @@ -351,7 +351,7 @@ ' raw (post raw alignment)\n',... ' cluster_cls (post classification)\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_average3d(varargin{2}, varargin{3}, varargin{4}); end case 'fsc' @@ -364,7 +364,7 @@ ' raw (post raw alignment)\n',... ' cluster_cls (post classification)\n']); elseif length(varargin) == 4 - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_fscGold_class(varargin{2}, varargin{3}, varargin{4}); else BH_fscGold_class(varargin{2}, varargin{3}, varargin{4},varargin{5},varargin{6}); @@ -377,7 +377,7 @@ 'cycle number\n',... '[experimental option 1/2/3, 1 - abs(ccc),2 - weighted,3 -abs(weighted)]']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); if length(varargin) == 3 BH_alignRaw3d_v2(varargin{2}, varargin{3}); else @@ -393,7 +393,7 @@ 'cycle number\n',... 'stage of alignment\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_alignReferences3d(varargin{2}, varargin{3}); end case 'alignCls' @@ -403,48 +403,10 @@ 'cycle number\n',... 'stage of alignment\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_alignClassRotAvg3d(varargin{2}, varargin{3}); end - % case 'alignFrames' - % if emcProgramHelp || ... - % (length(varargin) < 9 && length(varargin) > 11) - % fprintf(['\nnameIN\n',... - % 'nameOUT\n',... - % 'gpuIDX\n',... - % 'FPN\n',... - % 'pixelSizeIN\n',... - % 'pixelSizeOUT\n',... - % 'overSampleBy\n',... - % 'doLocal [particleRadiusAng, maxRes]\n']); - % else - % switch length(varargin) - % case 9 - % BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... - % EMC_str2double(varargin{5}),... - % EMC_str2double(varargin{6}),... - % EMC_str2double(varargin{7}),... - % EMC_str2double(varargin{8}),... - % EMC_str2double(varargin{9})); - % case 10 - % BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... - % EMC_str2double(varargin{5}),... - % EMC_str2double(varargin{6}),... - % EMC_str2double(varargin{7}),... - % EMC_str2double(varargin{8}),... - % EMC_str2double(varargin{9}),... - % EMC_str2double(varargin{10})); - % case 11 - % BH_alignSubFramesTot(varargin{2}, varargin{3}, varargin{4},... - % EMC_str2double(varargin{5}),... - % EMC_str2double(varargin{6}),... - % EMC_str2double(varargin{7}),... - % EMC_str2double(varargin{8}),... - % EMC_str2double(varargin{9}),... - % EMC_str2double(varargin{10}),... - % EMC_str2double(varargin{11})); - % end - % end + case 'pca' if emcProgramHelp || ... length(varargin) ~= 4 && length(varargin) ~= 5 @@ -456,7 +418,7 @@ % ' 2 from variance\n',... % ' 3 user supplied (not recommended)\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); if (EMC_str2double(varargin{4})) % project onto full set @@ -487,7 +449,7 @@ fprintf(['\nparam.m\n',... 'cycle number\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_clusterPub(varargin{2}, varargin{3}); end case 'ctf' @@ -505,7 +467,7 @@ ' param.m [/local/Scratch]\n']); else - emC_testParse(varargin{3}) + emC_testParse(varargin{3}); switch varargin{2} case 'estimate' @@ -559,13 +521,13 @@ 'cycle number\n',... 'nTiltStart\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); if length(varargin) == 4 tiltStart = EMC_str2double(varargin{4}); else tiltStart = 1; end - BH_synthetic_mapBack(varargin{2}, varargin{3},tiltStart); + BH_synthetic_mapBack(varargin{2}, varargin{3}, tiltStart); end case 'removeDuplicates' if emcProgramHelp || ... @@ -574,7 +536,7 @@ 'cycle number\n',... ]); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_removeDuplicates(varargin{2}, varargin{3} ); end case 'geometry' @@ -590,7 +552,7 @@ 'vectOP [0,0,0]\n',... 'STD, EVE, ODD\n']); else - emC_testParse(varargin{2}) + emC_testParse(varargin{2}); BH_geometryAnalysis(varargin{2}, varargin{3},varargin{4}, ... varargin{5}, varargin{6},varargin{7}); end @@ -692,18 +654,6 @@ - %%%%%%%% GLOBALS %%%%%%%%%%%%%%%%%%%% - - % These variables are to maintain some flexibility for parameters that have - % an ill-defined dependence on experimental factors. Preferably only until - % they can be resolved. - - global bh_global_window_cutoff; - try - bh_global_window_cutoff = emc.('windowCutoff'); - catch - bh_global_window_cutoff = -2; - end % These are for making shape based masks. I think the problem is likely % dependent on the current resolution of the sub-tomogram, and that a @@ -711,6 +661,13 @@ % Note that these must also be declared in the relevant functions + global emc_debug_print; + try + emc_debug_print = emc.('debugPrint'); + catch + emc_debug_print = false; + end + %%%%%%% BH_mask3d.m %%%%%%% global bh_global_binary_mask_low_pass; global bh_global_binary_mask_threshold; @@ -841,42 +798,7 @@ % mask_3d bh_global_vol_est_scaling = 0.0; end - - try - % 0 - off, 2 original (matches closely measured MTF), 1 stronger - % Anthing else, float, iX = scalar, dX = cap val e.g. - % opiton 1 100.04 and 2 (default) is 25.06 - bh_global_MTF = emc.('mtfVal'); - catch - bh_global_MTF = 2; - end - - global bh_global_print_shifts_in_particle_basis; - try - bh_global_print_shifts_in_particle_basis = emc.('printShiftsInParticleBasis'); - catch - bh_global_print_shifts_in_particle_basis = true; - end - - global bh_global_zero_lag_score; - try - bh_global_zero_lag_score = emc.('useZeroLagScore'); - catch - bh_global_zero_lag_score = false; - end - - global bh_global_ML_compressByFactor; - global bh_global_ML_angleTolerance; - try - bh_global_ML_compressByFactor = emc.('ML_compressByFactor'); - catch - bh_global_ML_compressByFactor = 2.0; - end - try - bh_global_ML_angleTolerance = emc.('ML_angleTolerance'); - catch - bh_global_ML_angleTolerance = 5; - end + fprintf('nExpGlobals %2.2f maskLP, %2.2f maskThr, %2.2f pcaMaskThr\n', ... bh_global_binary_mask_low_pass, ... diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index d7abfc3b..4f885510 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -12,14 +12,6 @@ error('args = PARAMETER_FILE, CYCLE, STAGEofALIGNMENT') end -global bh_global_ML_compressByFactor; -global bh_global_ML_angleTolerance; -if isempty(bh_global_ML_compressByFactor) - bh_global_ML_compressByFactor = 2.0; -end -if isempty(bh_global_ML_angleTolerance) - bh_global_ML_angleTolerance = 5; -end startTime = datetime("now"); CYCLE = EMC_str2double(CYCLE); @@ -94,7 +86,7 @@ if (emc.nPeaks > 1) fprintf('For ML approach:\nUsing a compression factor %3.3f\nUsing an angulare tolerance of %3.3f degrees\n', ... - bh_global_ML_compressByFactor, bh_global_ML_angleTolerance); + emc.ML_compressByFactor, emc.ML_angleTolerance); end % for now only turn on (optionally) in reference generation. @@ -140,7 +132,6 @@ else if (emc.multi_reference_alignment && emc.classification) fprintf('\n\nMutliRef and Classify enabled.\n'); - fprintf('Only creating the global class average for PCA\n\n.'); className = 0; saveClassSum = 0; classVector{1} = [0;1]; diff --git a/transformations/BH_multi_combineLowResInfo.m b/transformations/BH_multi_combineLowResInfo.m index 51df4bca..fee84179 100755 --- a/transformations/BH_multi_combineLowResInfo.m +++ b/transformations/BH_multi_combineLowResInfo.m @@ -18,8 +18,12 @@ % meta data, so add this in soon. % oddWeight = sum(nExtracted(iClassPos,1)) ./ sum(nExtracted(iClassPos,1:2)) % eveWeight = sum(nExtracted(iClassPos,2)) ./ sum(nExtracted(iClassPos,1:2)) - oddWeight = inputCounts{1}(2,iRef) ./ (inputCounts{1}(2,iRef) + inputCounts{2}(2,iRef)) - eveWeight = inputCounts{2}(2,iRef) ./ (inputCounts{1}(2,iRef) + inputCounts{2}(2,iRef)) + oddWeight = inputCounts{1}(2,iRef) ./ (inputCounts{1}(2,iRef) + inputCounts{2}(2,iRef)); + eveWeight = inputCounts{2}(2,iRef) ./ (inputCounts{1}(2,iRef) + inputCounts{2}(2,iRef)); + + if (abs(eveWeight - oddWeight) > 0.25) + fprintf('Warning: The weights for the two half-maps (%f and %f) are not equal, this may cause issues with the final map.\n', oddWeight, eveWeight); + end diff --git a/transformations/BH_multi_loadOrBin.m b/transformations/BH_multi_loadOrBin.m index 8c9d0a61..5e081e03 100755 --- a/transformations/BH_multi_loadOrBin.m +++ b/transformations/BH_multi_loadOrBin.m @@ -94,7 +94,7 @@ % system(sprintf('newstack -shrink %d -antialias 6 %s cache/%s_bin%d%s > /dev/null', ... % samplingRate,input_tilt_series_filename, imgName, samplingRate,imgExt)); otherwise - error('DIMENSION should be 2 or 3\n.') + error('DIMENSION should be 2 or 3\n') end From a1f570950d3a485cab4e33eec5dedc23572dfe06 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 21 Feb 2024 11:45:54 -0500 Subject: [PATCH 034/151] Working out some kinks in ctf 3d --- ctf/BH_ctfCalc.m | 11 +- ctf/BH_ctf_Correct3d.m | 432 ++++++++++------------ metaData/BH_parseParameterFile.m | 6 + transformations/emc_get_max_specimen_NZ.m | 4 +- 4 files changed, 211 insertions(+), 242 deletions(-) diff --git a/ctf/BH_ctfCalc.m b/ctf/BH_ctfCalc.m index 74559f8d..c4eb7518 100755 --- a/ctf/BH_ctfCalc.m +++ b/ctf/BH_ctfCalc.m @@ -152,17 +152,12 @@ Hqz(nanCheck) = 0; end - if Phase_Only < 0 - % FIXME I need to know if I am a half grid or els this fails! - - oX = floor(CTFSIZE(1)/2)+1; oY = floor(CTFSIZE(2)/2)+1; - if (calcOneD) if (preShiftedOrigin) rV = Hqz(oX:end); @@ -191,7 +186,6 @@ firstMin = firstMin + 6; end - if ( preShiftedOrigin && ~calcOneD) if doHalfGrid freqMin = radialGrid(firstMin,ceil((CTFSIZE(1)+1)./2)); @@ -200,18 +194,16 @@ try freqMin = radialGrid(ceil((CTFSIZE(1)+1)./2)+firstMin,ceil((CTFSIZE(2)+1)./2)); catch - ceil((CTFSIZE(1)+1)./2) + ceil((CTFSIZE(1)+1)./2); end maxRes = 0.5./radialGrid(1,ceil((CTFSIZE(2)+1)/2)); end - else freqMin = radialGrid(firstMin,1); freqZero = radialGrid(firstZero,1); maxRes = 0.5./radialGrid(ceil((CTFSIZE(1)+1)/2),1); end - if (thisZero > 0) lowCut = 1./(0.1*freqMin+0.9*freqZero); if isempty(lowCut) @@ -250,7 +242,6 @@ elseif Phase_Only == 1 - ctfMask = sign(Hqz); else diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 2eb2e6e7..b7fa3e0e 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -154,10 +154,6 @@ fprintf('tiltweight is %f %f\n',tiltWeight); - - -reconScaling = 1; - [tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(emc.fastScratchDisk, use_existing_tmpCache, 'ctf3d', false); @@ -342,7 +338,6 @@ EMC_parpool(nGPUs) end -% FIXME parfor iGPU = 1:nGPUs % for iGPU = 1:nGPUs @@ -355,7 +350,7 @@ % not present. TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); TLT = load(TLTNAME); - fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); + % fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); % Get all the tomogram names that belong to a given tilt-series. @@ -403,11 +398,13 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. parfor iGPU = 1:nGPUs +% for iGPU = 1:nGPUs + % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); % Loop over each tilt for iTilt = iterList{gpuList(iGPU)} - + slab_list = {}; if (recWithoutMat) if (loadSubTomoMeta) @@ -470,30 +467,19 @@ end end - - if samplingRate > 1 fullStack = sprintf('%aliStacks/%s_ali%d.fixed', ... tiltList{iTilt},mapBackIter+1); inputStack = sprintf('cache/%s_ali%d_bin%d.fixed',... tiltList{iTilt},mapBackIter+1,samplingRate); if ~exist(inputStack, 'file') - % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s > /dev/null',samplingRate,fullStack,inputStack); - % % binCMD = sprintf('newstack -bin %d -antialias 6 %s %s ',samplingRate,fullStack,inputStack); - % - % system(binCMD); BH_multi_loadOrBin(fullStack, samplingRate, 2, false); - end else inputStack = sprintf('aliStacks/%s_ali%d.fixed',... tiltList{iTilt},mapBackIter+1); end - % system(sprintf('header %s',inputStack)); - % iHeader = MRCImage(inputStack,0); - % STACK = gpuArray(single(getVolume(iHeader))); - maskedStack = single(getVolume(MRCImage(inputStack))); if (recWithoutMat) @@ -503,32 +489,34 @@ % NY = size(maskedStack,2)-1; NZ = floor(reconstructionParameters(1)) - maxZ = NZ; + specimen_NX_nm = NZ; iCoords = [NX,0,NY-1,NZ,0,0]; tomoNumber = 1; else - [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... + [ ~, specimen_NX_nm, tomoNumber, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... 0,1); end else - [ ~, maxZ, tomoNumber, ~ ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... + [ ~, specimen_NX_nm, tomoNumber, ~ ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... 0,1); end if ( flg2dCTF || recWithoutMat && loadSubTomoMeta) - nSections = 1; - ctf3dDepth = maxZ * 10 ^ -9; + n_slabs_to_reconstruct = 1; + ctf3dDepth = specimen_NX_nm * 10 ^ -9; else + % TODO: for very thick specimen, this may be preventing the avg from getting to high enough + % resolution to be useful. So far, this is only optimized on in vitro samples. dampeningMax = 0.90; [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... - TLT(1,17),TLT(1,18),abs(TLT(1,15)), ... - 2048, TLT(1,19), ... - resTarget,maxZ*10, ... - dampeningMax,CYCLE); + TLT(1,17),TLT(1,18),abs(TLT(1,15)), ... + 2048, TLT(1,19), ... + resTarget,specimen_NX_nm*10, ... + dampeningMax,CYCLE); fprintf('\n\nCalculated a ctfDepth of %2.2f nm for %s\n\n',ctf3dDepth*10^9,tiltList{iTilt}); if (ctf3dDepth > max_ctf3dDepth) ctf3dDepth = max_ctf3dDepth; @@ -538,26 +526,26 @@ % the mean defocus determination, although this could be corrected using % knowledge of particle positions given assurance that particles are the % primary source of signal (and not carbon for example). - nSections = ceil(maxZ/(ctf3dDepth*10^9)); + n_slabs_to_reconstruct = ceil(specimen_NX_nm/(ctf3dDepth*10^9)); % max odd number - nSections = nSections + ~mod(nSections,2); + n_slabs_to_reconstruct = n_slabs_to_reconstruct + ~mod(n_slabs_to_reconstruct,2); end - fprintf('with %3.3f nm sections, correcting %d tilt-series\n', ctf3dDepth*10^9, nSections); + fprintf('with %3.3f nm sections, correcting %d tilt-series\n', ctf3dDepth*10^9, n_slabs_to_reconstruct); % For each tomo create a list of slices that are to be reconstructed % for every section section. - [ sectionList ] = calcTomoSections(iCoords, tomoNumber, emc.pixel_size_angstroms, nSections, tiltList{iTilt}, ctf3dDepth); + [ slab_list ] = calc_slab_boundaries(iCoords, tomoNumber, emc.pixel_size_angstroms, n_slabs_to_reconstruct, tiltList{iTilt}, ctf3dDepth); if (recWithoutMat) avgZ = 0; surfaceFit = 0; else - [ avgZ, maxZ, tomoNumber, surfaceFit ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... + [ avgZ, specimen_NX_nm, tomoNumber, surfaceFit ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... - sectionList, 0); + slab_list, 0); end if ( shiftDefocusOrigin ) @@ -569,9 +557,8 @@ end - - for iSection = 1:nSections - + first_slab = true(nTomos,1); + for iSection = 1:n_slabs_to_reconstruct defFitFull = ''; preCombDefocus = 0; if (mapBackIter) @@ -591,16 +578,15 @@ % FIXME, when setting up the iterator, make clean copies for each % worker that are local in scope.e - - [ correctedStack ] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... - maxZ*10/emc.pixel_size_angstroms,flgDampenAliasedFrequencies,... - preCombDefocus,samplingRate,... - applyExposureFilter,surfaceFit,... - useSurfaceFit,invertDose,... - bh_global_turn_on_phase_plate,... - filterProjectionsForTomoCPRBackground,... - emc.whitenPS); + [ correctedStack ] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... + avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... + specimen_NX_nm*10/emc.pixel_size_angstroms,flgDampenAliasedFrequencies,... + preCombDefocus,samplingRate,... + applyExposureFilter,surfaceFit,... + useSurfaceFit,invertDose,... + bh_global_turn_on_phase_plate,... + filterProjectionsForTomoCPRBackground,... + emc.whitenPS); end % Write out the stack to the cache directory as a tmp file @@ -619,13 +605,11 @@ thisTomo = tomoNumber(iT); - if any(sectionList{iT}(iSection,:)+9999) - - + if (slab_list{iT}(iSection,1)) reconName = sprintf('%s/%s_ali%d_%d_%d.rec', ... tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - + if (loadSubTomoMeta) if (recWithoutMat) @@ -651,15 +635,12 @@ fclose(rawTLT_file); if (mapBackIter) - LOCAL = sprintf('%smapBack%d/%s_ali%d_ctf.local',CWD,mapBackIter,tiltList{iTilt}, ... mapBackIter); else LOCAL = sprintf('%sfixedStacks/%s.local',CWD,tiltList{iTilt}); end - - - + if exist(LOCAL,'file') flgLocal = 1; else @@ -667,27 +648,41 @@ flgLocal = 0; end - % hangover from slab padding, remove later. - padRec = 0; - - nTiltWorkers = 2; - nTotalSlices = (iCoords(thisTomo,3)-iCoords(thisTomo,2)+1); - tiltChunkSize = ceil(nTotalSlices/nTiltWorkers); + n_slices_in_Y = (iCoords(thisTomo,3)-iCoords(thisTomo,2)+1); + % round down and then we'll add any extra needed to the final chunk + tiltChunkSize = floor(n_slices_in_Y / emc.n_tilt_workers); + % This shoulid never happen, but to be safe + if (emc.n_tilt_workers > n_slices_in_Y) + error('n_tilt_workers is greater than the number of slices in the tilt series'); + end tiltChunks = iCoords(thisTomo,2):tiltChunkSize:iCoords(thisTomo,3); tiltChunks(end) = iCoords(thisTomo,3); + % Imod expects zero indexed slices + tiltChunks = tiltChunks - 1; totalSlices = [tiltChunks(1),tiltChunks(end)]; - + + % Make sure we didn't go OOB on the first chunk + if (tiltChunks(1) < 0) + tiltChunks(1) = 0; + n_slices_in_Y = tiltChunks(end) - tiltChunks(1) + 1; + end rCMD = sprintf(['tilt %s %s -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... - super_sample, expand_lines, ... - outputStack, reconName, rawTLT, gpuList(iGPU), ... - iCoords(thisTomo,1),floor(sectionList{iT}(iSection,5))+2*padRec,... - iCoords(thisTomo,5),sectionList{iT}(iSection,6)); - + super_sample, ... + expand_lines, ... + outputStack, ... + reconName, ... + rawTLT, ... + gpuList(iGPU), ... + iCoords(thisTomo,1),... % WIDTH = NX + floor(round(slab_list{iT}(iSection,5))), ... % THICKNESS = NZ + iCoords(thisTomo,5), ... % SHIFT X + slab_list{iT}(iSection,6)); + reconScaling = 1; % Explicitly set Radial to Nyquist if (flgLocal) rCMD = [rCMD sprintf('-LOCALFILE %s -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d',LOCAL,reconScaling)]; @@ -695,27 +690,31 @@ rCMD = [rCMD sprintf('-RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d',reconScaling)]; end - system(sprintf('rm -f %s.sh',reconName)); + if isfile(sprintf('%s.sh',reconName)) + system(sprintf('rm %s.sh',reconName)); + end recScript = fopen(sprintf('%s.sh',reconName),'w'); fprintf(recScript,'#!/bin/bash\n\n'); fprintf(recScript,'%s -SLICE -1,-1 -TOTALSLICES %d,%d\n',rCMD,totalSlices); - for iRecSec = 1:nTiltWorkers-1 - if iRecSec < nTiltWorkers -1 - iShift = 1; - else +1 + iShift = 1; + for iSlab = 1:length(tiltChunks)-1 + if (iSlab == length(tiltChunks)-1) iShift = 0; - end % /dev/null - fprintf(recScript,'%s -SLICE %d,%d -TOTALSLICES %d,%d > /dev/null &\n',rCMD, ... - tiltChunks(iRecSec),... - tiltChunks(iRecSec+1)-iShift,... - totalSlices); + end + fprintf(recScript,'%s -SLICE %d,%d -TOTALSLICES %d,%d > /dev/null &\n', ... + rCMD, ... + tiltChunks(iSlab),... + tiltChunks(iSlab+1) - iShift,... + totalSlices); end fprintf(recScript,'\n\nwait\n\n'); fclose(recScript); + pause(1); system(sprintf('chmod a=wrx %s.sh',reconName)); - [recError,~] = system(sprintf('%s.sh > /dev/null ',reconName)); % /dev/null + [recError,~] = system(sprintf('%s.sh > /dev/null',reconName)); % /dev/null if (recError) system(sprintf('%s.sh',reconName)); error('\n\nerror during reconstruction %s\n\n',reconName); @@ -724,37 +723,31 @@ % monitor. For symmetrical padding this doesn't matter, but keep % in mind. /dev/null trimCMD = sprintf('trimvol -rx -y %d,%d %s.TMPPAD %s > /dev/null ' , ... - padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName); - % % % trimCMD = sprintf('newstack -fromone -secs %d-%d %s.TMPPAD %s > /dev/null', ... - % % % padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) + 1,floor(round(slab_list{iT}(iSection,5))),reconName,reconName); [msg,~]= system(trimCMD); if (msg) fprintf('%d from trimCMD\n',msg) trimCMDPrintError = sprintf('trimvol -rx -y %d,%d %s.TMPPAD %s', ... - padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) - % % % trimCMDPrintError = sprintf('newstack -fromone -secs %d-%d %s.TMPPAD %s', ... - % % % padRec+1,floor(sectionList{iT}(iSection,5))+padRec,reconName,reconName) + 1,floor(round(slab_list{iT}(iSection,5))),reconName,reconName); system(trimCMDPrintError); + error('error during trimvol'); end system(sprintf('rm %s.TMPPAD', reconName)); - % fprintf([trimCMD ' \n']) - - end - + end % end loop over tomos for this section - system(sprintf('rm %s',outputStack)); - - end % end loop over sections - + if isfile(outputStack) + system(sprintf('rm %s',outputStack)); + end + end % end loop over sectionsF() + deltaZ = []; evalMask = []; maskedStack = []; for iT = 1:nTomos thisTomo = tomoNumber(iT); - if (bh_global_turn_on_phase_plate(1)) reconNameFull = sprintf('cache/%s_%d_bin%d_filtered.rec', ... tiltList{iTilt},thisTomo,samplingRate); @@ -766,36 +759,46 @@ tiltList{iTilt},thisTomo,samplingRate); end fprintf('in ctf3d reconNameFull is %s\n\n',reconNameFull); - - recCMD = 'newstack -fromone'; - for iSection = 1:nSections - reconName = sprintf('%s/%s_ali%d_%d_%d.rec', ... - tmpCache, tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - - if any(sectionList{iT}(iSection,:)+9999) - recCMD = [recCMD,sprintf(' -secs 1-%d %s', ... - floor(sectionList{iT}(iSection,5)), ... - reconName)]; - else - fprintf('no info for section %d for tomo %d\n',iSection,thisTomo); + + % Get the total number of sections for this tomo + n_total_sections = 0; + for iSection = 1:n_slabs_to_reconstruct + if(slab_list{iT}(iSection,1)) + n_total_sections = n_total_sections + 1; end end - - system([recCMD, sprintf(' %s > /dev/null ',reconNameFull)]); %/dev/null - - - for iSection = 1:nSections - cleanUp3 = sprintf('rm %s/%s_ali%d_%d_%d.rec', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - system(cleanUp3); - cleanUp4 = sprintf('rm %s/%s_ali%d_%d_%d.rec.sh', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - system(cleanUp4); - + + if (n_total_sections == 0) + fprintf('no sections for tomo %d\n',thisTomo); + continue + end + + file_of_outputs = sprintf('%s.filelist',reconNameFull); + recombineCMD = fopen(file_of_outputs,'w'); + fprintf(recombineCMD,'%d\n', n_total_sections); + + cleanup3 = sprintf('rm %s',file_of_outputs); + for iSection = 1:n_slabs_to_reconstruct + if(slab_list{iT}(iSection,1)) + this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, thisTomo, iSection); + cleanup3 = sprintf('%s %s',cleanup3,this_slab); + fprintf(recombineCMD, '%s\n', this_slab); + fprintf(recombineCMD, '1-%d\n',floor(round(slab_list{iT}(iSection,5)))); + end + end + fclose(recombineCMD); + pause(1); + recCMD = sprintf('newstack -fromone -FileOfInputs %s -output %s\n', file_of_outputs, reconNameFull) + + [err_msg, ~] = system(sprintf('%s > /dev/null ',recCMD)); %/dev/null + if (err_msg) + fprintf('error during recombination %s\n',reconNameFull); + system(recCMD); + error('error during recombination'); end - + system(cleanup3); end % end of recombination loop maskedStack = []; @@ -845,99 +848,79 @@ end -function [ sectionList ] = calcTomoSections(iCoords, tomoNumber, pixel_size_angstroms, nSections,tiltName, ctf3Depth) +function [ slab_list ] = calc_slab_boundaries(iCoords, tomoNumber, pixel_size_angstroms, n_slabs_to_reconstruct, tiltName, ctf_3d_depth_si) + %%% This function is to produce a list of z indices, starting from 1, to pass to imod for tilt based reconstruction nTomos = length(tomoNumber); -sectionList = cell(nTomos,1); +slab_list = cell(nTomos,1); for iTomo = 1:nTomos % min and max in absolute pixels min and max from 1:nZrecon - sectionList{iTomo} = zeros(nSections,6); + slab_list{iTomo} = zeros(n_slabs_to_reconstruct,6); end % With rounding this could end up a bit short except the top and bottom are both % half a section larger than minimally needed. -nSec = floor(ctf3Depth*10^10/pixel_size_angstroms) ; -nSec = nSec + ~mod(nSec,2); -halfSec = (nSec-1)/2; +slab_size_pixels = floor(ctf_3d_depth_si * 10^10 / pixel_size_angstroms); +slab_size_pixels = slab_size_pixels + ~mod(slab_size_pixels,2); +oS = emc_get_origin_index(slab_size_pixels); for iT = 1:length(tomoNumber) iTomo = tomoNumber(iT); % Origin + originshift + tomogram_nZ = iCoords(iTomo,4); + tomogram_origin_in_tomo_frame = emc_get_origin_index(tomogram_nZ); + tomogram_origin_wrt_specimen_frame = -iCoords(iTomo,6); + fraction_origin_shift = tomogram_origin_wrt_specimen_frame - round(tomogram_origin_wrt_specimen_frame); - -1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6), - reconRange = floor([-1.*(ceil((iCoords(iTomo,4)+1)/2)-1) + iCoords(iTomo,6),0]); - reconRange(2) = reconRange(1) + iCoords(iTomo,4) - 1; - nZ = 1; - flgFirstSec = 1; + tomogram_lower_bound = round(tomogram_origin_wrt_specimen_frame) - tomogram_origin_in_tomo_frame; + recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + tomogram_nZ - 1; + % For each slab see if this tomogram has any sections in it + for iSlab = 1:n_slabs_to_reconstruct - for iSection = 1:nSections - - sectionCenter = ((nSections-1)/-2+(iSection-1))*(nSec-1); + slab_origin_in_specimen_frame = ((n_slabs_to_reconstruct-1)/-2+(iSlab-1)) * slab_size_pixels; - % Check that sectionCenter is within range - if sectionCenter + halfSec < reconRange(1) || ... - sectionCenter - halfSec > reconRange(2) - sectionList{iT}(iSection,:) = -9999; + slab_lower_bound = slab_origin_in_specimen_frame - oS + 1; + slab_upper_bound = slab_origin_in_specimen_frame + oS - 1; + slab_range = slab_lower_bound:slab_upper_bound; + + is_in_range = ismember(recon_range_z_in_specimen_frame, slab_range); + valid_indices = recon_range_z_in_specimen_frame(is_in_range); + + slab_list{iT}(iSlab,5) = length(valid_indices); + if (slab_list{iT}(iSlab,5) > 0) + slab_list{iT}(iSlab,1) = 1; else - - if (sectionCenter - halfSec > 0) - sectionList{iT}(iSection,1) = max(sectionCenter - halfSec ,reconRange(1)); - if sectionList{iT}(iSection,1) ~= reconRange(1) - sectionList{iT}(iSection,1) = sectionList{iT}(iSection,1) +1; - end - elseif (sectionCenter - halfSec < 0) - sectionList{iT}(iSection,1) = max(sectionCenter - halfSec,reconRange(1)); - if sectionList{iT}(iSection,1) ~= reconRange(1) - sectionList{iT}(iSection,1) = sectionList{iT}(iSection,1) +1; - end - else - sectionList{iT}(iSection,1) = max(-halfSec,reconRange(1)); - end - - if (sectionCenter - halfSec > 0) - sectionList{iT}(iSection,2) = min(sectionCenter + halfSec,reconRange(2)); - elseif (sectionCenter - halfSec < 0) - sectionList{iT}(iSection,2) = min(sectionCenter + halfSec ,reconRange(2)); - else - sectionList{iT}(iSection,2) = min(halfSec,reconRange(2)); - end - - - % Check that first section starts in the correct place. If a small error, - % just shift the results, otherwise complain. - if (flgFirstSec) - if sectionList{iT}(iSection,1) ~= reconRange(1) - if abs(sectionList{iT}(iSection,1) - reconRange(1)) < 10 - sectionList{iT}(iSection,1) = reconRange(1); - fprintf('\n\nShifting first section %s_n%d\n\n',tiltName,iTomo); - else - error('section start %d is too far off from expected %d\n', ... - sectionList{iT}(iSection,1), reconRange(1)) - end - end - flgFirstSec = 0; - end - - secZ = sectionList{iT}(iSection,2) - sectionList{iT}(iSection,1); - sectionList{iT}(iSection,3) = nZ; - sectionList{iT}(iSection,4) = nZ + secZ; - sectionList{iT}(iSection,5) = secZ +1; - sectionList{iT}(iSection,6) = (secZ+1)./2 + sectionList{iT}(iSection,1); - nZ = nZ + secZ + 1; + continue; end - - - % not a good solution, but not sure just yet why I'm getting some occasionally - % weird results. - if sectionList{iT}(iSection,5) < 3 - sectionList{iT}(iSection,:) = -9999; + valid_region_origin = emc_get_origin_index(slab_list{iT}(iSlab,5)); + dZ_for_reconstructed_slab = -(valid_indices(valid_region_origin) + fraction_origin_shift); + slab_list{iT}(iSlab,6) = dZ_for_reconstructed_slab; %dZ + end + + % Check to ensure we don't have any tiny slabs leftover, if so, merge them into a neighboring slab + biggest_slab = max(slab_list{iT}(:,5)); + for iSlab = 1:n_slabs_to_reconstruct + if (slab_list{iT}(iSlab,1) && slab_list{iT}(iSlab, 5) / biggest_slab < 0.1) + if (iSlab > 1 && slab_list{iT}(iSlab-1,1)) + delta = slab_list{iT}(iSlab,5); + slab_list{iT}(iSlab-1,5) = slab_list{iT}(iSlab-1,5) + delta; + slab_list{iT}(iSlab,1) = 0; + % we are adding slices from above the specimen in Z so the z shift is negative + slab_list{iT}(iSlab-1,6) = slab_list{iT}(iSlab-1,6) - delta; + elseif (iSlab < n_slabs_to_reconstruct && slab_list{iT}(iSlab+1,1)) + delta = slab_list{iT}(iSlab,5); + slab_list{iT}(iSlab+1,5) = slab_list{iT}(iSlab+1,5) + slab_list{iT}(iSlab,5); + slab_list{iT}(iSlab,1) = 0; + % we are adding slices from below the specimen in Z so the z shift is positive + slab_list{iT}(iSlab+1,6) = slab_list{iT}(iSlab+1,6) + delta; + end end - end % end loop over sections - + end + % TroubleShoot tSHT = fopen(sprintf('.tblSht_%s_i%d.txt',tiltName,iTomo),'w'); fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords(iTomo,:)'); - fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', sectionList{iT}'); + fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', slab_list{iT}'); fclose(tSHT); end % end loop over tomos @@ -948,15 +931,15 @@ -function [correctedStack] = ctfMultiply_tilt(nSections,iSection,ctf3dDepth, ... - avgZ,TLT,pixel_size_angstroms,maskedStack,... - maxZ,flgDampenAliasedFrequencies,... - preCombDefocus,samplingRate,... - applyExposureFilter,surfaceFit,... - useSurfaceFit,invertDose, ... - phakePhasePlate, ... - filterProjectionsForTomoCPRBackground,... - flgWhitenPS) +function [correctedStack] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... + avgZ,TLT,pixel_size_angstroms,maskedStack,... + specimen_NX_nm,flgDampenAliasedFrequencies,... + preCombDefocus,samplingRate,... + applyExposureFilter,surfaceFit,... + useSurfaceFit,invertDose, ... + phakePhasePlate, ... + filterProjectionsForTomoCPRBackground,... + flgWhitenPS) % Correct in strips which is more expensive but (hopefully) more accurate. @@ -966,7 +949,7 @@ if isa(surfaceFit,'cell') surfaceFit = surfaceFit{iSection}; if ~isa(surfaceFit,'sfit') - fprintf('Warning, surfaceFit is not an sfit object\n'); + % fprintf('Warning, surfaceFit is not an sfit object\n'); useSurfaceFit = false; end else @@ -986,7 +969,7 @@ if (useSurfaceFit) defocusOffset = 0; else - defocusOffset = (((nSections-1)/-2+(iSection-1))*ctf3dDepth); + defocusOffset = (((n_slabs_to_reconstruct-1)/-2+(iSection-1))*ctf3dDepth); fprintf('Not using surface fit, so using offset %3.3e for section %d with COM offset %3.3e\n', defocusOffset, iSection, avgZ); % Assuming the majority of the fit defocus came from the subtomograms, then the estimated defocus value needs to be moved from % the origin of the specimen to the origin of the subtomograms. @@ -1031,16 +1014,18 @@ radialGrid = {radialGrid./(pixel_size_angstroms*10^-10),0,phi}; phi = []; -fprintf('%f %f\n',filterProjectionsForTomoCPRBackground,pixel_size_angstroms); if (filterProjectionsForTomoCPRBackground ~= 0) bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',pixel_size_angstroms); + fprintf('Filtering input projections to %f angstroms with %f pixel size\n',bpFilter,pixel_size_angstroms); + else bpFilter = 1; end + for iPrj = 1:nPrjs - maxEval = cosd(TLT(iPrj,4)).*(d1/2) + maxZ./2*abs(sind(TLT(iPrj,4))); + maxEval = cosd(TLT(iPrj,4)).*(d1/2) + specimen_NX_nm./2*abs(sind(TLT(iPrj,4))); oX = emc_get_origin_index(d1); oY = emc_get_origin_index(d2); iEvalMask = floor(oX-maxEval):ceil(oX+maxEval); @@ -1052,18 +1037,14 @@ end iExposureFilter = iExposureFilter .* bpFilter; - ddF = TLT(iPrj,12); dPhi = TLT(iPrj,13); D0 = abs(TLT(iPrj,15)); - %TLT(iPrj,16); - padVal = BH_multi_padVal([d1,d2],fastFTSize); trimVal = BH_multi_padVal(fastFTSize,[d1,d2]); - iProjection = BH_padZeros3d(maskedStack(:,:,TLT(iPrj,1)),padVal(1,:),padVal(2,:),'GPU','singleTaper'); iProjectionFT = fftn(iProjection).*iExposureFilter; clear iExposureFilter correctedPrj = zeros([d1,d2],'single','gpuArray'); @@ -1087,10 +1068,8 @@ else rZ = zeros([d1,d2],'single','gpuArray'); end - defocus_adj = D0 - (defocusOffset.*cosd(TLT(iPrj,4))); - % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) rA = BH_defineMatrix([0,TLT(iPrj,4),0],'SPIDER','inv'); @@ -1105,11 +1084,8 @@ % is always < 0) tZ( tX < 1 | tY < 1 | tX > d1 | tY > d2) = 1; - minDefocus = min(tZ(:)); maxDefocus = max(tZ(tZ < 1)); - % Spit out some info - % To track sampling in case I put in overlap samplingMask = zeros([d1,d2],'single','gpuArray'); @@ -1118,7 +1094,6 @@ defVect = [iDefocus + ddF, iDefocus - ddF, dPhi]; if (phakePhasePlate(1) > 0) - if numel(phakePhasePlate) == 2 modPower = floor(phakePhasePlate(2)); SNR = rem(phakePhasePlate(2),1); @@ -1126,13 +1101,10 @@ modPower = 1; SNR = 1; end - - [Hqz, ~] = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,1,SNR); Hqz = (-1).^modPower.*(phakePhasePlate(1).*Hqz).^1; - modHqz = []; else if (pixel_size_angstroms < 2.0) @@ -1175,11 +1147,11 @@ clear tile Hqz end -function [avgZ, maxZ, tomoNumber,surfaceFit] = calcAvgZ(masterTM,iCoords, ... +function [avgZ, specimen_NX_nm, tomoNumber,surfaceFit] = calcAvgZ(masterTM,iCoords, ... tiltName,tomoList,... nTomos, pixel_size_angstroms,... samplingRate,cycleNumber,... - sectionList,calcMaxZ) + slab_list,calcMaxZ) % Calculate the maximum extensions in Z and then how many separate sections % need to be corrected. @@ -1194,14 +1166,14 @@ val_to_pass = 'dummy'; end -[ maxZ, tomoNumber ] = emc_get_max_specimen_NZ(val_to_pass, ... +[ specimen_NX_nm, tomoNumber ] = emc_get_max_specimen_NZ(val_to_pass, ... iCoords * samplingRate, ... tomoList, ... nTomos, ... samplingRate); -maxZ = maxZ .* pixel_size_angstroms ./ 10; -fprintf('combining the thickness and shift on tilt %s, found a maxZ %3.3f nm\n',tiltName,maxZ); +specimen_NX_nm = specimen_NX_nm .* pixel_size_angstroms ./ 10; +fprintf('combining the thickness and shift on tilt %s, found a specimen_NX_nm %3.3f nm\n',tiltName,specimen_NX_nm); if (calcMaxZ) return; @@ -1227,14 +1199,14 @@ % for each tomogram get the size and origin in Z then find mean subTomo % position. -nSections = size(sectionList{1},1); -xFull = cell(nSections,1); -yFull = cell(nSections,1); -zFull = cell(nSections,1); -surfaceFit = cell(nSections,1); +n_slabs_to_reconstruct = size(slab_list{1},1); +xFull = cell(n_slabs_to_reconstruct,1); +yFull = cell(n_slabs_to_reconstruct,1); +zFull = cell(n_slabs_to_reconstruct,1); +surfaceFit = cell(n_slabs_to_reconstruct,1); % Initialize with empty arrays -for iSection = 1:nSections +for iSection = 1:n_slabs_to_reconstruct xFull{iSection} = []; yFull{iSection} = []; zFull{iSection} = []; @@ -1272,10 +1244,10 @@ nSubTomos = nSubTomos + length(subtomo_origin_wrt_specimen_origin); - for iSection = 1:nSections + for iSection = 1:n_slabs_to_reconstruct - iSecOrigin = sectionList{iT}(iSection,6); - iSecRadius = sectionList{iT}(iSection,5)/2; + iSecOrigin = slab_list{iT}(iSection,6); + iSecRadius = slab_list{iT}(iSection,5)/2; inSectionIDX = subtomo_origin_wrt_specimen_origin > iSecOrigin - iSecRadius & subtomo_origin_wrt_specimen_origin <= iSecOrigin + iSecRadius; @@ -1297,7 +1269,7 @@ avgZ = totalZ / nSubTomos*pixel_size_angstroms / 10*10^-9; -for iSection = 1:nSections +for iSection = 1:n_slabs_to_reconstruct if length(xFull{iSection}) >= 6 surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'poly22','Robust','on'); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index c90b2fc5..dec9dd20 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -577,6 +577,12 @@ emc.mtf_value = 2.0; end +% Number of tilt processes to run in parallel in ctf 3d. +if isfield(emc, 'n_tilt_workers') + EMC_assert_numeric(emc.n_tilt_workers, 1); +else + emc.n_tilt_workers = 4; +end end diff --git a/transformations/emc_get_max_specimen_NZ.m b/transformations/emc_get_max_specimen_NZ.m index fca60af4..f30237f5 100644 --- a/transformations/emc_get_max_specimen_NZ.m +++ b/transformations/emc_get_max_specimen_NZ.m @@ -14,8 +14,8 @@ tomoNumber(iTomo) = iTomo; end - % 4 is the unbinned size of the tomogram in Z - % 6 is location of the origin in Z relative to the origin of the specimen + % 4 is the unbinned pixel size of the tomogram in Z + % 6 is location of the specimen origin in Z relative to the origin of the tomogram nZ = subTomoMeta_coords(tomoNumber(iTomo),4) ./ samplingRate; oZ = subTomoMeta_coords(tomoNumber(iTomo),6) ./ samplingRate; From e79aa844ad54fc3dec50420f69cf7ab666f04158 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 21 Feb 2024 13:08:24 -0500 Subject: [PATCH 035/151] improve distribution of work in align raw --- alignment/BH_alignRaw3d_v2.m | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index aceee036..f5364faa 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -169,13 +169,13 @@ nTomograms = length(tomoList); tiltList = masterTM.tiltGeometry; -% % Sort the list by number of active subtomos to improve parallelism -% sortedTomoList = zeros(nTomograms,1); -% for iTomo = 1:nTomograms -% sortedTomoList(iTomo) = sum(geometry.(tomoList{iTomo})(:,26)~=-9999); -% end -% [~, sortedTomoIDX] = sort(sortedTomoList,'descend') - +% Sort the list by number of active subtomos to improve parallelism +sortedTomoList = zeros(nTomograms,1); +for iTomo = 1:nTomograms + sortedTomoList(iTomo) = sum(geometry.(tomoList{iTomo})(:,26)~=-9999); +end +[~, sortedTomoIDX] = sort(sortedTomoList,'descend'); +tomoList = tomoList(sortedTomoIDX); % mask defines area for angular search, peakRADIUS restricts translational @@ -217,8 +217,6 @@ maskRadius( (peakSearch > maskRadius) ); end - - % Read in the references. % Read in the references. refIMG = cell(2,1); From 83f8fcb1c05ca4523c39239675389b1ba98b93be Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 21 Feb 2024 13:16:03 -0500 Subject: [PATCH 036/151] cleaned up some print garbage --- ctf/BH_ctf_Correct3d.m | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index b7fa3e0e..63807212 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -635,8 +635,7 @@ fclose(rawTLT_file); if (mapBackIter) - LOCAL = sprintf('%smapBack%d/%s_ali%d_ctf.local',CWD,mapBackIter,tiltList{iTilt}, ... - mapBackIter); + LOCAL = sprintf('%smapBack%d/%s_ali%d_ctf.local',CWD,mapBackIter,tiltList{iTilt}, mapBackIter); else LOCAL = sprintf('%sfixedStacks/%s.local',CWD,tiltList{iTilt}); end @@ -697,7 +696,7 @@ recScript = fopen(sprintf('%s.sh',reconName),'w'); fprintf(recScript,'#!/bin/bash\n\n'); fprintf(recScript,'%s -SLICE -1,-1 -TOTALSLICES %d,%d\n',rCMD,totalSlices); -1 + iShift = 1; for iSlab = 1:length(tiltChunks)-1 if (iSlab == length(tiltChunks)-1) @@ -996,11 +995,6 @@ WAVELENGTH = TLT(1,18); AMPCONT = TLT(1,19); -%if d2 < padTileSize -% padYdim = padTileSize; -%else -% padYdim = d2; -%end if ( flgDampenAlias ) % Calculate a centered grid b/c real space convolution [radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... From 48fddc44001f1f2b36a6b6f42fafc096948127ae Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 21 Feb 2024 14:55:01 -0500 Subject: [PATCH 037/151] working out more kinks, the newstack command is upside down from what I would have expected --- alignment/BH_templateSearch3d_2.m | 1 - ctf/BH_ctf_Correct3d.m | 34 +++++++++++++------------- transformations/BH_multi_loadOrBuild.m | 4 +-- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index d2d6db6c..0e415a62 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -183,7 +183,6 @@ mapName = sprintf('%s_%d_bin%d',tomoName,tomoNumber,samplingRate); mapExt = '.rec'; -sprintf('recon/%s_recon.coords',tomoName) [ recGeom, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName) ); reconCoords = recGeom(tomoNumber,:); diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 63807212..30cb1b59 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -397,8 +397,8 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. -parfor iGPU = 1:nGPUs -% for iGPU = 1:nGPUs +parfor iGPU = 1:nGPUs +% for iGPU = 1:nGPUs % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); @@ -428,9 +428,8 @@ end iTomoList = cell(nTomos,1); - - - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); + + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1) TLT = load(TLTNAME); fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); @@ -690,7 +689,7 @@ end if isfile(sprintf('%s.sh',reconName)) - system(sprintf('rm %s.sh',reconName)); + system(sprintf('rm %s.sh',reconName)); end recScript = fopen(sprintf('%s.sh',reconName),'w'); @@ -698,14 +697,14 @@ fprintf(recScript,'%s -SLICE -1,-1 -TOTALSLICES %d,%d\n',rCMD,totalSlices); iShift = 1; - for iSlab = 1:length(tiltChunks)-1 - if (iSlab == length(tiltChunks)-1) + for iChunk = 1:length(tiltChunks)-1 + if (iChunk == length(tiltChunks)-1) iShift = 0; end fprintf(recScript,'%s -SLICE %d,%d -TOTALSLICES %d,%d > /dev/null &\n', ... rCMD, ... - tiltChunks(iSlab),... - tiltChunks(iSlab+1) - iShift,... + tiltChunks(iChunk),... + tiltChunks(iChunk+1) - iShift,... totalSlices); end fprintf(recScript,'\n\nwait\n\n'); @@ -731,13 +730,13 @@ system(trimCMDPrintError); error('error during trimvol'); end - system(sprintf('rm %s.TMPPAD', reconName)); + system(sprintf('rm %s.TMPPAD', reconName)); end end % end loop over tomos for this section if isfile(outputStack) - system(sprintf('rm %s',outputStack)); + system(sprintf('rm %s',outputStack)); end end % end loop over sectionsF() @@ -778,7 +777,8 @@ fprintf(recombineCMD,'%d\n', n_total_sections); cleanup3 = sprintf('rm %s',file_of_outputs); - for iSection = 1:n_slabs_to_reconstruct + % for iSection = 1:n_slabs_to_reconstruct + for iSection = n_slabs_to_reconstruct:-1:1 if(slab_list{iT}(iSection,1)) this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, thisTomo, iSection); cleanup3 = sprintf('%s %s',cleanup3,this_slab); @@ -797,7 +797,7 @@ error('error during recombination'); end - system(cleanup3); + system(cleanup3); end % end of recombination loop maskedStack = []; @@ -905,13 +905,13 @@ slab_list{iT}(iSlab-1,5) = slab_list{iT}(iSlab-1,5) + delta; slab_list{iT}(iSlab,1) = 0; % we are adding slices from above the specimen in Z so the z shift is negative - slab_list{iT}(iSlab-1,6) = slab_list{iT}(iSlab-1,6) - delta; + slab_list{iT}(iSlab-1,6) = (slab_list{iT}(iSlab-1,6) - delta); elseif (iSlab < n_slabs_to_reconstruct && slab_list{iT}(iSlab+1,1)) delta = slab_list{iT}(iSlab,5); slab_list{iT}(iSlab+1,5) = slab_list{iT}(iSlab+1,5) + slab_list{iT}(iSlab,5); slab_list{iT}(iSlab,1) = 0; % we are adding slices from below the specimen in Z so the z shift is positive - slab_list{iT}(iSlab+1,6) = slab_list{iT}(iSlab+1,6) + delta; + slab_list{iT}(iSlab+1,6) = (slab_list{iT}(iSlab+1,6) + delta); end end end @@ -969,7 +969,7 @@ defocusOffset = 0; else defocusOffset = (((n_slabs_to_reconstruct-1)/-2+(iSection-1))*ctf3dDepth); - fprintf('Not using surface fit, so using offset %3.3e for section %d with COM offset %3.3e\n', defocusOffset, iSection, avgZ); + fprintf('Not using surface fit, so using offset %3.3e nm for section %d with COM offset %3.3e nm with ctf3dDepth %3.3e\n', defocusOffset*10^9, iSection, avgZ*10^9, ctf3dDepth*10^9); % Assuming the majority of the fit defocus came from the subtomograms, then the estimated defocus value needs to be moved from % the origin of the specimen to the origin of the subtomograms. defocusOffset = (defocusOffset + avgZ); % The average height of the particles is factored into the surface fit diff --git a/transformations/BH_multi_loadOrBuild.m b/transformations/BH_multi_loadOrBuild.m index f392f3aa..0cde4155 100755 --- a/transformations/BH_multi_loadOrBuild.m +++ b/transformations/BH_multi_loadOrBuild.m @@ -46,8 +46,8 @@ !mkdir -p cache nameSplit = strsplit(tomoName,'_'); -tomoName = strjoin(nameSplit(1:end-1),'_') -tomoNumber = EMC_str2double(nameSplit{end}) +tomoName = strjoin(nameSplit(1:end-1),'_'); +tomoNumber = EMC_str2double(nameSplit{end}); rCoords = rCoords ./ SAMPLING; % fix is like floor but rounds towards zero, not sure why I'm doing this here anymore. From 71ea25cd4553098ab3d476d63d02825682acb3cf Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 22 Feb 2024 15:54:28 -0500 Subject: [PATCH 038/151] Breaking changes, breaking code, breaking my fast --- alignment/BH_alignRaw3d_v2.m | 23 +- alignment/BH_templateSearch3d_2.m | 22 +- coordinates/BH_defineMatrix.m | 2 +- coordinates/BH_multi_calcBinShift.m | 31 ++- coordinates/BH_multi_gridCoordinates.m | 8 +- coordinates/BH_multi_recGeom.m | 100 ++++++- ctf/BH_ctf_Correct3d.m | 323 ++++++++++++---------- ctf/BH_ctf_Estimate.m | 2 +- ctf/BH_ctf_Updatefft.m | 2 +- masking/BH_multi_projectionMask.m | 2 +- masking/BH_weightMaskMex.m | 27 +- metaData/BH_geometryAnalysis.m | 2 +- metaData/BH_geometryInitialize.m | 162 +++-------- metaData/BH_parseParameterFile.m | 7 + metaData/BH_removeDuplicates.m | 13 +- statistics/BH_pcaPub.m | 17 +- synthetic/BH_synthetic_mapBack.m | 54 ++-- synthetic/BH_to_cisTEM_mapBack.m | 39 +-- testScripts/check_conventions.m | 72 +++++ testScripts/clusterCompile.sh | 76 ----- testScripts/emClarity.m | 2 +- testScripts/fourierTransformer.m | 13 +- transformations/BH_average3d.m | 30 +- transformations/BH_multi_loadOrBin.m | 11 +- transformations/BH_multi_loadOrBuild.m | 163 +---------- transformations/emc_get_max_specimen_NZ.m | 34 ++- transformations/emc_get_origin_index.m | 10 +- 27 files changed, 592 insertions(+), 655 deletions(-) create mode 100644 testScripts/check_conventions.m delete mode 100755 testScripts/clusterCompile.sh diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index f5364faa..15080d5f 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -686,11 +686,8 @@ % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); - - % [ binShift, ~ ] = BH_multi_calcBinShift( coords, samplingRate); binShift = [0,0,0]; nSubTomos = size(positionList,1); @@ -703,20 +700,22 @@ % Can't clear inside the parfor, but make sure we don't have two tomograms % in memory at once. - tomoNumber = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = masterTM.mapBackGeometry.(tiltName).coords(tomoNumber,:); - reconGeometry = (masterTM.reconGeometry.(tomoList{iTomo}) ./ samplingRate); + reconCoords = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); if (emc.flgCutOutVolumes) volumeData = []; else - [ volumeData, ~ ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,0); - volHeader = getHeader(volumeData); + do_load = false; + [ volumeData ] = BH_multi_loadOrBuild(tomoList{iTomo}, ... + mapBackIter, ... + samplingRate,... + iGPUidx, ... + do_load); + volHeader = getHeader(volumeData); end @@ -838,7 +837,7 @@ if (make_SF3D) make_SF3D = false; % For now excluding the soften weight. - [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, emc.wiener_constant); + [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center, reconCoords, emc.wiener_constant); imgWdgInterpolator = ''; % The unshifted mask is kept in texture mem until no longer % needed diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 0e415a62..cc6cb6cf 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -1,5 +1,5 @@ function [] = BH_templateSearch3d_2( PARAMETER_FILE,... - tomoName,tomoNumber,TEMPLATE, ... + tomoName,tomoIdx,TEMPLATE, ... SYMMETRY, wedgeType, varargin) @@ -20,7 +20,7 @@ error('emClarity templateSearch paramN.m tiltN regionN referenceName symmetry(C1) '); end -tomoNumber = EMC_str2double(tomoNumber); +tomoIdx = EMC_str2double(tomoIdx); [ useGPU ] = BH_multi_checkGPU( gpuIDX ); @@ -180,13 +180,10 @@ mapPath = './cache'; -mapName = sprintf('%s_%d_bin%d',tomoName,tomoNumber,samplingRate); +mapName = sprintf('%s_%d_bin%d',tomoName,tomoIdx,samplingRate); mapExt = '.rec'; -[ recGeom, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName) ); - -reconCoords = recGeom(tomoNumber,:); -clear recGeom +[ recGeom, ~, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName) ); bp_vals(2) = 2.*max(latticeRadius); @@ -210,10 +207,13 @@ particleThickness = latticeRadius(3); - -[ tomogram, ~] = BH_multi_loadOrBuild( sprintf('%s_%d',tomoName,tomoNumber), ... - reconCoords, mapBackIter, samplingRate,... - gpuIDX, reconScaling,1,'',super_sample); +do_load = true; +[ tomogram ] = BH_multi_loadOrBuild(sprintf('%s_%d',tomoName,tomoIdx), ... + mapBackIter, ... + samplingRate,... + gpuIDX, ... + do_load, ... + ''); % We'll handle image statistics locally, but first place the global environment diff --git a/coordinates/BH_defineMatrix.m b/coordinates/BH_defineMatrix.m index eb1af360..1213e694 100755 --- a/coordinates/BH_defineMatrix.m +++ b/coordinates/BH_defineMatrix.m @@ -84,7 +84,7 @@ if strcmpi(DIRECTION, 'forward') || strcmpi(DIRECTION, 'fwd') || strcmpi(DIRECTION, 'invVector') angles = -1.*angles; -elseif strcmpi(DIRECTION, 'inverse') || strcmpi(DIRECTION, 'inv') || strcmpi(DIRECTION, 'forwardVector') +elseif strcmpi(DIRECTION, 'inverse') || strcmpi(DIRECTION, 'inv') || strcmpi(DIRECTION, 'fwdVector') % For interpolation the vectors are applied to a grid, so the sense must % be inverted to make the final transformation active. diff --git a/coordinates/BH_multi_calcBinShift.m b/coordinates/BH_multi_calcBinShift.m index 9aadda23..a5c18b0b 100644 --- a/coordinates/BH_multi_calcBinShift.m +++ b/coordinates/BH_multi_calcBinShift.m @@ -1,4 +1,4 @@ -function [ binSize, binShift ] = BH_multi_calcBinShift(coords, samplingRate) +function [ binSize, binShift ] = BH_multi_calcBinShift(coords, samplingRate, force_odd_dimension) % Address fractional shifts on binning % Coordinates are stored relative to the lower left corner of the full % tilt/tomo. On binning a shift is needed to keep that origin in the same @@ -7,17 +7,28 @@ % way to do this is to shift the data on binning in 2d, and leave the % coordinates alone. + % Expecting just the x,y,z for a tilt series and the binning. Also may + % shift to have an odd dimension so that Imod origin is always the same. + binSize = floor(coords./samplingRate); + if (force_odd_dimension) + binSize = binSize + (1-mod(binSize,2)); + end + + originFull = emc_get_origin_index(coords); + originBin = emc_get_origin_index(binSize); -% Expecting just the x,y,z for a tilt series and the binning. Also may -% shift to have an odd dimension so that Imod origin is always the same. -binSize = floor(coords./samplingRate); -binSize = binSize - (1-mod(binSize,2)); + % 1 ++++++^+++^ + % 2 _ _ _ _ _ _ + % 3 ___ ___ ___ + % If the continuous specimen in on line 1 and the unbinned image is sampling that specimen as in line 2 + % Our goal is to have the feature that is on origin 1 (pixel 4 = ^) to be on origin 2 (pixel 2 = ^) + + % You know, typing this out makes me think it is unneeded. + + % This is the shift we need to apply to the binned image to make sure + % that the origin is in the same place. + binShift = (samplingRate.*originBin - originFull) ./ samplingRate; -originFull = floor(coords ./2) + 1; -originBin = floor(binSize./2) + 1; -% This is the shift we need to apply to the binned image to make sure -% that the origin is in the same place. -binShift = -1.*(samplingRate.*originBin - originFull) ./ samplingRate; end diff --git a/coordinates/BH_multi_gridCoordinates.m b/coordinates/BH_multi_gridCoordinates.m index 7e370f3f..d966394c 100755 --- a/coordinates/BH_multi_gridCoordinates.m +++ b/coordinates/BH_multi_gridCoordinates.m @@ -83,7 +83,7 @@ flgTrans = 0; R = [1,0,0;0,1,0;0,0,1]; dXYZ = [0,0,0]'; - DIR = 'forwardVector'; + DIR = 'fwdVector'; MAG = {1}; % The majority of function calls that are not in a resample/rescale % program call this case, and don't expect a cell output. @@ -93,7 +93,7 @@ flgGridVectors = 1; R = [1,0,0;0,1,0;0,0,1]; dXYZ = TRANSFORMATION{3}; - DIR = 'forwardVector'; + DIR = 'fwdVector'; MAG = {TRANSFORMATION{6}}; case 'single' @@ -120,7 +120,7 @@ symInc = 0; end - if strcmpi(DIR, 'inv') || strcmpi(DIR,'forwardVector') + if strcmpi(DIR, 'inv') || strcmpi(DIR,'fwdVector') MAG = {TRANSFORMATION{6}}; else MAG = {1./TRANSFORMATION{6}}; % faster to just do A(I) but left as {{}} for clarity @@ -227,7 +227,7 @@ y1 = y1 - conventionShift(2); if (flg3D); z1 = z1 - conventionShift(3); end -if strcmpi(DIR, 'inv') || strcmpi(DIR, 'forwardVector') +if strcmpi(DIR, 'inv') || strcmpi(DIR, 'fwdVector') x1 = x1 - dXYZ(1); y1 = y1 - dXYZ(2); if (flg3D); z1 = z1 - dXYZ(3); end diff --git a/coordinates/BH_multi_recGeom.m b/coordinates/BH_multi_recGeom.m index 702000dd..07cfa9bd 100755 --- a/coordinates/BH_multi_recGeom.m +++ b/coordinates/BH_multi_recGeom.m @@ -1,31 +1,103 @@ -function [ recGeom, tiltName, nTomos] = BH_multi_recGeom( reconCoordName ) +function [ recGeom, tiltName, nTomos, tilt_geometry ] = BH_multi_recGeom( reconCoordName, mapBackIter ) %UNTITLED Summary of this function goes here % Detailed explanation goes here -% could use import data in later versions of matlab (it wasn't working in -% the compiled binaries with < 15a for some reason.) -recFile = fopen(reconCoordName,'r'); -tiltName = textscan(recFile,'%s',1) ; -tiltName = tiltName{1}{1}; -nTomos = textscan(recFile,'%d',1); nTomos = nTomos{1}; -recCoords = textscan(recFile,'%f'); -fclose(recFile); +% File format: +% 1. Name of the tilt-series the tomo is reconstructed from +% 2. Number of tomograms +% 3. For each tomogram, numbered sequentially: +% 1. NX +% 2. NY start +% 3. NY end +% 4. NZ +% 5. X shift +% 6. Z shift +recFile = importdata(reconCoordName); +tiltName = recFile.textdata{1}; +nTomos = recFile.data(1); +recCoords = recFile.data(2:end); + +tilt_geometry_name = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', tiltName, mapBackIter+1); +try + tilt_geometry_name = load(tilt_geometry_name); +catch + error('Could not load the tilt geometry file: %s', tilt_geometry_name); +end + +% Check that we have a multiple of 6 entries. This should have been caught in recSCript2.sh +if mod(numel(recCoords),6) ~= 0 + error('The number of entries in the reconCoord file is not a multiple of 6'); +end + %%% Some sanity checks -% First line should be the name of the tilt-series the tomo is -% reconstructed from. +% First line should be the name of the tilt-series the tomo is reconstructed from. [~,tiltNameFromTomo,~] = fileparts(reconCoordName); tiltStr = strsplit(tiltNameFromTomo,'_'); tiltNameFromTomo = strjoin(tiltStr(1:end-1),'_'); if ~strcmp(tiltNameFromTomo, tiltName) error('the tomo base name (%s) does not match the tiltName in the coords file (%s)',tiltNameFromTomo,tiltName); end +% The tilt_geometry should have rows that are a multiple of 26 (> 26 means more than one orientation per peak) +if mod(size(tilt_geometry_name,2),26) ~= 0 + error('The tilt geometry file does not have a multiple of 26 entries/row'); +end + +% This includes all possible tomos from a given tilt-series when defined. +% Tomos may be ignored when cleaning template matching results, or later if set to be ignored +% in geometryAnalysis or if there are zero sub-tomos left. +recGeom = cell(nTomos,1); +for iTomo = 1:nTomos + read_in_Coords = (1 + (iTomo-1)*6: 6 + (iTomo-1)*6); + tomoName = sprintf('%s_%d',tiltName, iTomo); -recGeom = zeros(nTomos,6); -for iSt = 1:nTomos - recGeom(iSt,:) = recCoords{1}(1 + (iSt-1)*6: 6 + (iSt-1)*6); + % Check to make sure no out of bounds conditions were created in X Y + % when the user created the model or point file + % TODO: only checking Y b/c that results in a crash. Checking X would probably make sense too. + if read_in_Coords(2) < -75 + error(sprintf('Out of bounds condition for %s yMin at %f, 'please change recon.txt recon.coords', tomoName, read_in_Coords(2))) + elseif read_in_Coords(2) < 1 + % If not too extreme, just clamp it to 1 + read_in_Coords(2) = 1; + end + yMax = tilt_geometry(1,21); + if read_in_Coords(3) > yMax + 75 + error(sprintf('Out of bounds condition for %s yMin at %f, 'please change recon.txt recon.coords', tomoName, read_in_Coords(3), yMax)) + elseif read_in_Coords(3) > yMax + read_in_Coords(3) = yMax; + end + + + tomoCoords = struct(); + tomoCoords.('y_i') = int32(read_in_Coords(2)); + tomoCoords.('y_f') = int32(read_in_Coords(3)); + tomoCoords.('NX') = int32(read_in_Coords(tomoIdx, 1)); + tomoCoords.('NY') = int32(read_in_Coords(tomoIdx, 3) - read_in_Coords(tomoIdx, 2) + 1); + tomoCoords.('NZ') = int32(read_in_Coords(tomoIdx, 4)); + tomoCoords.('dX_specimen_to_tomo') = -1*read_in_Coords(tomoIdx, 5); + tomoCoords.('dY_specimen_to_tomo') = + (emc_get_origin_index(tomoCoords.('NY')) ... + + read_in_Coords(tomoIdx, 2)) ... % origin of the tomogram in the full tilt projection + - emc_get_origin_index(tilt_geometry(1,21)), ... % less the origin of the tilt projection = origin of the tomogram wrt the tilt projection + tomoCoords.('dZ_specimen_to_tomo') = read_in_Coords(tomoIdx, 6); + + % Check that NX, NY, NZ are all positive + if tomoCoords.('NX') <= 0 + error('NX is not positive for %s', tomoName); + end + if tomoCoords.('NY') <= 0 + error('NY is not positive for %s', tomoName); + end + if tomoCoords.('NZ') <= 0 + error('NZ is not positive for %s', tomoName); + end + + + recGeom{iTomo} = tomoCoords; end + + % Note that the x/z shifts (col 5,6) are shifts given to IMOD, which are the opposite of the location of the origin (relative to the center) % To make it more confusing, since the reconstruction is done in a ref frame rotated about X, the Z shift is flipped so it matches the origin in Z end diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 30cb1b59..5e55734c 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -15,9 +15,11 @@ % Apply a Wiener filter with this many zeros during Ctf multiplication global bh_global_turn_on_phase_plate -masterTM = struct(); +subTomoMeta = struct(); resTarget = 15; +use_inverted_newstack = false; + % TODO remove thise params tiltWeight = [0.2,0]; shiftDefocusOrigin = 1; @@ -65,19 +67,20 @@ fprintf('\n Superampling in imod is [%s] with expandLines [%s]\n',super_sample ,expand_lines); %default to cycle number zero for %determining mean z height of particles -recWithoutMat = false; reconstructionParameters = 0; filterProjectionsForTomoCPRBackground=0; -loadSubTomoMeta = true; flgWhitenPS = [0,0,0.0]; use_existing_tmpCache=''; +recon_for_tomoCPR = false; +recon_for_templateMatching = false; +recon_for_subTomo = false; if nargin > 2 if isempty(EMC_str2double(varargin{1})) error('Extra argument to ctf 3d should be a vector [THICKNESS, BINNING] tiltN, or a string templateSearch'); else reconstructionParameters = EMC_str2double(varargin{1}); - recWithoutMat = true; if length(varargin) > 2 + recon_for_tomoCPR = true; % Full recon for tomoCPR bh_global_turn_on_phase_plate = varargin{3}; filterProjectionsForTomoCPRBackground = varargin{4}; @@ -85,18 +88,12 @@ use_existing_tmpCache = varargin{5}; end else - loadSubTomoMeta = false; - % Default to on for subregion picking - % If user has specified phakePhasePlate, don;t use ...otherwise - if isempty(bh_global_turn_on_phase_plate(1)) || bh_global_turn_on_phase_plate(1) == 0 - bh_global_turn_on_phase_plate = [1,2]; - end + error('This block should ont be reached.'); end end elseif nargin > 1 if strcmpi(varargin{1},'templateSearch') - recWithoutMat = true; - loadSubTomoMeta = false; + recon_for_templateMatching = true; if (bh_global_turn_on_phase_plate(1)) fprintf('WARNING: the filtered tomogram should only be used for viz, not template matching.'); end @@ -105,11 +102,16 @@ end else % Default to zero for normal use + recon_for_subTomo = true; if isempty(bh_global_turn_on_phase_plate) bh_global_turn_on_phase_plate = 0; end end +if (recon_for_tomoCPR + recon_for_templateMatching + recon_for_subTomo ~= 1) + error('Only one of the three modes can be used at a time'); +end + try % -1, whiten before ctf, 1 whiten after - test both. usr_flgWhitenPS = emc.('whitenPS'); @@ -157,30 +159,26 @@ [tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(emc.fastScratchDisk, use_existing_tmpCache, 'ctf3d', false); -if (recWithoutMat) +if (recon_for_tomoCPR || recon_for_templateMatching) useSurfaceFit = false; - if (loadSubTomoMeta) - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR; - masterTM = subTomoMeta; clear subTomoMeta - CYCLE = masterTM.currentCycle; - else - mapBackIter = 0; - CYCLE = 0; - end +end + +if (recon_for_templateMatching) + mapBackIter = 0; + CYCLE = 0; else load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; - masterTM = subTomoMeta; clear subTomoMeta - CYCLE = masterTM.currentCycle; + subTomoMeta = subTomoMeta; clear subTomoMeta + CYCLE = subTomoMeta.currentCycle; end cycleNumber = sprintf('cycle%0.3d',CYCLE); % This should be run after raw alignment and after cycle 0 if (CYCLE) - if isfield(masterTM.(cycleNumber),'RawAlign') + if isfield(subTomoMeta.(cycleNumber),'RawAlign') fprintf(' %s \n',cycleNumber); - elseif isfield(masterTM.(sprintf('cycle%0.3d',CYCLE-1)),'RawAlign') + elseif isfield(subTomoMeta.(sprintf('cycle%0.3d',CYCLE-1)),'RawAlign') cycleNumber = sprintf('cycle%0.3d',CYCLE-1); fprintf('Falling back the previous alignment cycle\n'); else @@ -213,17 +211,17 @@ %%%%% Take these from param file later. -if (reconstructionParameters(1)) +if (recon_for_tomoCPR) samplingRate = reconstructionParameters(2); else - if (loadSubTomoMeta) + if (recon_for_subTomo) samplingRate = emc.('Ali_samplingRate'); % This number is used to roughly balance the trade off between % achievable resolution, and run time during reconstruction as % determined by the thickness of each 3d slab reconstructed. Given that % we expect the resolution to improve beyond our current value, we % multiply by 1/2, which gives a (only loosely optimized) resTarget. - resTarget = mean(masterTM.('currentResForDefocusError')*0.5); + resTarget = mean(subTomoMeta.('currentResForDefocusError')*0.5); if (emc.whitenPS(1)) emc.whitenPS(2) = resTarget; end @@ -235,22 +233,9 @@ catch resTarget = 12; end - end - -end - -try - max_ctf3dDepth = emc.('max_ctf3dDepth'); -catch - max_ctf3dDepth = 500*10^-9; end -if (max_ctf3dDepth < 1 * 10^-9 || max_ctf3dDepth > 1000 * 10^-9) - error('max_ctf3dDepth should be between 1 and 1000 nm'); -else - fprintf('Using a max_ctfDepth of %2.2f nm\n',max_ctf3dDepth*10^9); -end fprintf('Using a target resolution of %2.2f Angstroms\n',resTarget); @@ -271,15 +256,20 @@ nTomosPerTilt = 0; recGeom = 0; -if (recWithoutMat) - if reconstructionParameters(1) && loadSubTomoMeta +if (recon_for_subTomo) + [tiltList,nTilts] = BH_returnIncludedTilts(subTomoMeta.mapBackGeometry); + tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); + +else + if (recon_for_tomoCPR) tiltList{1} = varargin{2}; nTilts = 1; - % We just need one valid subtomot + % We just need on + %e valid subtomot iTry = 1; tomoList{1} = ''; while iTry < 25 - if (isfield(masterTM.mapBackGeometry.tomoName,sprintf('%s_%d',tiltList{1},iTry))) + if (isfield(subTomoMeta.mapBackGeometry.tomoName,sprintf('%s_%d',tiltList{1},iTry))) tomoList{1} = sprintf('%s_%d',tiltList{1},iTry); break; end @@ -299,9 +289,9 @@ tiltList = cell(nTilts,1); nTomosTotal = 0; nTomosPerTilt = cell(nTilts,1); - recGeom = cell(nTilts,1); + tiltRecGeom = cell(nTilts,1); for iStack = 1:nTilts - [ recGeom{iStack}, tiltName, nTomosPossible] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name) ); + [ tiltRecGeom{iStack}, tiltName, nTomosPossible, tilt_geometry] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); nTomosTotal = nTomosTotal + nTomosPossible; nTomosPerTilt{iStack} = nTomosPossible; tiltList{iStack} = tiltName; @@ -317,9 +307,6 @@ end end -else - [tiltList,nTilts] = BH_returnIncludedTilts(masterTM.mapBackGeometry); - tomoList = fieldnames(masterTM.mapBackGeometry.tomoName); end @@ -346,19 +333,21 @@ iTomoList = {}; % For now, since the tilt geometry is not necessarily updated (it is manual) - % in the masterTM, check that newer (possible perTilt refined) data is + % in the subTomoMeta, check that newer (possible perTilt refined) data is % not present. TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); TLT = load(TLTNAME); % fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); + % Get all the tomogram names that belong to a given tilt-series. - if (~recWithoutMat) + % FIXME: I'm not sure it makes sense to restrict this block to for_subTomo + if (recon_for_subTomo) nTomos = 0; alreadyMade = 0; for iTomo = 1:length(tomoList) - if strcmp(tiltList{iTilt},masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) + if strcmp(tiltList{iTilt},subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) iTomoList{nTomos+1} = tomoList{iTomo}; nTomos = nTomos + 1; end @@ -406,26 +395,30 @@ for iTilt = iterList{gpuList(iGPU)} slab_list = {}; - if (recWithoutMat) - if (loadSubTomoMeta) - nTomos = 1; - else - % templaterch - nTomos = nTomosPerTilt{iTilt}; - iCoords = recGeom{iTilt}; - end + + if (recon_for_templateMatching) + % templaterch + nTomos = nTomosPerTilt{iTilt}; + % tiltRecGeom is a cell with each value being a cell returned by multi_recGeom + iCoords = tiltRecGeom{iTilt}; + % iCoords will be a cell indexed by each tomo wwith a struct .tomoCoords else - nTomos = masterTM.mapBackGeometry.(tiltList{iTilt}).nTomos; - iCoords = masterTM.mapBackGeometry.(tiltList{iTilt}).coords; + nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; + % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; + iCoords = cell(nTomos,1); + for iCoordIdx = 1:nTomos + iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); % FIXME + end + % else + % if (recon_for_tomoCPR) + % nTomos = 1; + % else + + % end end - if (recWithoutMat && ~loadSubTomoMeta) || ~recWithoutMat - targetSizeY = diff(floor(iCoords(:,2:3)),1,2)+1; - iCoords = iCoords ./ samplingRate; - iCoords(:,1:4) = floor(iCoords(:,1:4)); - iCoords(:,3) = iCoords(:,3) - (diff(floor(iCoords(:,2:3)),1,2)+1 - floor(targetSizeY./samplingRate)); - end + iTomoList = cell(nTomos,1); @@ -435,12 +428,12 @@ - if (~recWithoutMat) + if (recon_for_subTomo) % Get all the tomogram names that belong to a given tilt-series. nTomos = 0; alreadyMade = 0; for iTomo = 1:length(tomoList) - if strcmp(tiltList{iTilt},masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) + if strcmp(tiltList{iTilt},subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) iTomoList{nTomos+1} = tomoList{iTomo}; nTomos = nTomos + 1; end @@ -481,30 +474,39 @@ maskedStack = single(getVolume(MRCImage(inputStack))); - if (recWithoutMat) - if (reconstructionParameters(1) && loadSubTomoMeta) + if (recon_for_subTomo) + [ ~, specimen_NX_nm, tomoIdx, ~ ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... + iTomoList,nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + 0,1); + else + if (recon_for_tomoCPR) NX = size(maskedStack,1); NY = size(maskedStack,2); % NY = size(maskedStack,2)-1; NZ = floor(reconstructionParameters(1)) specimen_NX_nm = NZ; - iCoords = [NX,0,NY-1,NZ,0,0]; - tomoNumber = 1; + % iCoords = [NX,0,NY-1,NZ,0,0]; + tomoIdx = 1; + % FIXME + for iCoordIdx = 1:nTomos + iCoords{iCoordIdx}.tomoCoords.dX_specimen_to_tomo = 0; + iCoords{iCoordIdx}.tomoCoords.dY_specimen_to_tomo = 0; + iCoords{iCoordIdx}.tomoCoords.dZ_specimen_to_tomo = 0; + iCoords{iCoordIdx}.tomoCoords.NX = NX; + iCoords{iCoordIdx}.tomoCoords.NY = NY-1; + iCoords{iCoordIdx}.tomoCoords.NZ = NZ; + end else - [ ~, specimen_NX_nm, tomoNumber, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... + [ ~, specimen_NX_nm, tomoIdx, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... 0,1); end - else - [ ~, specimen_NX_nm, tomoNumber, ~ ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, emc.pixel_size_angstroms, ... - samplingRate, cycleNumber,... - 0,1); end - if ( flg2dCTF || recWithoutMat && loadSubTomoMeta) + if ( flg2dCTF || recon_for_tomoCPR) n_slabs_to_reconstruct = 1; ctf3dDepth = specimen_NX_nm * 10 ^ -9; else @@ -517,8 +519,8 @@ resTarget,specimen_NX_nm*10, ... dampeningMax,CYCLE); fprintf('\n\nCalculated a ctfDepth of %2.2f nm for %s\n\n',ctf3dDepth*10^9,tiltList{iTilt}); - if (ctf3dDepth > max_ctf3dDepth) - ctf3dDepth = max_ctf3dDepth; + if (ctf3dDepth > emc.max_ctf3dDepth) + ctf3dDepth = emc.max_ctf3dDepth; fprintf('Calculated ctfDepth exceeds user specified max, so actually using a max_ctfDepth of %2.2f nm\n',ctf3dDepth*10^9); end % sections centered at 0, which for now is also supposed to coincide with @@ -534,17 +536,17 @@ % For each tomo create a list of slices that are to be reconstructed % for every section section. - [ slab_list ] = calc_slab_boundaries(iCoords, tomoNumber, emc.pixel_size_angstroms, n_slabs_to_reconstruct, tiltList{iTilt}, ctf3dDepth); + [ slab_list ] = calc_slab_boundaries(iCoords, tomoIdx, emc.pixel_size_angstroms, n_slabs_to_reconstruct, tiltList{iTilt}, ctf3dDepth, samplingRate, use_inverted_newstack); - if (recWithoutMat) + if (recon_for_subTomo) + [ avgZ, specimen_NX_nm, tomoIdx, surfaceFit ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... + iTomoList,nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + slab_list, 0); + else avgZ = 0; surfaceFit = 0; - else - [ avgZ, specimen_NX_nm, tomoNumber, surfaceFit ] = calcAvgZ(masterTM,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, emc.pixel_size_angstroms, ... - samplingRate, cycleNumber,... - slab_list, 0); end if ( shiftDefocusOrigin ) @@ -602,7 +604,7 @@ % Loop over tomos reconstructing section and appending a file to for iT = 1:nTomos - thisTomo = tomoNumber(iT); + thisTomo = tomoIdx(iT); if (slab_list{iT}(iSection,1)) @@ -610,14 +612,16 @@ tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - if (loadSubTomoMeta) - if (recWithoutMat) - TA = sortrows(masterTM.tiltGeometry.(tomoList{1}),1); - else - TA = sortrows(masterTM.tiltGeometry.(sprintf('%s_%d',tiltList{iTilt},thisTomo)),1); - end - TA = TA(:,4); - else + if (recon_for_tomoCPR) + TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{1}),1); + end + + if (recon_for_subTomo) + TA = sortrows(subTomoMeta.tiltGeometry.(sprintf('%s_%d',tiltList{iTilt},thisTomo)),1); + end + TA = TA(:,4); + + if (recon_for_templateMatching) if (mapBackIter) % FIXME: I don't think this block should work, it should only be the tilt angles! error('THis block should not be reached.') @@ -645,16 +649,23 @@ fprintf('Did not find local alignment information at %s\n',LOCAL); flgLocal = 0; end - - n_slices_in_Y = (iCoords(thisTomo,3)-iCoords(thisTomo,2)+1); + + slices_in_y = floor(iCoords{thisTomo}.tomoCoords.NY ./ samplingRate); % round down and then we'll add any extra needed to the final chunk - tiltChunkSize = floor(n_slices_in_Y / emc.n_tilt_workers); + tiltChunkSize = floor(slices_in_y / emc.n_tilt_workers); % This shoulid never happen, but to be safe - if (emc.n_tilt_workers > n_slices_in_Y) + if (emc.n_tilt_workers > iCoords{thisTomo}.tomoCoords.NY) error('n_tilt_workers is greater than the number of slices in the tilt series'); end - tiltChunks = iCoords(thisTomo,2):tiltChunkSize:iCoords(thisTomo,3); - tiltChunks(end) = iCoords(thisTomo,3); + y_i = floor(iCoords{thisTomo}.tomoCoords.y_i ./ samplingRate); + y_f = floor(iCoords{thisTomo}.tomoCoords.y_f ./ samplingRate); + if (slices_in_y ~= y_f - y_i + 1) + fprintf('slices_in_y is %d\n',slices_in_y); + fprintf('y_f - y_i + 1 is %d\n',y_f - y_i + 1); + error('slices_in_y does not match y_f - y_i + 1'); + end + tiltChunks = y_i:tiltChunkSize:y_f; + tiltChunks(end) = y_f; % Imod expects zero indexed slices tiltChunks = tiltChunks - 1; totalSlices = [tiltChunks(1),tiltChunks(end)]; @@ -674,9 +685,9 @@ reconName, ... rawTLT, ... gpuList(iGPU), ... - iCoords(thisTomo,1),... % WIDTH = NX + floor(iCoords{thisTomo}.tomoCoords.NX ./ samplingRate),... % WIDTH = NX floor(round(slab_list{iT}(iSection,5))), ... % THICKNESS = NZ - iCoords(thisTomo,5), ... % SHIFT X + floor(iCoords{thisTomo}.tomoCoords.dX_specimen_to_tomo ./ samplingRate), ... % SHIFT X slab_list{iT}(iSection,6)); @@ -745,11 +756,13 @@ maskedStack = []; for iT = 1:nTomos - thisTomo = tomoNumber(iT); + thisTomo = tomoIdx(iT); + % Note that bh_global_turn_on_phase_plate could be true for any of the recon_for_stage bools, so it must + % be checked first. if (bh_global_turn_on_phase_plate(1)) reconNameFull = sprintf('cache/%s_%d_bin%d_filtered.rec', ... tiltList{iTilt},thisTomo,samplingRate); - elseif reconstructionParameters(1) + elseif recon_for_tomoCPR reconNameFull = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec', ... CWD,tiltList{iTilt},thisTomo,samplingRate); else @@ -777,8 +790,13 @@ fprintf(recombineCMD,'%d\n', n_total_sections); cleanup3 = sprintf('rm %s',file_of_outputs); + if (use_inverted_newstack) + slab_order = n_total_sections:-1:1; + else + slab_order = 1:n_total_sections; + end % for iSection = 1:n_slabs_to_reconstruct - for iSection = n_slabs_to_reconstruct:-1:1 + for iSection = slab_order if(slab_list{iT}(iSection,1)) this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, thisTomo, iSection); cleanup3 = sprintf('%s %s',cleanup3,this_slab); @@ -847,10 +865,10 @@ end -function [ slab_list ] = calc_slab_boundaries(iCoords, tomoNumber, pixel_size_angstroms, n_slabs_to_reconstruct, tiltName, ctf_3d_depth_si) +function [ slab_list ] = calc_slab_boundaries(iCoords, tomoIdx, pixel_size_angstroms, n_slabs_to_reconstruct, tiltName, ctf_3d_depth_si, samplingRate, use_inverted_newstack) %%% This function is to produce a list of z indices, starting from 1, to pass to imod for tilt based reconstruction -nTomos = length(tomoNumber); +nTomos = length(tomoIdx); slab_list = cell(nTomos,1); for iTomo = 1:nTomos % min and max in absolute pixels min and max from 1:nZrecon @@ -863,16 +881,16 @@ slab_size_pixels = slab_size_pixels + ~mod(slab_size_pixels,2); oS = emc_get_origin_index(slab_size_pixels); -for iT = 1:length(tomoNumber) - iTomo = tomoNumber(iT); +for iT = 1:length(tomoIdx) + iTomo = tomoIdx(iT); % Origin + originshift - tomogram_nZ = iCoords(iTomo,4); - tomogram_origin_in_tomo_frame = emc_get_origin_index(tomogram_nZ); - tomogram_origin_wrt_specimen_frame = -iCoords(iTomo,6); - fraction_origin_shift = tomogram_origin_wrt_specimen_frame - round(tomogram_origin_wrt_specimen_frame); + tomo_origin_wrt_tilt_origin = iCoords.dZ_specimen_to_tomo; + tomo_origin_in_tomo_frame = emc_get_origin_index(iCoords.NZ); + + fraction_origin_shift = tomo_origin_wrt_tilt_origin - round(tomogram_origin_wrt_specimen_frame); - tomogram_lower_bound = round(tomogram_origin_wrt_specimen_frame) - tomogram_origin_in_tomo_frame; - recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + tomogram_nZ - 1; + tomogram_lower_bound = round(tomo_origin_wrt_tilt_origin) - tomo_origin_in_tomo_frame; + recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + iCoords.NZ - 1; % For each slab see if this tomogram has any sections in it for iSlab = 1:n_slabs_to_reconstruct @@ -892,7 +910,13 @@ continue; end valid_region_origin = emc_get_origin_index(slab_list{iT}(iSlab,5)); + % FIXME: this is an important departure from what I undrestood DM's convention to be. + % The easiest way to test this will be to do a sub region selection that is very clear what "up" means. + % This should produce the correct Z shift, but it requires inverting the order of the output stacks dZ_for_reconstructed_slab = -(valid_indices(valid_region_origin) + fraction_origin_shift); + if ~(use_inverted_newstack) + dZ_for_reconstructed_slab = -dZ_for_reconstructed_slab; + end slab_list{iT}(iSlab,6) = dZ_for_reconstructed_slab; %dZ end @@ -1065,7 +1089,7 @@ defocus_adj = D0 - (defocusOffset.*cosd(TLT(iPrj,4))); % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) - rA = BH_defineMatrix([0,TLT(iPrj,4),0],'SPIDER','inv'); + rA = BH_defineMatrix(TLT(iPrj,4),'TILT','fwdVector') ; % Transform the specimen plane tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ + oX); @@ -1141,11 +1165,16 @@ clear tile Hqz end -function [avgZ, specimen_NX_nm, tomoNumber,surfaceFit] = calcAvgZ(masterTM,iCoords, ... - tiltName,tomoList,... - nTomos, pixel_size_angstroms,... - samplingRate,cycleNumber,... - slab_list,calcMaxZ) +function [avgZ, specimen_NX_nm, tomoIdx,surfaceFit] = calcAvgZ(subTomoMeta, ... + iCoords, ... + tiltName, ... + tomoList,... + nTomos, ... + pixel_size_angstroms,... + samplingRate, ... + cycleNumber,... + slab_list, ... + calcMaxZ) % Calculate the maximum extensions in Z and then how many separate sections % need to be corrected. @@ -1154,14 +1183,14 @@ avgZ = 0; -if isa(masterTM,'struct') - val_to_pass = masterTM.mapBackGeometry.tomoName; +if isa(subTomoMeta,'struct') + val_to_pass = subTomoMeta.mapBackGeometry.tomoName; else val_to_pass = 'dummy'; end -[ specimen_NX_nm, tomoNumber ] = emc_get_max_specimen_NZ(val_to_pass, ... - iCoords * samplingRate, ... +[ specimen_NX_nm, tomoIdx ] = emc_get_max_specimen_NZ(val_to_pass, ... + iCoords, ... tomoList, ... nTomos, ... samplingRate); @@ -1176,12 +1205,12 @@ % For now use cycle000, if adding a refinment focused on a specific set of % particles, then consider that later. try - initGeom = masterTM.(cycleNumber).RawAlign; + initGeom = subTomoMeta.(cycleNumber).RawAlign; fprintf('Loaded the geometry for RawAlign %s\n',cycleNumber); catch fprintf('Failed to load the geometry for RawAlign %s\nTrying cycle000\n',cycleNumber); try - initGeom = masterTM.cycle000.geometry; + initGeom = subTomoMeta.cycle000.geometry; catch error(['Could not load the initial geometry subTomoMeta.%s.geometry\n or--',... 'subTomoMeta.cycle000.geometry\n'],cycleNumber); @@ -1209,19 +1238,23 @@ for iT = 1:nTomos - iTomo = tomoNumber(iT); + iTomo = tomoIdx(iT); % X in the Y frame means a vector from the Y lower left to the X origin % X origin wrt Y origin is a vector from the origin of Y to the X origin - tomoReconCoords = masterTM.reconGeometry.(tomoList{iT}) ./ samplingRate; - tomo_origin_in_tomo_frame = emc_get_origin_index(tomoReconCoords(1,1:3)); - tomo_origin_wrt_specimen_origin = -1.*tomoReconCoords(2,1:3); + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... + reconGeometry.dY_specimen_to_tomo, ... + reconGeometry.dZ_specimen_to_tomo]; + tomo_origin_in_tomo_frame = emc_get_origin_index([reconGeometry.NX, ... + reconGeometry.NY, ... + reconGeometry.NZ]); iTomoName = sprintf('%s_%d',tiltName,iTomo); % Get the z-coordinates of the origin for all included subtomograms relative to the lower left of the tomogram % shouldn't be any removed particles at this stage but later there would be. - subtomo_origin_z_in_tomo_frame = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,13) ./ samplingRate; + subtomo_origin_z_in_tomo_frame = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,13); % We should not get to this point if all subtomograms have been removed. if isempty(subtomo_origin_z_in_tomo_frame) @@ -1229,8 +1262,8 @@ end % shift from lower left to centered and include the tomos offset from the - subtomo_origin_wrt_specimen_origin = subtomo_origin_z_in_tomo_frame - tomo_origin_in_tomo_frame(3) + tomo_origin_wrt_specimen_origin(3); - + subtomo_origin_wrt_specimen_origin = subtomo_origin_z_in_tomo_frame - tomo_origin_in_tomo_frame(3) + tomo_origin_wrt_tilt_origin(3); + subtomo_origin_wrt_specimen_origin ./ samplingRate; totalZ = totalZ + sum(subtomo_origin_wrt_specimen_origin); fprintf('%s tomo has %d subTomos with mean Z %3.3f nm\n', ... iTomoName, length(subtomo_origin_wrt_specimen_origin), ... @@ -1245,10 +1278,10 @@ inSectionIDX = subtomo_origin_wrt_specimen_origin > iSecOrigin - iSecRadius & subtomo_origin_wrt_specimen_origin <= iSecOrigin + iSecRadius; - x = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,11)./samplingRate; - x = x - tomo_origin_in_tomo_frame(1) + tomo_origin_wrt_specimen_origin(1); - y = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,12)./samplingRate; - y = y - tomo_origin_in_tomo_frame(2) + tomo_origin_wrt_specimen_origin(2); + x = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,11); + x = (x - tomo_origin_in_tomo_frame(1) + tomo_origin_wrt_tilt_origin(1))./samplingRate; + y = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,12); + y = (y - tomo_origin_in_tomo_frame(2) + tomo_origin_wrt_tilt_origin(2))./samplingRate; xFull{iSection} = [xFull{iSection} ; x(inSectionIDX)]; diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 86b3f744..5b62e518 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -431,7 +431,7 @@ shiftMETHOD,'singleTaper'); if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); + bhF = fourierTransformer(iProjection); end diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index aa77c119..4e7e1cf1 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -374,7 +374,7 @@ 'GPU','singleTaper'); if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); + bhF = fourierTransformer(iProjection); end diff --git a/masking/BH_multi_projectionMask.m b/masking/BH_multi_projectionMask.m index 7503e09e..139dced3 100755 --- a/masking/BH_multi_projectionMask.m +++ b/masking/BH_multi_projectionMask.m @@ -39,7 +39,7 @@ for iPrj = 1:d3 R = BH_defineMatrix([TLT(iPrj,6),TLT(iPrj,4),TLT(iPrj,6)], ... - 'Bah','forwardVector'); + 'Bah','fwdVector'); rInv = R'; diff --git a/masking/BH_weightMaskMex.m b/masking/BH_weightMaskMex.m index f05afc27..6db4344a 100644 --- a/masking/BH_weightMaskMex.m +++ b/masking/BH_weightMaskMex.m @@ -1,5 +1,4 @@ -function [ SF3D ] = BH_weightMaskMex(SIZE, SAMPLING, TLT, ... - xyzSubTomo,reconGeometry, wiener_constant) +function [ SF3D ] = BH_weightMaskMex(SIZE, SAMPLING, TLT, subtomo_origin_in_tomo_frame, reconGeometry, wiener_constant) %UNTITLED Summary of this function goes here % Detailed explanation goes here @@ -11,13 +10,17 @@ TLT = gather(single(sortrows(TLT,1))); tiltAngles = TLT(:,4); -% reconGeometry = subTomoMeta.('reconGeometry').(tomoList{iTomo}); -pixelSize = gather(single(TLT(:,16).*SAMPLING.*10^10)); -reconShift = reconGeometry(2,:); % already at the appropriate sampling rate. -originVol = ceil((reconGeometry(1,1:3)+1)./2); +% We are only using these coordinates to figure out a change in defocus, so there is no need to worry about sampling rate. +pixelSize_angstrom = gather(single(TLT(:,16).*10^10)); +tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... + reconGeometry.dY_specimen_to_tomo, ... + reconGeometry.dZ_specimen_to_tomo]; +tomo_origin_in_tomo_frame = emc_get_origin_index([reconGeometry.NX, ... + reconGeometry.NY, ... + reconGeometry.NZ]); -prjVector = xyzSubTomo - originVol + reconShift; +subtomo_origin_in_specimen_frame = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_tilt_origin; iCs = single(TLT(:,17).*10^3); iWavelength = single(TLT(:,18).*10^10); @@ -31,9 +34,11 @@ % Need a defocus offset based on XYZ position in the tomogram for iPrj = 1:nTilts - rTilt = BH_defineMatrix(TLT(iPrj,4),'TILT','forwardVector') ; - prjCoords = rTilt * prjVector'; - iDefocus(iPrj) = iDefocus(iPrj)-(prjCoords(3).*pixelSize(iPrj)); + % If there are local alignments, then this isn't quite right, but they additional rotations about Z are < 1 degree, which + % shouldn't have a major impact here. + rTilt = BH_defineMatrix(TLT(iPrj,4),'TILT','fwdVector') ; + prjCoords = rTilt * subtomo_origin_in_specimen_frame'; + iDefocus(iPrj) = iDefocus(iPrj)-(prjCoords(3).*pixelSize_angstrom(iPrj)); end iDefocus = gather(single(iDefocus)); @@ -44,7 +49,7 @@ fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics(:)); -[SF3D] = mexSF3D(doHalfMask,doSqCTF,SIZE,pixelSize,iWavelength,iCs, ... +[SF3D] = mexSF3D(doHalfMask,doSqCTF,SIZE,pixelSize_angstrom * SAMPLING,iWavelength,iCs, ... gather(single(iDefocus + iddF)), ... gather(single(iDefocus - iddF)), ... idPHI,iPhaseShift,nTilts,tiltAngles, ... diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 9bc1e3fb..70443f26 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -402,7 +402,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... masterTM.(cycleNumber).RawAlign = rmfield(masterTM.(cycleNumber).RawAlign,(f{iOrig})); end if isfield(masterTM.mapBackGeometry.tomoName.(f{iOrig})) - tN = masterTM.mapBackGeometry.tomoName.(f{iOrig}).tomoNumber; + tN = masterTM.mapBackGeometry.tomoName.(f{iOrig}).tomoIdx; tName = masterTM.mapBackGeometry.tomoName.(f{iOrig}).tiltName; if isfield(masterTM.mapBackGeometry,tName) masterTM.mapBackGeometry.(tName).nTomos = masterTM.mapBackGeometry.(tName).nTomos - 1; diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index 29103e12..1ff39084 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -91,14 +91,12 @@ % This will have to do until a better approach based on PSF of positions % in projection space linked together can be used to define groups that % don't have co-mingled resolution. When splitOnTOmos, always run in serial, so -% that we have the best chance of distributing the defocus variateion/tomoqualtiy. +% that we have the best chance of diTomoributing the defocus variateion/tomoqualtiy. splitOnTomos = emc.('fscGoldSplitOnTomos'); if (splitOnTomos) nGPUs = 1; fprintf('override nGPUs to just 1 for initial step to evenly split crowded tomos because fscGoldSplitOnTomos is true') end -nOrientations=1;%nOrientations = emc.('pseudoMLnumber'); -nCTFgroups = 9; % Resolution lower than this is not gold standard, and will also be mixed % in the references to keep orientations from diverging. Should be > 2.25 x @@ -123,8 +121,7 @@ % If we are in the working directory, following template matching, there should % be the director convmap, holding convolution maps, model files etc. -checkDir = exist('convmap'); -if checkDir ~= 7 +if ~isdir('convmap') error('Did not find directory named ') end @@ -145,19 +142,16 @@ % There is no real need to have a separate data here. The main difference % is these have all tomos from one tilt-series and instead of the origin on -% Y, they list the slices. Should convert to include both values and also +% Y, they liTomo the slices. Should convert to include both values and also % include the size of the tilt-series which could then be removed from the % TLT geometry. I think it important to not duplicate the information as % this could lead to bugs if one is changed and the other not. The only % other concern is then linking each tomogram to the parent tilt-series. for iStack = 1:nStacks - - [ recGeom, tiltName, nTomosPossible] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name) ); + [ recGeom, tiltName, nTomosPossible, tilt_geometry ] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); % Initialize - - subTomoMeta.('mapBackGeometry').(tiltName).('coords') = zeros(nTomosPossible,6); if (doImport) iPath = dir(sprintf('convmap/%s_*.csv',tiltName)); else @@ -169,95 +163,47 @@ if nTomos > nTomosPossible error('The number of model files in convmap/*.mod is greater than the number in the recon/*.coords\n'); elseif nTomos < nTomosPossible - fprintf('\n\nThere are fewer model files in convmap for %s than are described in your coords file.\n',tiltName); + fprintf('\n\n\tThere are fewer model files in convmap for %s than are described in your coords file.\n',tiltName); fprintf('This is okay, but take note this is what you intended.\n'); end - subTomoMeta.('mapBackGeometry').(sprintf('%s',tiltName)).('nTomos') = nTomos; - subTomoMeta.('mapBackGeometry').(sprintf('%s',tiltName)).('tomoCprRePrjSize') = 512; + subTomoMeta.('mapBackGeometry').(tiltName).('nTomos') = nTomos; + subTomoMeta.('mapBackGeometry').(tiltName).('tomoCprRePrjSize') = 512; - for iSt = 1:nTomos + for iTomo = 1:nTomos if (doImport) - modName = strsplit(iPath(iSt).name,'.csv'); + modName = strsplit(iPath(iTomo).name,'.csv'); modName = strsplit(modName{1},'_'); - tomoNumber = EMC_str2double(modName{2}); + tomoIdx = EMC_str2double(modName{2}); else - modName = strsplit(iPath(iSt).name,'_'); - tomoNumber = EMC_str2double(modName{end-1}); + modName = strsplit(iPath(iTomo).name,'_'); + tomoIdx = EMC_str2double(modName{end-1}); end + + % We are storing this info to make it available when checking for duplicates + fileInfo{iTomo,1} = tiltName; + fileInfo{iTomo,2} = sprintf('%s_%d', tiltName, tomoIdx); + fileInfo{iTomo,3} = sprintf('%s_%d_bin%d',tiltName, tomoIdx, dupSampling); + tomoName = sprintf('%s_%d',tiltName,tomoIdx); + subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2}) = tilt_geometry; - subTomoMeta.('mapBackGeometry').(tiltName).('coords')(tomoNumber,:) = recGeom(tomoNumber,:); - subTomoMeta.('mapBackGeometry').('tomoName').(... - sprintf('%s_%d',tiltName,tomoNumber)).('tiltName') = tiltName; - subTomoMeta.('mapBackGeometry').('tomoName').(... - sprintf('%s_%d',tiltName,tomoNumber)).('tomoNumber') = tomoNumber; - end - - - + % Store a reference to the parent tilt-series for every tomogram + subTomoMeta.('mapBackGeometry').('tomoName').(tomoName).('tiltName') = tiltName; + % Store the tomoIdx for every tomogram, currently used to refer back to recGEom, but I'm going to put this into a struct + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('tomoIdx') = tomoIdx; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('is_active') = true; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_i') = recGeom{tomoIdx}.tomoCoords.y_i; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_f') = recGeom{tomoIdx}.tomoCoords.y_f; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NX') = recGeom{tomoIdx}.tomoCoords.NX; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NY') = recGeom{tomoIdx}.tomoCoords.NY; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NZ') = recGeom{tomoIdx}.tomoCoords.NZ; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dY_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dY_specimen_to_tomo; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dZ_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dZ_specimen_to_tomo; + end end % end of loop over stacks - -for iTomo = 1:nTomogramsTotal - - modName = strsplit(getPath(iTomo).name,'_'); - if (doImport) - tiltName = modName{1}; - tomoNumber = strsplit(modName{2},'.csv'); - tomoNumber = EMC_str2double(tomoNumber{1}) - else - tiltName = strjoin(modName(1:end-2),'_'); - tomoNumber = EMC_str2double(modName{end-1}); - end - - fileInfo{iTomo,1} = tiltName; - fileInfo{iTomo,2} = sprintf('%s_%d',tiltName,tomoNumber); - fileInfo{iTomo,3} = sprintf('%s_%d_bin%d',tiltName,tomoNumber,dupSampling); - fileInfo{iTomo,4} = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltName,mapBackIter+1); - - subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2}) = load(fileInfo{iTomo,4}); - - recCoords = importdata(sprintf('./recon/%s_recon.coords',tiltName)); - recCoords = recCoords.data; - - % The reconstruction could be defined based on the aliStacks or the - % fixedStacks. the dimensions - - recGeom = [recCoords(2 + (tomoNumber-1)*6), ... % NX - recCoords(4 + (tomoNumber-1)*6) - recCoords(3 + (tomoNumber-1)*6) + 1, ... % NY - recCoords(5 + (tomoNumber-1)*6), ... % NZ - -1*recCoords(6 + (tomoNumber-1)*6), ... OX (negative shift X in imod reconstruction command); - floor((recCoords(4 + (tomoNumber-1)*6) + recCoords(3 + (tomoNumber-1)*6) - 1)/2 - (subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2})(1,21))/2),... % oY -- need the tilt series size - recCoords(7 + (tomoNumber-1)*6)]; %OZ (negative shift Z in imod reconstruction command -- but rotated during reconstruction so the -1 is implicit); - - subTomoMeta.('reconGeometry').(fileInfo{iTomo,2}) = [recGeom(1:3);recGeom(4:6)]; - - - - % Check to make sure no out of bounds conditions were created in X Y - rXrY = subTomoMeta.mapBackGeometry.(tiltName).coords; - - if rXrY(tomoNumber,2) < -75 - error(['Out of bounds condition for %s yMin at %f,'... - 'please change recon.txt recon.coords'], ... - fileInfo{iTomo,2},rXrY(tomoNumber,2)) - elseif rXrY(tomoNumber,2) < 1 - subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,2) = 1; - end - yMax = subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2})(1,21); - if rXrY(tomoNumber,3) > yMax + 75 - error(['Out of bounds condition for %s yMax at %f (max %d),'... - 'please change recon.txt recon.coords'], ... - fileInfo{iTomo,2},rXrY(tomoNumber,3),yMax) - elseif rXrY(tomoNumber,3) > yMax - subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,3) = yMax; - end - - - -end - % For now, just assuming all of the maps are in the same place and have the same % suffix - generalize later. @@ -266,10 +212,10 @@ end -iterList = cell(nGPUs,1); +iterLiTomo = cell(nGPUs,1); for iGPU = 1:nGPUs - iterList{iGPU} = iGPU:nGPUs:nTomogramsTotal; - iterList{iGPU} + iterLiTomo{iGPU} = iGPU:nGPUs:nTomogramsTotal; + iterLiTomo{iGPU} end try @@ -288,7 +234,7 @@ D = gpuDevice(iGPU); tomoResults = struct(); - for iTomo = iterList{iGPU} + for iTomo = iterLiTomo{iGPU} if (doImport) mapName = fileInfo{iTomo,2}; @@ -298,9 +244,8 @@ % Load in the template matching geometry for the tomogram, and the model file % which may (or may not) have been edited. - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(fileInfo{iTomo,2}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(fileInfo{iTomo,2}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(fileInfo{iTomo,2}).tiltName; - coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); try tmpSearchGeom = importdata(sprintf('convmap/%s.csv',mapName)); catch @@ -327,14 +272,8 @@ % I'm assuming that the proper scaling was done, let the user know fprintf('\nImporting coordinates, assuming to be scaled properly to match full reconstruction size\n'); - else - % [ ~, binShiftTomo ] = BH_multi_calcBinShift( coords, dupInTheLoop); - - % tmpSearchGeom(:,11:13) = tmpSearchGeom(:,11:13) + repmat(binShiftTomo,size(tmpSearchGeom,1),1); end - - % Leave in for now, but check with Yunjie to remove for new import % style % New check for all -1 and then convert to Protomo @@ -391,36 +330,22 @@ % Make sure nothing has gone wrong in translating the convmap to the % full size - - - % if (flgLookForPoints) && any(abs([sx,sy,sz].*dupInTheLoop - subTomoMeta.('reconGeometry').(fileInfo{iTomo,2})(1,1:3)) > 2.*dupInTheLoop) - % fprintf('convmap/%s_convmap.mrc\n',mapName); - % error('The binned (bin%d) convmap [%d %d %d] and recon size [%d %d %d] are > %f diff\n',dupInTheLoop,sx,sy,sz,subTomoMeta.('reconGeometry').(fileInfo{iTomo,2})(1,1:3),dupInTheLoop) - % end - - - % Assuming that if (flgLookForPoints) positionMatrix = zeros(sx, sy, sz, 'single', 'gpuArray'); positionIDX = zeros(sx, sy, sz, 'uint32'); - % Make a volume with ones in the position of the centers of the tomos. - for iSubTomo = 1:size(tmpSearchGeom,1) subTomoOrigin = fix(tmpSearchGeom(iSubTomo,11:13)./dupInTheLoop); if any(subTomoOrigin < 1 + dupRadius) || any([sx,sy,sz] < subTomoOrigin + dupRadius) tmpSearchGeom(iSubTomo,26:26:26*emc.nPeaks) = -9999; - else positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = 1; positionIDX(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... tmpSearchGeom(iSubTomo, 4); - end - end % loop building position matrix for iSubTomo = 1:size(modGeom,1) @@ -439,14 +364,14 @@ overlapMatrix = convn(positionMatrix, gpuArray(dupMask), 'same'); - idxList = positionIDX((overlapMatrix > 1)); + idxLiTomo = positionIDX((overlapMatrix > 1)); size(overlapMatrix) size(positionIDX) - tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom(ismember(tmpSearchGeom(:,4), idxList),:) - sum(idxList(:)) - sum(ismember(tmpSearchGeom(:,4), idxList)) + tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom(ismember(tmpSearchGeom(:,4), idxLiTomo),:) + sum(idxLiTomo(:)) + sum(ismember(tmpSearchGeom(:,4), idxLiTomo)) else tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom; @@ -463,7 +388,7 @@ nIDX = 1; for iGPU = 1:nGPUs - for iTomo = iterList{iGPU} + for iTomo = iterLiTomo{iGPU} mapName = fileInfo{iTomo,2}; tmpGeom = parResults{iGPU}.(mapName); @@ -485,7 +410,6 @@ fprintf('nSubTomos initial = %d\n', (nIDX-1)); subTomoMeta.('nSubTomoInitial') = nIDX-1; -% subTomoMeta.('nOrientationsPerParticle') = nOrientations; preFscSplit = gather(subTomoMeta); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index dec9dd20..2f350e82 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -584,6 +584,13 @@ emc.n_tilt_workers = 4; end +if isfield(emc, 'max_ctf3dDepth') + EMC_assert_numeric(emc.max_ctf3dDepth, 1, [1 * 10^-9 || 1000 * 10^-9]); +else + emc.max_ctf3dDepth = 100*10^-9; +end + + end diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index 80a95e88..50df3b6f 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -59,11 +59,10 @@ nTotal = nTotal + length(includeList); - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - tomoName = sprintf('%s_%d',tiltName,tomoNumber); - - recGeom = subTomoMeta.reconGeometry.(tomoName); + tomoName = sprintf('%s_%d',tiltName,tomoIdx); + reconGeometry = masterTM.mapBackGeometry.tomoCoords.(tomoList{iTomo}); % iHeader = getHeader(MRCImage(tomoName)); clear postionMatrix positionIDX @@ -71,9 +70,9 @@ % sx = floor(iHeader.nX ./ dupSampling); % sy = floor(iHeader.nY ./ dupSampling); % sz = floor(iHeader.nZ ./ dupSampling); - sx = floor(recGeom(1,1)./dupSampling); - sy = floor(recGeom(1,2)./dupSampling); - sz = floor(recGeom(1,3)./dupSampling); + sx = floor(reconGeometry.NX./dupSampling); + sy = floor(reconGeometry.NY./dupSampling); + sz = floor(reconGeometry.NZ./dupSampling); positionMatrix = zeros([sx,sy,sz],'single','gpuArray'); positionIDX = zeros([sx,sy,sz],'single'); diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index e4f52f3e..fca277bb 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -657,10 +657,9 @@ tomoName = tomoList{iTomo}; iGPU = 1; - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - reconGeometry = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); + reconCoords = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); @@ -668,9 +667,12 @@ if (emc.flgCutOutVolumes) volumeData = []; else - [ volumeData, ~ ] = BH_multi_loadOrBuild( tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate, iGPU, reconScaling,0); + do_load = false; + [ volumeData ] = BH_multi_loadOrBuild(tomoList{iTomo}, ... + mapBackIter, ... + samplingRate, ... + iGPU, ... + do_load); volHeader = getHeader(volumeData); end @@ -743,8 +745,7 @@ make_sf3d = false; radialGrid = ''; padWdg = [0,0,0;0,0,0]; - [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, ... - TLT, center,reconGeometry, wiener_constant); + [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, TLT, center, reconCoords, wiener_constant); end % If flgGold there is no change, otherwise temporarily resample the diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index f0636e3e..27dc96c8 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -375,7 +375,7 @@ tiltHeader = getHeader(MRCImage(tilt_binned_filename, 0)); tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{1}).tiltName; [ maxZ ] = emc_get_max_specimen_NZ(subTomoMeta.mapBackGeometry.tomoName, ... - subTomoMeta.mapBackGeometry.(tiltName).coords, ... + subTomoMeta.mapBackGeometry.tomoCoords, ... tomoList, ... nTomograms, ... samplingRate); @@ -386,7 +386,7 @@ % the true "in microsope" dimension reconstruction_size = [tiltHeader.nX, tiltHeader.nY, maxZ]; - originRec = emc_get_origin_index(reconstruction_size); + binned_specimen_origin_in_specimen_frame = emc_get_origin_index(reconstruction_size); avgTomo = cell(3,1); % These two are mutually exclusive for now, but not enforced. @@ -515,9 +515,9 @@ fclose(iXF); positionList = geometry.(tomoList{iTomo}); - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); + coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoIdx,1:4); positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); @@ -536,14 +536,15 @@ iTLT = floor(tiltHeader.nZ); - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - % reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - tomoReconCoords = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); - - tomo_origin_in_tomo_frame = emc_get_origin_index(tomoReconCoords(1,1:3)); - tomo_origin_wrt_tilt_origin = tomoReconCoords(2,1:3); - + reconGeometry = masterTM.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... + reconGeometry.dY_specimen_to_tomo, ... + reconGeometry.dZ_specimen_to_tomo]; + tomo_origin_in_tomo_frame = emc_get_origin_index([reconGeometry.NX, ... + reconGeometry.NY, ... + reconGeometry.NZ]); nPrjs = size(TLT,1); nSubTomos = size(positionList,1); @@ -583,12 +584,12 @@ % We need to rotate the model 90 degrees around X to match the "natural" reconstruction reference frame of imod % that is [x,z,-y] - modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector'); + modelRot = BH_defineMatrix([0,90,0],'Bah','fwdVector'); for iSubTomo = 1:nSubTomos subtomo_rot_matrix = reshape(positionList(iSubTomo,17:25),3,3); - subtomo_origin_in_tomo_frame = (positionList(iSubTomo,11:13) ./ samplingRate); + subtomo_origin_in_tomo_frame = positionList(iSubTomo,11:13); subtomo_origin_wrt_tilt_origin = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_tilt_origin; iRefIDX = 1; @@ -608,11 +609,11 @@ end - % This extra shift came from experiments with real data but is both annoying and not understood. + % FIXME: This extra shift came from experiments with real data but is both annoying and not understood. subtomo_origin_wrt_tilt_origin = subtomo_origin_wrt_tilt_origin - emc.flgPreShift; % subTomo origin relative to reconLowerLeft - subtomo_origin_in_sample = originRec + subtomo_origin_wrt_tilt_origin; + subtomo_origin_in_sample = binned_specimen_origin_in_specimen_frame + subtomo_origin_wrt_tilt_origin./samplingRate; % Resample a copy of the average to match the position in the tomogram % The third entry is a dummy, normally used to make sure at least the @@ -678,11 +679,14 @@ % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_tilt_origin' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2])', fidIDX); + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', (modelRot * subtomo_origin_wrt_tilt_origin')./samplingRate + ... + [binned_specimen_origin_in_specimen_frame(1),binned_specimen_origin_in_specimen_frame(3),binned_specimen_origin_in_specimen_frame(2)]' - ... + emc.prjVectorShift([1,3,2])', ... + fidIDX); % Save a non-rotated model with each class on its own object for visualization if (emc.save_mapback_classes) - fprintf(coordCLASS,'%d 1 %0.4f %0.4f %0.4f\n', iClassIDX, subtomo_origin_wrt_tilt_origin' + originRec'- emc.prjVectorShift'); + fprintf(coordCLASS,'%d 1 %0.4f %0.4f %0.4f\n', iClassIDX, subtomo_origin_wrt_tilt_origin'./samplingRate + binned_specimen_origin_in_specimen_frame'- emc.prjVectorShift'); end for iPrj = 1:nPrjs @@ -691,19 +695,17 @@ % imod is indexing from zero zCoord = iPrj_nat; % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus)% For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) - rTilt = BH_defineMatrix([0,TLT(iPrj_nat,4),0],'SPIDER','inv'); + rTilt = BH_defineMatrix(TLT(iPrj_nat,4),'TILT','fwdVector'); prjCoords = rTilt*subtomo_origin_wrt_tilt_origin'; - fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, samplingRate.*prjCoords(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15)); - % d1 = -1.*((samplingRate.*prjCoords(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15)) - TLT(iPrj_nat,12))*10^10; - % d2 = -1.*((samplingRate.*prjCoords(3).*unsampled_pixel_size.*10^-10+TLT(iPrj_nat,15)) + TLT(iPrj_nat,12))*10^10; + fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, prjCoords(3).*unsampled_pixel_size.*10^-10+abs(TLT(iPrj_nat,15))); d1 = (abs(TLT(iPrj_nat,15)) - samplingRate.*subtomo_origin_wrt_tilt_origin(3).*unsampled_pixel_size.*10^-10) * 10^9; % Defocus value adjusted for Z coordinate in the tomogram. nm d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n', ... - fidIDX, tomoNumber,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(subtomo_rot_matrix,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); + fidIDX, tomoIdx,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(subtomo_rot_matrix,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); % These shifts are a record of transformation from the raw data, but here % we are comparing with [CTF] corrected data, from which the @@ -1097,8 +1099,8 @@ globalPeakMask = zeros([sTX,sTY,1],'single'); - globalPeakMask(originRec(1) -globalPeak : originRec(1) + globalPeak,... - originRec(2) -globalPeak : originRec(2) + globalPeak) = 1; + globalPeakMask(binned_specimen_origin_in_specimen_frame(1) -globalPeak : binned_specimen_origin_in_specimen_frame(1) + globalPeak,... + binned_specimen_origin_in_specimen_frame(2) -globalPeak : binned_specimen_origin_in_specimen_frame(2) + globalPeak) = 1; globalBinary = ( globalPeakMask > 0 ); % Zero and only changed if CTF is refined. @@ -1164,7 +1166,7 @@ % loop begins, they fail to load. It is fine as a regular for loop % though - annoying, but very little overhead. It would be nice % to know what is going on here. - bhF = fourierTransformer(randn(CTFSIZE,'single','gpuArray'),'OddSizeOversampled'); + bhF = fourierTransformer(randn(CTFSIZE,'single','gpuArray')); iMrcObj = MRCImage(tiltSeries,0); iMrcObjRef = MRCImage(sprintf('%smapBack%d/%s_1_mapBack.st',mbOUT{1:3}),0); @@ -1325,7 +1327,7 @@ - estPeak = [mRx, mRy] - originRec(1:2) + [comPRJX, comPRJY]; + estPeak = [mRx, mRy] - binned_specimen_origin_in_specimen_frame(1:2) + [comPRJX, comPRJY]; glbList = fopen(sprintf('%smapBack%d/%s_%03d.global',mbOUT{1:3},iPrj),'w'); % Add unique indicies to prevent ambiquity when comparing with paral fprintf(glbList,'%f degree tilt at %f %f\n', TLT(iPrj,4),estPeak); diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 1f90cd99..44836894 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -273,7 +273,7 @@ tiltHeader = getHeader(MRCImage(tilt_filename, 0)); tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{1}).tiltName; [ maxZ ] = emc_get_max_specimen_NZ(subTomoMeta.mapBackGeometry.tomoName, ... - subTomoMeta.mapBackGeometry.(tiltName).coords, ... + subTomoMeta.mapBackGeometry.tomoCoords, ... tomoList, ... nTomograms, ... 1); @@ -374,9 +374,9 @@ end positionList = geometry.(tomoList{iTomo}); - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); + coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoIdx,1:4); positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); @@ -388,13 +388,16 @@ sTX = floor(tiltHeader.nX); sTY = floor(tiltHeader.nY); - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - % reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - tomoReconCoords = (subTomoMeta.reconGeometry.(tomoList{iTomo})); + reconGeometry = masterTM.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... + reconGeometry.dY_specimen_to_tomo, ... + reconGeometry.dZ_specimen_to_tomo]; + tomo_origin_in_tomo_frame = emc_get_origin_index([reconGeometry.NX, ... + reconGeometry.NY, ... + reconGeometry.NZ]); - tomo_origin_in_tomo_frame = emc_get_origin_index(tomoReconCoords(1,1:3)); - tomo_origin_wrt_tilt_origin = tomoReconCoords(2,1:3); nPrjs = size(TLT,1); nSubTomos = size(positionList,1); @@ -406,7 +409,7 @@ continue; end - modelRot = BH_defineMatrix([0,90,0],'Bah','forwardVector'); + modelRot = BH_defineMatrix([0,90,0],'Bah','fwdVector'); for iSubTomo = 1:nSubTomos @@ -432,7 +435,7 @@ % imod is indexing from zero zCoord = iPrj_nat; % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) - rTilt = BH_defineMatrix([0,TLT(iPrj_nat,4),0],'SPIDER','inv'); + rTilt = BH_defineMatrix(TLT(iPrj_nat,4),'TILT','fwdVector') ; prjCoords = rTilt*subtomo_origin_wrt_tilt_origin'; @@ -444,7 +447,7 @@ d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n', ... - fidIDX, tomoNumber, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_nat,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_nat), postExposure(iPrj_nat), positionList(iSubTomo,7)); + fidIDX, tomoIdx, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_nat,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_nat), postExposure(iPrj_nat), positionList(iSubTomo,7)); else fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',-9999, -9999,-9999,1.0,1.0,1.0,1,1,1,1,1,1,1,1,1,0,0,1); end @@ -683,15 +686,19 @@ if ( x_start > 0 && y_start > 0 && x_start + tileSize(1) - 1 < sTX && y_start + tileSize(2) - 1 < sTY ) output_particle_stack(:,:,iGpuDataCounter) = STACK(x_start:x_start+tileSize(1)-1,y_start:y_start+tileSize(2)-1,TLT(iPrj,1)); + % The trasformation of the particle is e1,e2,e3,esym into it's postion in the tomogram frame, then + % the tomogram is tilted about the original Y axis and then the original Z + % The angles stored are those used for interpolation, i.e. produced from BH_define_matrix([e1, e2, e3], 'Bah', 'inv' (or 'forwardVector')) + % The angles are flipped in order so that the rotation matrix is R3*R2*R1 (really they should be flipped and negated, so the abvoe should be -e1, -e2, -e3, -esym) if (useFixedNotAliStack) - rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); + rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','fwdVector'); RF = fullXform(TLT(iPrj,1),1:4); rotFull = rTilt*[RF(1), RF(2), 0; RF(3), RF(4), 0; 0, 0, 1]*reshape(wrkPar(iFid,7:15),3,3); else - - % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); - rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','forwardVector'); + % This gives us Rz*Ry + rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','fwdVector'); + % this fives Rz*Ry*e3*e2*e1 * interpolant would rotate the particle by Rz*Ry*e1*e2*e3 rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); end @@ -732,7 +739,7 @@ % df1 = ( wrkPar(iFid,4) + wrkPar(iFid,5)) * 10; % df2 = ( wrkPar(iFid,4) - wrkPar(iFid,5)) * 10; % dfA = wrkPar(iFid,6) - % fidIDX, tomoNumber, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_nat,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_nat), postExposure(iPrj_nat), positionList(iSubTomo,7)); + % fidIDX, tomoIdx, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_nat,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_nat), postExposure(iPrj_nat), positionList(iSubTomo,7)); df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; diff --git a/testScripts/check_conventions.m b/testScripts/check_conventions.m new file mode 100644 index 00000000..59f98cce --- /dev/null +++ b/testScripts/check_conventions.m @@ -0,0 +1,72 @@ +function [ ] = check_conventions() + +% Make some basic data so we can test transformations and indexing options. +eve_size=12; +size_val = {[1,1].*eve_size + 1, [1,1].*eve_size}; +img = {zeros(size_val{1}, 'single'), zeros(size_val{2}, 'single')}; +filename_init = {'/tmp/odd.mrc', '/tmp/eve.mrc'}; +filename_xf = {'/tmp/odd_rot.mrc', '/tmp/eve_rot.mrc'}; + +% Pick a non 1.0 pixel size to look for any issues with the indexing. +pixel_size = 1.4; + +% The center of the pixel at this index is the origin of the image (zero). +% It is intended that on any transformations, an object in the continuous data remain centered at this index. +origin = cell(2,1); +for i = 1:2 + origin{i} = emc_get_origin_index(size_val{i}); + % Place a single point at X=3 and then transform the image to make sure we are where we expect to be + img{i}(origin{i}(1) + 3, origin{i}(2)) = 1; + SAVE_IMG(img{i}, filename_init{i}, pixel_size); +end + + +% First test imod transformations +% Define a rotation that rotates around Z axis by 90 degrees, this will place the point at X=3 to Y=3 +% Note that if convention were fwd, or invVector, this would rotate the interpolant vector Y=3 to X=3, resulting in the data at X=3 to be at Y=3. +rot_mat = BH_defineMatrix([0,0,90], 'Bah', 'fwdVector'); +% Imod uses a row major matrix, so we need to transpose the matrix +rot_mat = rot_mat'; + +for i = 1:2 + % Imod defines an origin that is always centered in an image, which means for + % eve images, it is inbetween pixels, and for odd images, it is at the center of a pixel. + % The origin may be calculated as (float(size) + 1.f)/2.f + % emClarity uses the more standard convention of the origin being size/2 (zero based indexing, integer division). + % So for matlab with 1 based, we calculate floor(size/2) + 1 + % The following will produce the shifts needed to shift the origin to the center of the pixel. + origin_offset = [0.5,0.5,0.5]' * (1-mod(i, 2)); + test_shifts = rot_mat * origin_offset + origin_offset + + % NOTE: this means that if imod defines a transformation between two images (eg with tiltxcorr or tiltalign) + % say as in the example of 10,7 -> 7,10, it would be fine if odd size, but for even sized, it would instead + % determine that the transformation was a rotation followed by this shift, so if we want to + % apply this in emClarity, + + f = fopen('/tmp/rot.xf', 'w'); + % From the newstack man page: + % A11 A12 A21 A22 DX DY + % where the coordinate (X, Y) is transformed to (X', Y') by: + % X' = A11 * (X - Xci) + A12 * (Y - Yci) + DX + Xco + % Y' = A21 * (X - Xci) + A22 * (Y - Yci) + DY + Yco + % where (Xci, Yci) and (Xco, Yco) are the center coordinates of the input + % and output images, respectively. + fprintf(f, '%f %f %f %f %f %f\n', rot_mat([1,2,4,5]), test_shifts(1:2)); + fclose(f); + + err_msg = system(sprintf('newstack -TransformFile %s %s %s > /dev/null', '/tmp/rot.xf', filename_init{i}, filename_xf{i})); + if (err_msg ~= 0) + error('Failed to transform the image using newstack'); + end +end + +test_img = getVolume(MRCImage('/tmp/rot.mrc')); +if any(size(test_img) ~= size(eve)) + error('The transformed image has a different size than the original'); +end +% test_img = round(test_img); +if test_img(origin_eve, origin_eve + 3) ~= 1 + test_img + error('The transformed image does not have the point at the expected location'); +end +end \ No newline at end of file diff --git a/testScripts/clusterCompile.sh b/testScripts/clusterCompile.sh deleted file mode 100755 index f8126b78..00000000 --- a/testScripts/clusterCompile.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash - - -shortHead=$(git rev-parse --short HEAD) - -mFile="emClarity.m" - -post="_${shortHead}" - -outName="$(basename ${mFile} .m)${post}" - -binaryOutName="test2018" -scriptOutName=18b -matlab -nosplash -nodisplay -nojvm -r "mcc -m ${mFile} -a fitInMap.py -a emC_ctffind -o "$(basename ${mFile} .m)_${binaryOutName}" ; exit" & - -#-R -nodisplay gui obviously needs a display - -wait - rm mccExcludedFiles.log - rm readme.txt - rm run_*.sh - rm requiredMCRProducts.txt - -if [ -f emClarity ] ; then - mv emClarity emClarity~ -fi - -# Matlab (mcc) complains if ther is an underscore in the name. -#mv emClarity${binaryOutName} emClarity_${binaryOutName} - -{ - -echo '#!/bin/bash' -echo '' -echo '#Please modify this line to point to the text file in your MCR root' -echo '#where you pasted the lines suggested to add to LD_LIBRARY_PATH during install.' -echo "#MCR_BASH="/work/thirdParty/MATLAB/mcr_bash_${scriptOutName}.sh"" -echo 'MCR_BASH=""' -echo '' -echo '' -echo '#Please modify this line to point to the install for emClarity binary' -echo '#emClarity_ROOT=/work/emClarity' -echo 'emClarity_ROOT=""' -echo '' -echo '' -echo '' -echo '' -echo 'if [ -f ${MCR_BASH} ]; then' -echo ' source ${MCR_BASH}' -echo 'else' -echo ' echo "Did not find your mcr_bash file, did you fill it in above?"' -echo ' exit 1' -echo 'fi' -echo '' -echo "if [ ! -f \${emClarity_ROOT}/emClarity_${binaryOutName} ]; then" -echo ' echo "Did not find the binary on the path, did you fill it in above?"' -echo ' exit 1' -echo 'fi' -echo '' -echo "argList="${shortHead} "" -echo 'while [ $# -gt 0 ]; do' -echo ' token=$1' -echo ' argList="${argList} ${token}"' -echo ' shift' -echo 'done' -echo '' -echo "\${emClarity_ROOT}/emClarity_${binaryOutName} \${argList}" - - -} > emClarity_${scriptOutName} - -chmod a=wrx emClarity_${scriptOutName} - -mkdir -p ../bin -mv emClarity_${scriptOutName} -mv emClarity_${binaryOutName} /groups/grigorieff/home/himesb/thirdParty/emClarity diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index ecc1d38c..93ce6ba2 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -565,7 +565,7 @@ ~ismember(length(varargin),[7,8]) fprintf(['\nparam.m\n',... 'tomoName\n',... - 'tomoNumber\n', ... + 'tomoIdx\n', ... 'template name\n',... 'symmetry\n',... '[threshold override]\n',... diff --git a/testScripts/fourierTransformer.m b/testScripts/fourierTransformer.m index a5a5d455..83ad6e27 100644 --- a/testScripts/fourierTransformer.m +++ b/testScripts/fourierTransformer.m @@ -31,7 +31,6 @@ paddedVolumeIsNonZero = true; shouldPad = false; - OddSizeOversampled = 0; useFwdSwapForInverse; % works for even sized images @@ -44,11 +43,7 @@ if nargin > 1 if (ischar(varargin{1})) - if (strcmpi(varargin{1},'OddSizeOversampled')) - obj.OddSizeOversampled = 1; - else - error('Did not recognize the extra argument when intializing the fourierTransformer'); - end + error('Did not recognize the extra argument when intializing the fourierTransformer'); else if (isnumeric(varargin{1})) if (numel(varargin{1}) == 6) @@ -234,7 +229,7 @@ function delete(obj) [ obj.phaseCenter, dV, dW] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... {'none'},1,0,0,{'halfgrid'}); if ((obj.inputSize(1) == obj.inputSize(2)) && (obj.inputSize(2) == obj.inputSize(3))) - sx = obj.halfDimSize-1+obj.OddSizeOversampled; + sx = obj.halfDimSize-1+obj.phaseSwapOffset(1); obj.phaseCenter = exp(-2i.*pi.*sx.*(obj.phaseCenter+dV+dW)); clear dU dV dW else @@ -248,7 +243,9 @@ function delete(obj) end - + % Note that in cisTEM, the same shift is applied (multiplication), so not only does + % the shift need to be different for even/odd, it depends if shifting forward or backward + % Here, we just invert the shift for the inverse. if strcmp(direction,'fwd') inputVol = inputVol .* obj.phaseCenter; elseif strcmp(direction,'inv') diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 4f885510..517ba2c0 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -725,9 +725,9 @@ tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,1:4); + reconGeometry = masterTM.mapBackGeometry.tomoCoords.(tomoList{iTomo}); [ binShift ] = [0,0,0];%BH_multi_calcBinShift( coords, samplingRate); % Load in the geometry for the tomogram, and get number of subTomos. @@ -739,21 +739,22 @@ volumeData = []; %fprintf('loading tomo %d\n',iTomo); - tomoNumber = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoNumber; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoNumber,:); - reconGeometry = (subTomoMeta.reconGeometry.(tomoList{iTomo}) ./ samplingRate); + if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) volumeData = []; else reconScaling = 1; - [ volumeData, ~ ] = BH_multi_loadOrBuild( ... - tomoList{iTomo}, ... - reconCoords, mapBackIter, ... - samplingRate,iGPUidx,reconScaling,0); - + do_load = false; + [ volumeData ] = BH_multi_loadOrBuild(tomoList{iTomo}, ... + mapBackIter, ... + samplingRate, ... + iGPUidx,... + do_load); + volHeader = getHeader(volumeData); end @@ -853,17 +854,12 @@ center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); wdgIDX = positionList(iSubTomo,9); - - % tmpang = BH_defineMatrix([0,0,-14],'Bah','inv'); - % angles = reshape(angles,3,3)*tmpang; - % if (flgFinalAvg) - % angles = reshape(angles,3,3)*oddRot; - % end + TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); if (make_sf3d) - [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center,reconGeometry, emc.wiener_constant); + [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center, reconGeometry, emc.wiener_constant); make_sf3d = false; end diff --git a/transformations/BH_multi_loadOrBin.m b/transformations/BH_multi_loadOrBin.m index 5e081e03..15b13017 100755 --- a/transformations/BH_multi_loadOrBin.m +++ b/transformations/BH_multi_loadOrBin.m @@ -66,8 +66,12 @@ pixelSize = iHeader.cellDimensionX/iHeader.nX; % Assuming X/Y the same and Z might be incorrect. - [binSize, binShift] = BH_multi_calcBinShift([iHeader.nX, iHeader.nY], samplingRate); - + % FIXME: forcing odd bin size so that transformations are the same as IMOD where the origin is between pixels for + % even sized images. + force_odd_dimension = true; + [binSize, binShift] = BH_multi_calcBinShift([iHeader.nX, iHeader.nY], samplingRate, force_odd_dimension); + % FIXME + binShift = binShift .* 0; % Gridding correction for the interpolation in the binning. Not % sure this is quite right, but it looks much better. TODO FIXME [ R ] = BH_multi_gridCoordinates([iHeader.nX,iHeader.nY],'Cartesian','GPU', {'none'},1,1,1); @@ -79,7 +83,8 @@ iProjection = gpuArray(getVolume(tiltObj,[],[],iPrj,'keep')); if (iPrj == 1) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); + % FIXME: The swapping + bhF = fourierTransformer(iProjection); end iProjection = bhF.invFFT(bhF.fwdFFT(R.*iProjection,0,0,[1e-6,600,samplingRate*pixelSize,pixelSize]),2); diff --git a/transformations/BH_multi_loadOrBuild.m b/transformations/BH_multi_loadOrBuild.m index 0cde4155..56d3519e 100755 --- a/transformations/BH_multi_loadOrBuild.m +++ b/transformations/BH_multi_loadOrBuild.m @@ -1,172 +1,59 @@ -function [ IMG_OUT, reconGeom ] = BH_multi_loadOrBuild( tomoName, ... - rCoords, ... - mapBackIter, ... - SAMPLING, ... - gpuIDX,... - reconScaling, ... - varargin) +function [ IMG_OUT ] = BH_multi_loadOrBuild(tomoName, ... + mapBackIter, ... + SAMPLING, ... + gpuIDX,... + varargin) %Check to see if a cached binned image exists, either load or reconstruct % Switched to using imod's newstack and binvol to create binning and % removed inline binning from my workflow. IMG_OUT = ''; -% If gpuIDX is negative, this is from template matching, so allow -% reconstruction of non-CTF stack. Otherwise throw error if existing -% reconstruction is not available. -if gpuIDX < 0 - doRecon = 1; - gpuIDX = abs(gpuIDX); -else - doRecon = 0; -end - - -if nargin > 6 +if nargin > 4 flgLoad = varargin{1}; end recon = ''; -if nargin > 7 +if nargin > 5 recon = varargin{2}; end -ctf = ''; ali = 'ali'; -super_sample = ''; +super_sample = ''; % not used expand_lines = ''; -if nargin > 8 +if nargin > 6 if ~isempty(varargin{3}) super_sample = varargin{3}; end - - % expand_lines = varargin{4} end !mkdir -p cache nameSplit = strsplit(tomoName,'_'); tomoName = strjoin(nameSplit(1:end-1),'_'); -tomoNumber = EMC_str2double(nameSplit{end}); +tomoIdx = EMC_str2double(nameSplit{end}); -rCoords = rCoords ./ SAMPLING; -% fix is like floor but rounds towards zero, not sure why I'm doing this here anymore. -rCoords(1:4) = fix(rCoords(1:4)); - - - -checkStack = sprintf('%sStacks/%s_ali%d%s.fixed',ali,tomoName,mapBackIter+1,ctf); +checkStack = sprintf('%sStacks/%s_ali%d.fixed',ali,tomoName,mapBackIter+1); if isempty(recon) % Otherwise name is varargin 8 from mapBack - recon = sprintf('cache/%s_%d_bin%d.rec',tomoName,tomoNumber,SAMPLING); + recon = sprintf('cache/%s_%d_bin%d.rec',tomoName,tomoIdx,SAMPLING); end if SAMPLING > 1 - - stack = sprintf('cache/%s_ali%d%s_bin%d.fixed',tomoName,mapBackIter+1,ctf,SAMPLING); + stack = sprintf('cache/%s_ali%d_bin%d.fixed',tomoName,mapBackIter+1,SAMPLING); if ~exist(stack, 'file') BH_multi_loadOrBin(checkStack, SAMPLING, 2, true); %%%%% med filt flag end - else - - stack = sprintf('%sStacks/%s_ali%d%s.fixed',ali,tomoName,mapBackIter+1,ctf); - + stack = sprintf('%sStacks/%s_ali%d.fixed',ali,tomoName,mapBackIter+1); end - -if exist(recon,'file') || ~doRecon +if exist(recon,'file') header = getHeader(MRCImage(stack,0)); - [ reconGeom ] = calc_rg( header, rCoords ); -elseif (doRecon) - - if (mapBackIter) - TLT = sprintf('mapBack%d/%s_ali%d_ctf.tlt',mapBackIter,tomoName,... - mapBackIter); - LOCAL = sprintf('mapBack%d/%s_ali%d_ctf.local',mapBackIter,tomoName, ... - mapBackIter); - else - TLT = sprintf('fixedStacks/%s.tlt',tomoName); - LOCAL = sprintf('fixedStacks/%s.local',tomoName); - end - - if exist(LOCAL,'file') - flgLocal = 1; - else - fprintf('Did not find local alignment information at %s\n',LOCAL); - flgLocal = 0; - end - - % check to see if the binned stack exists and is readable - [initialCheckFail,~] = system(sprintf('header %s',stack)); - - if (initialCheckFail) - % See if the file exists but is being written by another process. - if exist( stack, 'file') - % It is there but possibly being written, run imod wait which throws an error if not growing. - BH_imodWait(stack) - else - - error('Did not find the full aligned stack at %s\n',stack); - - end - - else - - fprintf('Reconstructing from cached stack %s\n', stack); - - end - - header = getHeader(MRCImage(stack)); - - if exist(recon,'file') - fprintf('Using cached file %s\n', recon); - [ reconGeom ] = calc_rg( header, rCoords ); - else - - - % Check that no out of bounds occur on slices - - if (rCoords(2) == 0) - fprintf('shifting slices up 1\n') - rCoords(2) = 1; - end - if (rCoords(3) > header.nY) - if (header.nY - rCoords(3) > 2) - fprintf('Slice index is too high.\n') - else - rCoords(3) = header.nY-1; - end - end - - [ reconGeom ] = calc_rg( header, rCoords ); - - rCMD = sprintf(['-input %s -output %s -TILTFILE %s -UseGPU %d ', ... - '-WIDTH %d -SLICE %d,%d -THICKNESS %d -SHIFT %f,%f '],... - stack, recon, TLT, gpuIDX, rCoords(1:6)); - - - % Explicitly set Radial to Nyquist - if (flgLocal) - rCMD = [rCMD sprintf('-LOCALFILE %s -RotateBy90 -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%f',LOCAL,reconScaling)]; - else - rCMD = [rCMD sprintf('-RotateBy90 -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%f',reconScaling)]; - end - - if system('which tilt') - error('Did not find IMOD tilt funciton on path') - else - fprintf('Reconstructing from newly cached stack %s\n', stack); - fprintf('tilt %s %s %s\n',rCMD,super_sample,expand_lines) - system(sprintf('tilt %s %s %s',rCMD,super_sample,expand_lines)); - end - - end else error('An appropriate reconstruction was not found for %s\n',recon) - end if (strcmpi(recon,'tomoCPR')) @@ -199,24 +86,4 @@ end end - - - function [ reconGeom ] = calc_rg( header, rCoords ) - - % Origin in the binned tilt series - oY = 1 + floor(header.nY ./ 2); - % Size in the binned tilt series - nY = rCoords(3) - rCoords(2) + 1; - % Origin in the reconstructed area (active shift from origin in tilt series) - dY = floor(rCoords(2) + nY/2) - oY; - reconGeom = zeros(2,3); - % FIXME index 4 and 5 should be 3 and 4 - reconGeom(1,1:3) = [rCoords(1), nY, rCoords(4)]; - % value specify location of origin, but SHIFT in IMOD's tilt takes the - % location to shift the origin too, so multiply oX by -1. The notion for Z is - % flipped since imod does reconstruction on a rotated frame. I.e. a positive - % number shifts the recon "up" which when rotated to the microscope frame is - % actually "down" (in Z) so no need to multipliy oZ by -1. - reconGeom(2,1:3) = round([-1*rCoords(5),dY,rCoords(6)]); - end end diff --git a/transformations/emc_get_max_specimen_NZ.m b/transformations/emc_get_max_specimen_NZ.m index f30237f5..98f3e456 100644 --- a/transformations/emc_get_max_specimen_NZ.m +++ b/transformations/emc_get_max_specimen_NZ.m @@ -1,27 +1,41 @@ -function [ max_specimen_nz, tomoNumber ] = emc_get_max_specimen_NZ(subTomoMeta_tomoName, subTomoMeta_coords, tomo_name_list, n_tomograms, samplingRate) +function [ max_specimen_nz, tomoIdx ] = emc_get_max_specimen_NZ(subTomoMeta_tomoName, subTomoMeta_coords, tomo_name_list, n_tomograms, samplingRate) max_z_value = -inf; min_z_value = inf; - tomoNumber = zeros(n_tomograms,1); + tomoIdx = zeros(n_tomograms,1); for iTomo = 1:n_tomograms % subTomoMeta_tomoName = subTomoMeta.mapBackGeometry.tomoName % subTomoMeta_coords = subTomoMeta.mapBackGeometry.(tiltName).coords if isa(subTomoMeta_tomoName,'struct') - tomoNumber(iTomo) = subTomoMeta_tomoName.(tomo_name_list{iTomo}).tomoNumber; + tomoIdx(iTomo) = subTomoMeta_tomoName.(tomo_name_list{iTomo}).tomoIdx; else - tomoNumber(iTomo) = iTomo; + tomoIdx(iTomo) = iTomo; end % 4 is the unbinned pixel size of the tomogram in Z % 6 is location of the specimen origin in Z relative to the origin of the tomogram - nZ = subTomoMeta_coords(tomoNumber(iTomo),4) ./ samplingRate; - oZ = subTomoMeta_coords(tomoNumber(iTomo),6) ./ samplingRate; - - % We need to consider the shift of the tomogram relative to the specimen origin, - % which is -oZ - dZ = -oZ; + if isa(subTomoMeta_coords,'cell') + nZ = subTomoMeta_coords{iTomo}.NZ; + dZ = subTomoMeta_coords{iTomo}.dZ_specimen_to_tomo; + else + + if isfield(subTomoMeta_coords, 'NZ') + nZ = subTomoMeta_coords.NZ; + dZ = subTomoMeta_coords.dZ_specimen_to_tomo; + else + if isfield(subTomoMeta_coords, tomo_name_list{iTomo}) + nZ = subTomoMeta_coords.(tomo_name_list{iTomo}).NZ; + dZ = subTomoMeta_coords.(tomo_name_list{iTomo}).dZ_specimen_to_tomo; + else + error('The field NZ or the field %s is not present in the subTomoMeta_coords', tomo_name_list{iTomo}); + end + end + end + + NZ = NZ ./ samplingRate; + dZ = dZ ./ samplingRate; if (dZ + nZ / 2 > max_z_value) max_z_value = dZ + nZ / 2; diff --git a/transformations/emc_get_origin_index.m b/transformations/emc_get_origin_index.m index ca904dd8..a8b9db56 100644 --- a/transformations/emc_get_origin_index.m +++ b/transformations/emc_get_origin_index.m @@ -1,4 +1,4 @@ -function [ origin ] = emc_get_origin_index( obj ) +function [ origin_index ] = emc_get_origin_index( obj ) if isempty(obj) error('Input must not be empty'); @@ -6,7 +6,9 @@ if ~isnumeric(obj) error('Input must be a numeric object'); end - - origin = floor(obj/2) + 1; - + if any(obj <= 0) + error('Input must be a positive number'); + end + + origin_index = floor(obj/2) + 1; end \ No newline at end of file From 8bb68df9f037d815bd1bb342a2a686982c455418 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 22 Feb 2024 18:20:28 -0500 Subject: [PATCH 039/151] working for init and ctf3d, though the newstack op still is inverted --- coordinates/BH_multi_recGeom.m | 31 ++++---- ctf/BH_ctf_Correct3d.m | 90 +++++++++++------------ metaData/BH_geometryInitialize.m | 59 +++++++-------- metaData/BH_parseParameterFile.m | 2 +- metaData/BH_returnIncludedTilts.m | 2 +- transformations/emc_get_max_specimen_NZ.m | 2 +- 6 files changed, 87 insertions(+), 99 deletions(-) diff --git a/coordinates/BH_multi_recGeom.m b/coordinates/BH_multi_recGeom.m index 07cfa9bd..47f46a0d 100755 --- a/coordinates/BH_multi_recGeom.m +++ b/coordinates/BH_multi_recGeom.m @@ -19,7 +19,7 @@ tilt_geometry_name = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', tiltName, mapBackIter+1); try - tilt_geometry_name = load(tilt_geometry_name); + tilt_geometry = load(tilt_geometry_name); catch error('Could not load the tilt geometry file: %s', tilt_geometry_name); end @@ -39,7 +39,8 @@ error('the tomo base name (%s) does not match the tiltName in the coords file (%s)',tiltNameFromTomo,tiltName); end % The tilt_geometry should have rows that are a multiple of 26 (> 26 means more than one orientation per peak) -if mod(size(tilt_geometry_name,2),26) ~= 0 +if mod(size(tilt_geometry,2),26) ~= 0 && mod(size(tilt_geometry,2),23) ~= 0 + tilt_geometry error('The tilt geometry file does not have a multiple of 26 entries/row'); end @@ -48,38 +49,38 @@ % in geometryAnalysis or if there are zero sub-tomos left. recGeom = cell(nTomos,1); for iTomo = 1:nTomos - read_in_Coords = (1 + (iTomo-1)*6: 6 + (iTomo-1)*6); + read_in_Coords = recCoords(1 + (iTomo-1)*6: 6 + (iTomo-1)*6); tomoName = sprintf('%s_%d',tiltName, iTomo); % Check to make sure no out of bounds conditions were created in X Y % when the user created the model or point file % TODO: only checking Y b/c that results in a crash. Checking X would probably make sense too. if read_in_Coords(2) < -75 - error(sprintf('Out of bounds condition for %s yMin at %f, 'please change recon.txt recon.coords', tomoName, read_in_Coords(2))) + error(sprintf('Out of bounds condition for %s yMin at %f, please change recon.txt recon.coords', tomoName, read_in_Coords(2))) elseif read_in_Coords(2) < 1 % If not too extreme, just clamp it to 1 read_in_Coords(2) = 1; end yMax = tilt_geometry(1,21); if read_in_Coords(3) > yMax + 75 - error(sprintf('Out of bounds condition for %s yMin at %f, 'please change recon.txt recon.coords', tomoName, read_in_Coords(3), yMax)) + error(sprintf('Out of bounds condition for %s ymin at %f ymax at %f, please change recon.txt recon.coords', tomoName, read_in_Coords(3), yMax)) elseif read_in_Coords(3) > yMax read_in_Coords(3) = yMax; end tomoCoords = struct(); - tomoCoords.('y_i') = int32(read_in_Coords(2)); - tomoCoords.('y_f') = int32(read_in_Coords(3)); - tomoCoords.('NX') = int32(read_in_Coords(tomoIdx, 1)); - tomoCoords.('NY') = int32(read_in_Coords(tomoIdx, 3) - read_in_Coords(tomoIdx, 2) + 1); - tomoCoords.('NZ') = int32(read_in_Coords(tomoIdx, 4)); - tomoCoords.('dX_specimen_to_tomo') = -1*read_in_Coords(tomoIdx, 5); - tomoCoords.('dY_specimen_to_tomo') = + tomoCoords.('y_i') = (read_in_Coords(2)); + tomoCoords.('y_f') = (read_in_Coords(3)); + tomoCoords.('NX') = (read_in_Coords(1)); + tomoCoords.('NY') = (read_in_Coords(3) - read_in_Coords( 2) + 1); + tomoCoords.('NZ') = (read_in_Coords(4)); + tomoCoords.('dX_specimen_to_tomo') = -1*read_in_Coords(5); + tomoCoords.('dY_specimen_to_tomo') = ... (emc_get_origin_index(tomoCoords.('NY')) ... - + read_in_Coords(tomoIdx, 2)) ... % origin of the tomogram in the full tilt projection - - emc_get_origin_index(tilt_geometry(1,21)), ... % less the origin of the tilt projection = origin of the tomogram wrt the tilt projection - tomoCoords.('dZ_specimen_to_tomo') = read_in_Coords(tomoIdx, 6); + + read_in_Coords(2)) ... % origin of the tomogram in the full tilt projection + - emc_get_origin_index(tilt_geometry(1,21)); + tomoCoords.('dZ_specimen_to_tomo') = read_in_Coords( 6); % Check that NX, NY, NZ are all positive if tomoCoords.('NX') <= 0 diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 5e55734c..13dda6e3 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -18,7 +18,6 @@ subTomoMeta = struct(); resTarget = 15; -use_inverted_newstack = false; % TODO remove thise params tiltWeight = [0.2,0]; @@ -169,7 +168,6 @@ else load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; - subTomoMeta = subTomoMeta; clear subTomoMeta CYCLE = subTomoMeta.currentCycle; end @@ -255,7 +253,7 @@ nTomosPerTilt = 0; recGeom = 0; - +tiltRecGeom = 0; if (recon_for_subTomo) [tiltList,nTilts] = BH_returnIncludedTilts(subTomoMeta.mapBackGeometry); tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); @@ -326,8 +324,8 @@ end parfor iGPU = 1:nGPUs -% for iGPU = 1:nGPUs - +% for iGPU = 1:nGPUs %%revert + for iTilt = iterList{gpuList(iGPU)} iTomoList = {}; @@ -387,7 +385,7 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. parfor iGPU = 1:nGPUs -% for iGPU = 1:nGPUs +% for iGPU = 1:nGPUs %%revert % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); @@ -395,6 +393,10 @@ for iTilt = iterList{gpuList(iGPU)} slab_list = {}; + % if ~strcmp(tiltList{iTilt},'TS_121') + % continue + % % revert + % end if (recon_for_templateMatching) % templaterch @@ -407,7 +409,7 @@ % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; iCoords = cell(nTomos,1); for iCoordIdx = 1:nTomos - iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iCoordIdx}); % FIXME end % else @@ -417,7 +419,6 @@ % end end - iTomoList = cell(nTomos,1); @@ -458,6 +459,7 @@ continue end end + if samplingRate > 1 fullStack = sprintf('%aliStacks/%s_ali%d.fixed', ... @@ -494,9 +496,9 @@ iCoords{iCoordIdx}.tomoCoords.dX_specimen_to_tomo = 0; iCoords{iCoordIdx}.tomoCoords.dY_specimen_to_tomo = 0; iCoords{iCoordIdx}.tomoCoords.dZ_specimen_to_tomo = 0; - iCoords{iCoordIdx}.tomoCoords.NX = NX; - iCoords{iCoordIdx}.tomoCoords.NY = NY-1; - iCoords{iCoordIdx}.tomoCoords.NZ = NZ; + iCoords{iCoordIdx}.tomoCoords.NX = NX * samplingRate; + iCoords{iCoordIdx}.tomoCoords.NY = NY * samplingRate; + iCoords{iCoordIdx}.tomoCoords.NZ = NZ * samplingRate; end else [ ~, specimen_NX_nm, tomoIdx, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... @@ -513,6 +515,7 @@ % TODO: for very thick specimen, this may be preventing the avg from getting to high enough % resolution to be useful. So far, this is only optimized on in vitro samples. dampeningMax = 0.90; + [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... TLT(1,17),TLT(1,18),abs(TLT(1,15)), ... 2048, TLT(1,19), ... @@ -536,7 +539,7 @@ % For each tomo create a list of slices that are to be reconstructed % for every section section. - [ slab_list ] = calc_slab_boundaries(iCoords, tomoIdx, emc.pixel_size_angstroms, n_slabs_to_reconstruct, tiltList{iTilt}, ctf3dDepth, samplingRate, use_inverted_newstack); + [ slab_list ] = calc_slab_boundaries(iCoords, tomoIdx, emc.pixel_size_angstroms, n_slabs_to_reconstruct, tiltList{iTilt}, ctf3dDepth, samplingRate); if (recon_for_subTomo) @@ -570,6 +573,7 @@ end end + if (PosControl2d) correctedStack = maskedStack; else @@ -607,11 +611,9 @@ thisTomo = tomoIdx(iT); if (slab_list{iT}(iSection,1)) - reconName = sprintf('%s/%s_ali%d_%d_%d.rec', ... tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); - - + if (recon_for_tomoCPR) TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{1}),1); end @@ -650,20 +652,16 @@ flgLocal = 0; end - slices_in_y = floor(iCoords{thisTomo}.tomoCoords.NY ./ samplingRate); % round down and then we'll add any extra needed to the final chunk - tiltChunkSize = floor(slices_in_y / emc.n_tilt_workers); + tiltChunkSize = ceil(iCoords{thisTomo}.NY ./ samplingRate ./ emc.n_tilt_workers); % This shoulid never happen, but to be safe - if (emc.n_tilt_workers > iCoords{thisTomo}.tomoCoords.NY) + if (emc.n_tilt_workers > iCoords{thisTomo}.NY) error('n_tilt_workers is greater than the number of slices in the tilt series'); end - y_i = floor(iCoords{thisTomo}.tomoCoords.y_i ./ samplingRate); - y_f = floor(iCoords{thisTomo}.tomoCoords.y_f ./ samplingRate); - if (slices_in_y ~= y_f - y_i + 1) - fprintf('slices_in_y is %d\n',slices_in_y); - fprintf('y_f - y_i + 1 is %d\n',y_f - y_i + 1); - error('slices_in_y does not match y_f - y_i + 1'); - end + y_i = floor(iCoords{thisTomo}.y_i ./ samplingRate); + y_f = floor(iCoords{thisTomo}.y_f ./ samplingRate); + + tiltChunks = y_i:tiltChunkSize:y_f; tiltChunks(end) = y_f; % Imod expects zero indexed slices @@ -676,7 +674,7 @@ n_slices_in_Y = tiltChunks(end) - tiltChunks(1) + 1; end - + rCMD = sprintf(['tilt %s %s -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... super_sample, ... @@ -685,9 +683,9 @@ reconName, ... rawTLT, ... gpuList(iGPU), ... - floor(iCoords{thisTomo}.tomoCoords.NX ./ samplingRate),... % WIDTH = NX + floor(iCoords{thisTomo}.NX ./ samplingRate),... % WIDTH = NX floor(round(slab_list{iT}(iSection,5))), ... % THICKNESS = NZ - floor(iCoords{thisTomo}.tomoCoords.dX_specimen_to_tomo ./ samplingRate), ... % SHIFT X + floor(iCoords{thisTomo}.dX_specimen_to_tomo ./ samplingRate), ... % SHIFT X slab_list{iT}(iSection,6)); @@ -790,14 +788,15 @@ fprintf(recombineCMD,'%d\n', n_total_sections); cleanup3 = sprintf('rm %s',file_of_outputs); - if (use_inverted_newstack) - slab_order = n_total_sections:-1:1; - else - slab_order = 1:n_total_sections; - end + % if (use_inverted_newstack) + slab_order = n_slabs_to_reconstruct:-1:1; + % else + % slab_order = 1:n_slabs_to_reconstruct; + % end % for iSection = 1:n_slabs_to_reconstruct for iSection = slab_order - if(slab_list{iT}(iSection,1)) + + if (slab_list{iT}(iSection,1)) this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, thisTomo, iSection); cleanup3 = sprintf('%s %s',cleanup3,this_slab); fprintf(recombineCMD, '%s\n', this_slab); @@ -806,8 +805,8 @@ end fclose(recombineCMD); pause(1); - recCMD = sprintf('newstack -fromone -FileOfInputs %s -output %s\n', file_of_outputs, reconNameFull) - + recCMD = sprintf('newstack -fromone -FileOfInputs %s -output %s\n', file_of_outputs, reconNameFull); + [err_msg, ~] = system(sprintf('%s > /dev/null ',recCMD)); %/dev/null if (err_msg) fprintf('error during recombination %s\n',reconNameFull); @@ -884,13 +883,13 @@ for iT = 1:length(tomoIdx) iTomo = tomoIdx(iT); % Origin + originshift - tomo_origin_wrt_tilt_origin = iCoords.dZ_specimen_to_tomo; - tomo_origin_in_tomo_frame = emc_get_origin_index(iCoords.NZ); + tomo_origin_wrt_tilt_origin = iCoords{iT}.dZ_specimen_to_tomo ./ samplingRate; + tomo_origin_in_tomo_frame = emc_get_origin_index(iCoords{iT}.NZ ./ samplingRate); - fraction_origin_shift = tomo_origin_wrt_tilt_origin - round(tomogram_origin_wrt_specimen_frame); + fraction_origin_shift = tomo_origin_wrt_tilt_origin - round(tomo_origin_wrt_tilt_origin); - tomogram_lower_bound = round(tomo_origin_wrt_tilt_origin) - tomo_origin_in_tomo_frame; - recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + iCoords.NZ - 1; + tomogram_lower_bound = floor((tomo_origin_wrt_tilt_origin - tomo_origin_in_tomo_frame)); + recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + ceil(iCoords{iT}.NZ./samplingRate) - 1; % For each slab see if this tomogram has any sections in it for iSlab = 1:n_slabs_to_reconstruct @@ -910,13 +909,8 @@ continue; end valid_region_origin = emc_get_origin_index(slab_list{iT}(iSlab,5)); - % FIXME: this is an important departure from what I undrestood DM's convention to be. - % The easiest way to test this will be to do a sub region selection that is very clear what "up" means. - % This should produce the correct Z shift, but it requires inverting the order of the output stacks dZ_for_reconstructed_slab = -(valid_indices(valid_region_origin) + fraction_origin_shift); - if ~(use_inverted_newstack) - dZ_for_reconstructed_slab = -dZ_for_reconstructed_slab; - end + slab_list{iT}(iSlab,6) = dZ_for_reconstructed_slab; %dZ end @@ -942,7 +936,7 @@ % TroubleShoot tSHT = fopen(sprintf('.tblSht_%s_i%d.txt',tiltName,iTomo),'w'); - fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords(iTomo,:)'); + fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords{iT}.NX, iCoords{iT}.NY, iCoords{iT}.NZ, iCoords{iT}.dX_specimen_to_tomo, iCoords{iT}.dY_specimen_to_tomo, iCoords{iT}.dZ_specimen_to_tomo); fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', slab_list{iT}'); fclose(tSHT); end % end loop over tomos diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index 1ff39084..d2250c86 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -136,7 +136,7 @@ nTomogramsTotal = length(getPath); fileInfo = cell(nTomogramsTotal,4); - +n_tomos_added = 1; getCoords = dir('recon/*.coords'); nStacks = length(getCoords); @@ -159,7 +159,6 @@ end nTomos = length(iPath); - fprintf('nTomos %d by mods, nTomosPossible %d, by coords\n',nTomos,nTomosPossible); if nTomos > nTomosPossible error('The number of model files in convmap/*.mod is greater than the number in the recon/*.coords\n'); elseif nTomos < nTomosPossible @@ -168,7 +167,7 @@ end subTomoMeta.('mapBackGeometry').(tiltName).('nTomos') = nTomos; subTomoMeta.('mapBackGeometry').(tiltName).('tomoCprRePrjSize') = 512; - + for iTomo = 1:nTomos if (doImport) @@ -181,26 +180,27 @@ end % We are storing this info to make it available when checking for duplicates - fileInfo{iTomo,1} = tiltName; - fileInfo{iTomo,2} = sprintf('%s_%d', tiltName, tomoIdx); - fileInfo{iTomo,3} = sprintf('%s_%d_bin%d',tiltName, tomoIdx, dupSampling); + fileInfo{n_tomos_added,1} = tiltName; + fileInfo{n_tomos_added,2} = sprintf('%s_%d', tiltName, tomoIdx); + fileInfo{n_tomos_added,3} = sprintf('%s_%d_bin%d',tiltName, tomoIdx, dupSampling); tomoName = sprintf('%s_%d',tiltName,tomoIdx); - subTomoMeta.('tiltGeometry').(fileInfo{iTomo,2}) = tilt_geometry; + subTomoMeta.('tiltGeometry').(fileInfo{n_tomos_added,2}) = tilt_geometry; + n_tomos_added = n_tomos_added + 1; % Store a reference to the parent tilt-series for every tomogram subTomoMeta.('mapBackGeometry').('tomoName').(tomoName).('tiltName') = tiltName; % Store the tomoIdx for every tomogram, currently used to refer back to recGEom, but I'm going to put this into a struct - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('tomoIdx') = tomoIdx; + subTomoMeta.('mapBackGeometry').('tomoName').(tomoName).('tomoIdx') = tomoIdx; subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('is_active') = true; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_i') = recGeom{tomoIdx}.tomoCoords.y_i; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_f') = recGeom{tomoIdx}.tomoCoords.y_f; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NX') = recGeom{tomoIdx}.tomoCoords.NX; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NY') = recGeom{tomoIdx}.tomoCoords.NY; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NZ') = recGeom{tomoIdx}.tomoCoords.NZ; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dY_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dY_specimen_to_tomo; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dZ_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dZ_specimen_to_tomo; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_i') = recGeom{tomoIdx}.y_i; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_f') = recGeom{tomoIdx}.y_f; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NX') = recGeom{tomoIdx}.NX; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NY') = recGeom{tomoIdx}.NY; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NZ') = recGeom{tomoIdx}.NZ; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.dX_specimen_to_tomo; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dY_specimen_to_tomo') = recGeom{tomoIdx}.dY_specimen_to_tomo; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dZ_specimen_to_tomo') = recGeom{tomoIdx}.dZ_specimen_to_tomo; end end % end of loop over stacks @@ -208,7 +208,7 @@ % suffix - generalize later. if nGPUs > nTomogramsTotal - nGPUs = nTomogramsTotal + nGPUs = nTomogramsTotal; end @@ -229,7 +229,7 @@ dupInTheLoop = dupSampling parfor iGPU = 1:nGPUs - % % % for iGPU = 1:nGPUs + % for iGPU = 1:nGPUs % revert D = gpuDevice(iGPU); @@ -244,6 +244,11 @@ % Load in the template matching geometry for the tomogram, and the model file % which may (or may not) have been edited. + if isempty(fileInfo{iTomo,2}) + fprintf("\n\tWarning, missing tomoname for fileInfo{%d,2}\n",iTomo); + pause(1) + continue + end tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(fileInfo{iTomo,2}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(fileInfo{iTomo,2}).tiltName; try @@ -259,15 +264,10 @@ tmpSearchGeom(:,[11:16]) = tmpCSV; for iAng = 1:size(tmpSearchGeom,1) - - angleSgn - convention - direction tmpSearchGeom(iAng,17:25) = reshape( ... BH_defineMatrix( ... angleSgn .* tmpSearchGeom(iAng,14:16),... convention,direction),1,9); - end % I'm assuming that the proper scaling was done, let the user know @@ -312,7 +312,6 @@ iHeader = getHeader(MRCImage(tiltName)); % first convert the imod model file to a temporary text file tmpFile = sprintf('tmp_%d.txt',iGPU); - sprintf('model2point convmap/%s.mod %s', mapName, tmpFile) system(sprintf('model2point convmap/%s.mod %s', mapName, tmpFile)) modGeom = load(tmpFile); system(sprintf('rm %s', tmpFile)); @@ -365,13 +364,8 @@ overlapMatrix = convn(positionMatrix, gpuArray(dupMask), 'same'); idxLiTomo = positionIDX((overlapMatrix > 1)); - size(overlapMatrix) - size(positionIDX) - - - tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom(ismember(tmpSearchGeom(:,4), idxLiTomo),:) - sum(idxLiTomo(:)) - sum(ismember(tmpSearchGeom(:,4), idxLiTomo)) + + tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom(ismember(tmpSearchGeom(:,4), idxLiTomo),:); else tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom; @@ -381,7 +375,6 @@ end parResults{iGPU} = tomoResults; - end % By my convention, every particle should have a unique id. @@ -400,7 +393,7 @@ for iSubTomo = 1:size(tmpGeom,1) tmpGeom(iSubTomo, 4:26:26*emc.nPeaks) = nIDX; - nIDX = nIDX +1; + nIDX = nIDX + 1; end subTomoMeta.('cycle000').('geometry').(mapName) = tmpGeom; diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 2f350e82..bad1d9ee 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -585,7 +585,7 @@ end if isfield(emc, 'max_ctf3dDepth') - EMC_assert_numeric(emc.max_ctf3dDepth, 1, [1 * 10^-9 || 1000 * 10^-9]); + EMC_assert_numeric(emc.max_ctf3dDepth, 1, [1 * 10^-9, 1000 * 10^-9]); else emc.max_ctf3dDepth = 100*10^-9; end diff --git a/metaData/BH_returnIncludedTilts.m b/metaData/BH_returnIncludedTilts.m index 831b9565..d193277e 100644 --- a/metaData/BH_returnIncludedTilts.m +++ b/metaData/BH_returnIncludedTilts.m @@ -3,7 +3,7 @@ % Detailed explanation goes here STACK_LIST_tmp = fieldnames(mapBackGeom); -STACK_LIST_tmp = STACK_LIST_tmp(~ismember(STACK_LIST_tmp,{'tomoName','viewGroups'})); +STACK_LIST_tmp = STACK_LIST_tmp(~ismember(STACK_LIST_tmp,{'tomoName','tomoCoords'})); tiltList = cell(length(STACK_LIST_tmp),1); nTilts = 0; diff --git a/transformations/emc_get_max_specimen_NZ.m b/transformations/emc_get_max_specimen_NZ.m index 98f3e456..5fde9fdf 100644 --- a/transformations/emc_get_max_specimen_NZ.m +++ b/transformations/emc_get_max_specimen_NZ.m @@ -34,7 +34,7 @@ end end - NZ = NZ ./ samplingRate; + nZ = nZ ./ samplingRate; dZ = dZ ./ samplingRate; if (dZ + nZ / 2 > max_z_value) From 79b0dfb9022e69ddd8afad356a4b13f554766c40 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 22 Feb 2024 18:59:15 -0500 Subject: [PATCH 040/151] well --- alignment/BH_alignRaw3d_v2.m | 42 ++++++++++++++++------------------ coordinates/BH_multi_recGeom.m | 4 ++-- ctf/BH_ctf_Correct3d.m | 27 +++++++++++----------- metaData/BH_removeDuplicates.m | 2 +- transformations/BH_average3d.m | 35 ++++++++++++---------------- 5 files changed, 51 insertions(+), 59 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 15080d5f..93b3dba7 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -137,7 +137,6 @@ % % % pathList= subTomoMeta.mapPath; % % % extList = subTomoMeta.mapExt; -masterTM = subTomoMeta; clear subTomoMeta refVector = cell(2,1); refGroup = cell(2,1); @@ -167,7 +166,7 @@ % Get the number of tomograms to process. tomoList = fieldnames(geometry); nTomograms = length(tomoList); -tiltList = masterTM.tiltGeometry; +tiltList = subTomoMeta.tiltGeometry; % Sort the list by number of active subtomos to improve parallelism sortedTomoList = zeros(nTomograms,1); @@ -236,20 +235,20 @@ weightNAME = sprintf('class_%d_Locations_Ref_%s_Wgt', refName, halfSet); - imgCounts{iGold} = masterTM.(cycleNumber).(imgNAME){3}; + imgCounts{iGold} = subTomoMeta.(cycleNumber).(imgNAME){3}; [ refTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(imgNAME){1}, ... - masterTM.(cycleNumber).(imgNAME){2},... + subTomoMeta.(cycleNumber).(imgNAME){1}, ... + subTomoMeta.(cycleNumber).(imgNAME){2},... sizeWindow); [ wdgTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... - masterTM.(cycleNumber).(weightNAME){1},... - masterTM.(cycleNumber).(weightNAME){2},... + subTomoMeta.(cycleNumber).(weightNAME){1},... + subTomoMeta.(cycleNumber).(weightNAME){2},... sizeCalc); - sizeREF = masterTM.(cycleNumber).(imgNAME){2}{1}(2:2:6)'; + sizeREF = subTomoMeta.(cycleNumber).(imgNAME){2}{1}(2:2:6)'; if (emc.move_reference_by_com) % % % % % % % [ comMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); @@ -354,7 +353,7 @@ if (emc.classification || emc.multi_reference_alignment) for iRef = 1:nReferences(1) - fscINFO = masterTM.(cycleNumber).('fitFSC').(sprintf('Ref%d',iRef)); + fscINFO = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Ref%d',iRef)); [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... 'GPU', {'none'}, 1, 0, 1 ); @@ -375,7 +374,7 @@ for iRef = 1 - fscINFO = masterTM.(cycleNumber).('fitFSC').('Ref1'); + fscINFO = subTomoMeta.(cycleNumber).('fitFSC').('Ref1'); [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... 'GPU', {'none'}, 1, 0, 1 ); radialGrid = single(radialGrid./emc.pixel_size_angstroms); @@ -544,7 +543,7 @@ end -% [masterTM] = BH_recordAngularSampling( masterTM, cycleNumber, angleStep, inPlaneSearch); +% [subTomoMeta] = BH_recordAngularSampling( subTomoMeta, cycleNumber, angleStep, inPlaneSearch); nCount = 1; @@ -682,29 +681,29 @@ % Load the tomo into gpu tomoName = tomoList{iTomo}; - tiltGeometry = masterTM.tiltGeometry.(tomoList{iTomo}); + tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); - tomoIdx = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; binShift = [0,0,0]; nSubTomos = size(positionList,1); - iTiltName = masterTM.mapBackGeometry.tomoName.(tomoName).tiltName; + iTiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoName).tiltName; % Can't clear inside the parfor, but make sure we don't have two tomograms % in memory at once. - tomoIdx = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; - tiltName = masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; reconCoords = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); - TLT = masterTM.('tiltGeometry').(tomoList{iTomo}); + TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); if (emc.flgCutOutVolumes) volumeData = []; @@ -1551,12 +1550,11 @@ save('bestAngles.mat', 'bestAngles'); [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, emc.nPeaks, rotConvention, updateWeights, emc.update_class_by_ccc); - masterTM.(cycleNumber).('RawAlign') = rawAlign; - masterTM.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); - masterTM.('updatedWeights') = true; + subTomoMeta.(cycleNumber).('RawAlign') = rawAlign; + subTomoMeta.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); + subTomoMeta.('updatedWeights') = true; clear bestAngles rawAlign - subTomoMeta = masterTM; save(emc.('subTomoMeta'), 'subTomoMeta'); end diff --git a/coordinates/BH_multi_recGeom.m b/coordinates/BH_multi_recGeom.m index 47f46a0d..5c64326b 100755 --- a/coordinates/BH_multi_recGeom.m +++ b/coordinates/BH_multi_recGeom.m @@ -75,12 +75,12 @@ tomoCoords.('NX') = (read_in_Coords(1)); tomoCoords.('NY') = (read_in_Coords(3) - read_in_Coords( 2) + 1); tomoCoords.('NZ') = (read_in_Coords(4)); - tomoCoords.('dX_specimen_to_tomo') = -1*read_in_Coords(5); + tomoCoords.('dX_specimen_to_tomo') = read_in_Coords(5); tomoCoords.('dY_specimen_to_tomo') = ... (emc_get_origin_index(tomoCoords.('NY')) ... + read_in_Coords(2)) ... % origin of the tomogram in the full tilt projection - emc_get_origin_index(tilt_geometry(1,21)); - tomoCoords.('dZ_specimen_to_tomo') = read_in_Coords( 6); + tomoCoords.('dZ_specimen_to_tomo') = read_in_Coords(6); % Check that NX, NY, NZ are all positive if tomoCoords.('NX') <= 0 diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 13dda6e3..50956762 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -316,15 +316,15 @@ iterList{gpuList(iGPU)} = iGPU+(tiltStart-1):nGPUs:nTilts; iterList{gpuList(iGPU)}; end -try - EMC_parpool(nGPUs) -catch - delete(gcp('nocreate')) - EMC_parpool(nGPUs) -end - -parfor iGPU = 1:nGPUs -% for iGPU = 1:nGPUs %%revert +% try +% EMC_parpool(nGPUs) +% catch +% delete(gcp('nocreate')) +% EMC_parpool(nGPUs) +% end + +% parfor iGPU = 1:nGPUs +for iGPU = 1:nGPUs %%revert for iTilt = iterList{gpuList(iGPU)} @@ -384,8 +384,8 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. -parfor iGPU = 1:nGPUs -% for iGPU = 1:nGPUs %%revert +% parfor iGPU = 1:nGPUs +for iGPU = 1:nGPUs %%revert % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); @@ -789,13 +789,12 @@ cleanup3 = sprintf('rm %s',file_of_outputs); % if (use_inverted_newstack) - slab_order = n_slabs_to_reconstruct:-1:1; + % slab_order = n_slabs_to_reconstruct:-1:1; % else - % slab_order = 1:n_slabs_to_reconstruct; + slab_order = 1:n_slabs_to_reconstruct; % end % for iSection = 1:n_slabs_to_reconstruct for iSection = slab_order - if (slab_list{iT}(iSection,1)) this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, thisTomo, iSection); cleanup3 = sprintf('%s %s',cleanup3,this_slab); diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index 50df3b6f..ec339f0b 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -62,7 +62,7 @@ tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; tomoName = sprintf('%s_%d',tiltName,tomoIdx); - reconGeometry = masterTM.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); % iHeader = getHeader(MRCImage(tomoName)); clear postionMatrix positionIDX diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 517ba2c0..723f437a 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -644,7 +644,7 @@ parVect = 1:nParProcesses; parfor iParProc = parVect - % for iParProc = parVect + % for iParProc = parVect % r % Get the gpuIDX assigned to this process gpuIDXList = mod(parVect+emc.nGPUs,emc.nGPUs)+1; @@ -727,7 +727,7 @@ tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconGeometry = masterTM.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); [ binShift ] = [0,0,0];%BH_multi_calcBinShift( coords, samplingRate); % Load in the geometry for the tomogram, and get number of subTomos. @@ -739,9 +739,6 @@ volumeData = []; %fprintf('loading tomo %d\n',iTomo); - tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; - tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) volumeData = []; @@ -859,7 +856,7 @@ TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); if (make_sf3d) - [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center, reconGeometry, emc.wiener_constant); + [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center.*samplingRate, reconGeometry, emc.wiener_constant); make_sf3d = false; end @@ -1458,7 +1455,6 @@ if ~( flgEstSNR ) load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - masterTM = subTomoMeta; %%%%%%%%%%%%%55 Reweight now that the FSC is calculated for iGold = 1:2 @@ -1473,13 +1469,13 @@ className, fieldPrefix, halfSet); [ refIMG{iGold} ] = BH_unStackMontage4d(1:maxClasses, ... - masterTM.(cycleNumber).(imgIN){1}, ... - masterTM.(cycleNumber).(imgIN){2},... + subTomoMeta.(cycleNumber).(imgIN){1}, ... + subTomoMeta.(cycleNumber).(imgIN){2},... sizeWindow); [ refWGT{iGold} ] = BH_unStackMontage4d(1:maxClasses, ... - masterTM.(cycleNumber).(wgtIN){1},... - masterTM.(cycleNumber).(wgtIN){2},... + subTomoMeta.(cycleNumber).(wgtIN){1},... + subTomoMeta.(cycleNumber).(wgtIN){2},... sizeCalc); end @@ -1519,14 +1515,14 @@ end try - fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRefPrev)); - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); - mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); + fscParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('%s%d',savePrefix,iRefPrev)); + aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d',savePrefix,iRefPrev)); + mskParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Mask%s%d',savePrefix,iRefPrev)); catch fprintf('\nReverting from %s to REf in loading fitFSC\n',savePrefix); - fscParams = masterTM.(cycleNumber).('fitFSC').(sprintf('%s%d','Ref',iRefPrev)); - aliParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)); - mskParams = masterTM.(cycleNumber).('fitFSC').(sprintf('Mask%s%d','Ref',iRefPrev)); + fscParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('%s%d','Ref',iRefPrev)); + aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)); + mskParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Mask%s%d','Ref',iRefPrev)); end iOdd = iRef; iEve = iRef; @@ -1571,7 +1567,7 @@ end imgIN = sprintf('class_%d_Locations_%s_%s_NoWgt', ... className, fieldPrefix, halfSet); - imgCounts = masterTM.(cycleNumber).(imgIN){3}; + imgCounts = subTomoMeta.(cycleNumber).(imgIN){3}; % Save the unweighted, weighted imgs, weightes, optionally filtered. if (flgFinalAvg) @@ -1588,14 +1584,13 @@ className, fieldPrefix, halfSet); classOut = sprintf('class_%d_Locations_%s_%s', className,fieldPrefix, halfSet); - masterTM.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; SAVE_IMG(montOUT, imout, emc.pixel_size_angstroms); end %%%%%%% end - subTomoMeta = masterTM; subTomoMeta.('CUTPADDING') = emc.CUTPADDING; if (emc.flgCutOutVolumes && volumesNeedToBeExtracted) subTomoMeta.('volumesAreCutOut') = 1; From 91e42c8b081a27bbfc06b8f02003f50ac6d18c07 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 22 Feb 2024 20:52:16 -0500 Subject: [PATCH 041/151] There is still something a bit jenky going on, but I'm running through a few cycles of alignment to see if it is a relative error or absolut. --- ctf/BH_ctf_Correct3d.m | 79 ++++++++++++++++++++--------------------- testScripts/mCompile.sh | 2 +- 2 files changed, 39 insertions(+), 42 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 50956762..0cd606ea 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -316,15 +316,15 @@ iterList{gpuList(iGPU)} = iGPU+(tiltStart-1):nGPUs:nTilts; iterList{gpuList(iGPU)}; end -% try -% EMC_parpool(nGPUs) -% catch -% delete(gcp('nocreate')) -% EMC_parpool(nGPUs) -% end - -% parfor iGPU = 1:nGPUs -for iGPU = 1:nGPUs %%revert +try + EMC_parpool(nGPUs) +catch + delete(gcp('nocreate')) + EMC_parpool(nGPUs) +end + +parfor iGPU = 1:nGPUs +% for iGPU = 1:nGPUs %%revert for iTilt = iterList{gpuList(iGPU)} @@ -345,7 +345,7 @@ nTomos = 0; alreadyMade = 0; for iTomo = 1:length(tomoList) - if strcmp(tiltList{iTilt},subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) + if strcmp(tiltList{iTilt}, subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) iTomoList{nTomos+1} = tomoList{iTomo}; nTomos = nTomos + 1; end @@ -384,8 +384,8 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. -% parfor iGPU = 1:nGPUs -for iGPU = 1:nGPUs %%revert +parfor iGPU = 1:nGPUs +% for iGPU = 1:nGPUs %%revert % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); @@ -393,42 +393,17 @@ for iTilt = iterList{gpuList(iGPU)} slab_list = {}; - % if ~strcmp(tiltList{iTilt},'TS_121') + % if ~strcmp(tiltList{iTilt},'TS_116') % continue % % revert % end - if (recon_for_templateMatching) - % templaterch - nTomos = nTomosPerTilt{iTilt}; - % tiltRecGeom is a cell with each value being a cell returned by multi_recGeom - iCoords = tiltRecGeom{iTilt}; - % iCoords will be a cell indexed by each tomo wwith a struct .tomoCoords - else - nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; - % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; - iCoords = cell(nTomos,1); - for iCoordIdx = 1:nTomos - iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iCoordIdx}); - % FIXME - end - % else - % if (recon_for_tomoCPR) - % nTomos = 1; - % else - - % end - end - - iTomoList = cell(nTomos,1); - - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1) TLT = load(TLTNAME); fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); - + iTomoList = {}; if (recon_for_subTomo) % Get all the tomogram names that belong to a given tilt-series. nTomos = 0; @@ -460,6 +435,28 @@ end end + if (recon_for_templateMatching) + % templaterch + nTomos = nTomosPerTilt{iTilt}; + % tiltRecGeom is a cell with each value being a cell returned by multi_recGeom + iCoords = tiltRecGeom{iTilt}; + % iCoords will be a cell indexed by each tomo wwith a struct .tomoCoords + else + nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; + % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; + iCoords = cell(nTomos,1); + for iCoordIdx = 1:nTomos + iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(iTomoList{iCoordIdx}); + % FIXME + end + % else + % if (recon_for_tomoCPR) + % nTomos = 1; + % else + + % end + end + if samplingRate > 1 fullStack = sprintf('%aliStacks/%s_ali%d.fixed', ... @@ -685,7 +682,7 @@ gpuList(iGPU), ... floor(iCoords{thisTomo}.NX ./ samplingRate),... % WIDTH = NX floor(round(slab_list{iT}(iSection,5))), ... % THICKNESS = NZ - floor(iCoords{thisTomo}.dX_specimen_to_tomo ./ samplingRate), ... % SHIFT X + iCoords{thisTomo}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X slab_list{iT}(iSection,6)); @@ -908,7 +905,7 @@ continue; end valid_region_origin = emc_get_origin_index(slab_list{iT}(iSlab,5)); - dZ_for_reconstructed_slab = -(valid_indices(valid_region_origin) + fraction_origin_shift); + dZ_for_reconstructed_slab = (valid_indices(valid_region_origin) + fraction_origin_shift); slab_list{iT}(iSlab,6) = dZ_for_reconstructed_slab; %dZ end diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 65d59928..5ce9387e 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=0 -nightly=4 +nightly=5 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 554993a0dd869d78e6abd64e1216debc1dfa79f2 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 23 Feb 2024 01:15:55 -0500 Subject: [PATCH 042/151] Cleaning a few things up and I think I'm happy with the coordinate transformations for CTF3d. The binning and offsets are still a bit of a cluster and there is still duplicated code to deal with surrounding common ops on getting info out of the STM. Also need to replace the cell/lists that result in jenky indexing with structs accessed by tomoname. --- alignment/BH_alignRaw3d_v2.m | 3 +- coordinates/BH_multi_calcBinShift.m | 4 +- coordinates/BH_multi_recGeom.m | 2 + ctf/BH_ctf_Correct3d.m | 79 +++++++++++++++------------- ctf/BH_ctf_Estimate.m | 2 +- ctf/BH_ctf_Updatefft.m | 2 +- masking/BH_multi_projectionMask.m | 3 +- metaData/BH_geometryInitialize.m | 20 +++---- synthetic/BH_synthetic_mapBack.m | 17 +++--- synthetic/BH_to_cisTEM_mapBack.m | 19 +++---- testScripts/fourierTransformer.m | 7 ++- transformations/BH_multi_loadOrBin.m | 6 +-- 12 files changed, 87 insertions(+), 77 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 93b3dba7..13b174be 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -685,8 +685,7 @@ % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); - tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; - tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + binShift = [0,0,0]; nSubTomos = size(positionList,1); diff --git a/coordinates/BH_multi_calcBinShift.m b/coordinates/BH_multi_calcBinShift.m index a5c18b0b..66feccdb 100644 --- a/coordinates/BH_multi_calcBinShift.m +++ b/coordinates/BH_multi_calcBinShift.m @@ -11,7 +11,7 @@ % shift to have an odd dimension so that Imod origin is always the same. binSize = floor(coords./samplingRate); if (force_odd_dimension) - binSize = binSize + (1-mod(binSize,2)); + binSize = binSize - (1-mod(binSize,2)); end originFull = emc_get_origin_index(coords); @@ -27,7 +27,7 @@ % This is the shift we need to apply to the binned image to make sure % that the origin is in the same place. - binShift = (samplingRate.*originBin - originFull) ./ samplingRate; + binShift = -1.*(samplingRate.*originBin - originFull) ./ samplingRate; end diff --git a/coordinates/BH_multi_recGeom.m b/coordinates/BH_multi_recGeom.m index 5c64326b..9827cfd0 100755 --- a/coordinates/BH_multi_recGeom.m +++ b/coordinates/BH_multi_recGeom.m @@ -50,6 +50,8 @@ recGeom = cell(nTomos,1); for iTomo = 1:nTomos read_in_Coords = recCoords(1 + (iTomo-1)*6: 6 + (iTomo-1)*6); + % This is not necessarily correct, e.,g you could have 4 bin10 tomos from one tilt, and not keep any model points + % from tomo 3, then you would have tomos 1,2,4 but here we are using 1:3. tomoName = sprintf('%s_%d',tiltName, iTomo); % Check to make sure no out of bounds conditions were created in X Y diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 0cd606ea..864250a1 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -393,12 +393,31 @@ for iTilt = iterList{gpuList(iGPU)} slab_list = {}; - % if ~strcmp(tiltList{iTilt},'TS_116') - % continue - % % revert - % end + if (recon_for_subTomo) + nTomos = masterTM.mapBackGeometry.(tiltList{iTilt}).nTomos; + iCoords = masterTM.mapBackGeometry.(tiltList{iTilt}).coords; + % FIXME + else + if (recon_for_tomoCPR) + nTomos = 1; + else + % templaterch + nTomos = nTomosPerTilt{iTilt}; + iCoords = recGeom{iTilt}; + end + end + + if (recWithoutMat && ~loadSubTomoMeta) || ~recWithoutMat + targetSizeY = diff(floor(iCoords(:,2:3)),1,2)+1; + iCoords = iCoords ./ samplingRate; + iCoords(:,1:4) = floor(iCoords(:,1:4)); + iCoords(:,3) = iCoords(:,3) - (diff(floor(iCoords(:,2:3)),1,2)+1 - floor(targetSizeY./samplingRate)); + end + iTomoList = cell(nTomos,1); + + - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1) + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt}, mapBackIter + 1 ); TLT = load(TLTNAME); fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); @@ -435,27 +454,7 @@ end end - if (recon_for_templateMatching) - % templaterch - nTomos = nTomosPerTilt{iTilt}; - % tiltRecGeom is a cell with each value being a cell returned by multi_recGeom - iCoords = tiltRecGeom{iTilt}; - % iCoords will be a cell indexed by each tomo wwith a struct .tomoCoords - else - nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; - % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; - iCoords = cell(nTomos,1); - for iCoordIdx = 1:nTomos - iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(iTomoList{iCoordIdx}); - % FIXME - end - % else - % if (recon_for_tomoCPR) - % nTomos = 1; - % else - - % end - end + if samplingRate > 1 @@ -650,9 +649,9 @@ end % round down and then we'll add any extra needed to the final chunk - tiltChunkSize = ceil(iCoords{thisTomo}.NY ./ samplingRate ./ emc.n_tilt_workers); + tiltChunkSize = floor(iCoords{thisTomo}.NY ./ samplingRate ./ emc.n_tilt_workers); % This shoulid never happen, but to be safe - if (emc.n_tilt_workers > iCoords{thisTomo}.NY) + if (emc.n_tilt_workers > floor(iCoords{thisTomo}.NY ./ samplingRate)) error('n_tilt_workers is greater than the number of slices in the tilt series'); end y_i = floor(iCoords{thisTomo}.y_i ./ samplingRate); @@ -905,6 +904,12 @@ continue; end valid_region_origin = emc_get_origin_index(slab_list{iT}(iSlab,5)); + % This is a vector from the origin of the sample to the origin of the slab. + % The shift passed to imod-tilt moves the reconstructed area in the opposite sense. + % All slabs need to be shifted to the specimen origin (0) from tilts perspective, and then the are assembled into the final volume. + % This means a slab at Z > 0 needs to be shifted in the negative direction, which means supplying + % a shift that is also > 0, moving the volume "up" in the rotated coordinate system (imod -Z) + % I know ... this is a shit show. dZ_for_reconstructed_slab = (valid_indices(valid_region_origin) + fraction_origin_shift); slab_list{iT}(iSlab,6) = dZ_for_reconstructed_slab; %dZ @@ -918,14 +923,14 @@ delta = slab_list{iT}(iSlab,5); slab_list{iT}(iSlab-1,5) = slab_list{iT}(iSlab-1,5) + delta; slab_list{iT}(iSlab,1) = 0; - % we are adding slices from above the specimen in Z so the z shift is negative - slab_list{iT}(iSlab-1,6) = (slab_list{iT}(iSlab-1,6) - delta); + % we are adding slices from above the specimen in Z so the z shift is positive + slab_list{iT}(iSlab-1,6) = (slab_list{iT}(iSlab-1,6) + delta); elseif (iSlab < n_slabs_to_reconstruct && slab_list{iT}(iSlab+1,1)) delta = slab_list{iT}(iSlab,5); slab_list{iT}(iSlab+1,5) = slab_list{iT}(iSlab+1,5) + slab_list{iT}(iSlab,5); slab_list{iT}(iSlab,1) = 0; - % we are adding slices from below the specimen in Z so the z shift is positive - slab_list{iT}(iSlab+1,6) = (slab_list{iT}(iSlab+1,6) + delta); + % we are adding slices from below the specimen in Z so the z shift is negative + slab_list{iT}(iSlab+1,6) = (slab_list{iT}(iSlab+1,6) - delta); end end end @@ -1180,10 +1185,10 @@ end [ specimen_NX_nm, tomoIdx ] = emc_get_max_specimen_NZ(val_to_pass, ... - iCoords, ... - tomoList, ... - nTomos, ... - samplingRate); + iCoords, ... + tomoList, ... + nTomos, ... + samplingRate); specimen_NX_nm = specimen_NX_nm .* pixel_size_angstroms ./ 10; fprintf('combining the thickness and shift on tilt %s, found a specimen_NX_nm %3.3f nm\n',tiltName,specimen_NX_nm); @@ -1253,7 +1258,7 @@ % shift from lower left to centered and include the tomos offset from the subtomo_origin_wrt_specimen_origin = subtomo_origin_z_in_tomo_frame - tomo_origin_in_tomo_frame(3) + tomo_origin_wrt_tilt_origin(3); - subtomo_origin_wrt_specimen_origin ./ samplingRate; + subtomo_origin_wrt_specimen_origin = subtomo_origin_wrt_specimen_origin ./ samplingRate; totalZ = totalZ + sum(subtomo_origin_wrt_specimen_origin); fprintf('%s tomo has %d subTomos with mean Z %3.3f nm\n', ... iTomoName, length(subtomo_origin_wrt_specimen_origin), ... diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 5b62e518..86b3f744 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -431,7 +431,7 @@ shiftMETHOD,'singleTaper'); if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection); + bhF = fourierTransformer(iProjection,'OddSizeOversampled'); end diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 4e7e1cf1..aa77c119 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -374,7 +374,7 @@ 'GPU','singleTaper'); if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection); + bhF = fourierTransformer(iProjection,'OddSizeOversampled'); end diff --git a/masking/BH_multi_projectionMask.m b/masking/BH_multi_projectionMask.m index 139dced3..2fe619d4 100755 --- a/masking/BH_multi_projectionMask.m +++ b/masking/BH_multi_projectionMask.m @@ -38,8 +38,7 @@ for iPrj = 1:d3 - R = BH_defineMatrix([TLT(iPrj,6),TLT(iPrj,4),TLT(iPrj,6)], ... - 'Bah','fwdVector'); + R = BH_defineMatrix([TLT(iPrj,6),TLT(iPrj,4),TLT(iPrj,6)], 'Bah','fwdVector'); rInv = R'; diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index d2250c86..ab1934a8 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -180,10 +180,11 @@ end % We are storing this info to make it available when checking for duplicates + tomoName = sprintf('%s_%d', tiltName,tomoIdx); fileInfo{n_tomos_added,1} = tiltName; - fileInfo{n_tomos_added,2} = sprintf('%s_%d', tiltName, tomoIdx); + fileInfo{n_tomos_added,2} = tomoName; fileInfo{n_tomos_added,3} = sprintf('%s_%d_bin%d',tiltName, tomoIdx, dupSampling); - tomoName = sprintf('%s_%d',tiltName,tomoIdx); + subTomoMeta.('tiltGeometry').(fileInfo{n_tomos_added,2}) = tilt_geometry; n_tomos_added = n_tomos_added + 1; @@ -212,10 +213,9 @@ end -iterLiTomo = cell(nGPUs,1); +iterList = cell(nGPUs,1); for iGPU = 1:nGPUs - iterLiTomo{iGPU} = iGPU:nGPUs:nTomogramsTotal; - iterLiTomo{iGPU} + iterList{iGPU} = iGPU:nGPUs:nTomogramsTotal; end try @@ -229,12 +229,12 @@ dupInTheLoop = dupSampling parfor iGPU = 1:nGPUs - % for iGPU = 1:nGPUs % revert + % for iGPU = 1:nGPUs D = gpuDevice(iGPU); tomoResults = struct(); - for iTomo = iterLiTomo{iGPU} + for iTomo = iterList{iGPU} if (doImport) mapName = fileInfo{iTomo,2}; @@ -363,9 +363,9 @@ overlapMatrix = convn(positionMatrix, gpuArray(dupMask), 'same'); - idxLiTomo = positionIDX((overlapMatrix > 1)); + idxList = positionIDX((overlapMatrix > 1)); - tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom(ismember(tmpSearchGeom(:,4), idxLiTomo),:); + tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom(ismember(tmpSearchGeom(:,4), idxList),:); else tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom; @@ -381,7 +381,7 @@ nIDX = 1; for iGPU = 1:nGPUs - for iTomo = iterLiTomo{iGPU} + for iTomo = iterList{iGPU} mapName = fileInfo{iTomo,2}; tmpGeom = parResults{iGPU}.(mapName); diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 27dc96c8..1747a6cd 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -251,13 +251,15 @@ tilt_filename = sprintf('%saliStacks/%s_ali%d.fixed', CWD, tiltNameList{iTiltSeries}, mapBackIter + 1); [~, tltName, tltExt] = fileparts(tilt_filename); tilt_binned_filename = sprintf('%scache/%s_bin%d%s', CWD, tltName, samplingRate, tltExt); - for iTomo = 1:size(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords,1) - % This is dumb, fix it to be explicit. - if any(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords(iTomo,:)) - tomoList{tomoIDX} = sprintf('%s_%d',tiltNameList{iTiltSeries},iTomo); - - % Only increment if values found. - tomoIDX = tomoIDX + 1; + fn = fieldnames(subTomoMeta.mapBackGeometry.tomoName); + for iTomo = 1:numel(fn) + if (subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName == tiltNameList{iTiltSeries}) + % This is dumb, fix it to be explicit. + if (subTomoMeta.mapBackGeometry.tomoCoords.(fn{iTomo}).is_active) + tomoList{tomoIDX} = fn{iTomo}; + % Only increment if values found. + tomoIDX = tomoIDX + 1; + end end end @@ -517,7 +519,6 @@ positionList = geometry.(tomoList{iTomo}); tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoIdx,1:4); positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 44836894..eb1fee00 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -180,14 +180,16 @@ % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); tomoList = {}; tomoIDX = 1; - for iTomo = 1:size(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords,1) - % This is dumb, fix it to be explicit. - if any(subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).coords(iTomo,:)) - tomoList{tomoIDX} = sprintf('%s_%d',tiltNameList{iTiltSeries},iTomo); - - % Only increment if values found. - tomoIDX = tomoIDX + 1; - end + fn = fieldnames(subTomoMeta.mapBackGeometry.tomoName); + for iTomo = 1:numel(fn) + if (subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName == tiltNameList{iTiltSeries}) + % This is dumb, fix it to be explicit. + if (subTomoMeta.mapBackGeometry.tomoCoords.(fn{iTomo}).is_active) + tomoList{tomoIDX} = fn{iTomo}; + % Only increment if values found. + tomoIDX = tomoIDX + 1; + end + end end [~,tiltBaseName,~] = fileparts(tilt_filename); @@ -376,7 +378,6 @@ positionList = geometry.(tomoList{iTomo}); tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - coords = subTomoMeta.mapBackGeometry.(tiltName).coords(tomoIdx,1:4); positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); diff --git a/testScripts/fourierTransformer.m b/testScripts/fourierTransformer.m index 83ad6e27..05cc8997 100644 --- a/testScripts/fourierTransformer.m +++ b/testScripts/fourierTransformer.m @@ -31,6 +31,7 @@ paddedVolumeIsNonZero = true; shouldPad = false; +OddSizeOversampled = 0; useFwdSwapForInverse; % works for even sized images @@ -43,7 +44,11 @@ if nargin > 1 if (ischar(varargin{1})) +if (strcmpi(varargin{1},'OddSizeOversampled')) + obj.OddSizeOversampled = 1; + else error('Did not recognize the extra argument when intializing the fourierTransformer'); +end else if (isnumeric(varargin{1})) if (numel(varargin{1}) == 6) @@ -229,7 +234,7 @@ function delete(obj) [ obj.phaseCenter, dV, dW] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... {'none'},1,0,0,{'halfgrid'}); if ((obj.inputSize(1) == obj.inputSize(2)) && (obj.inputSize(2) == obj.inputSize(3))) - sx = obj.halfDimSize-1+obj.phaseSwapOffset(1); + sx = obj.halfDimSize-1+obj.OddSizeOversampled; obj.phaseCenter = exp(-2i.*pi.*sx.*(obj.phaseCenter+dV+dW)); clear dU dV dW else diff --git a/transformations/BH_multi_loadOrBin.m b/transformations/BH_multi_loadOrBin.m index 15b13017..5d38bb5b 100755 --- a/transformations/BH_multi_loadOrBin.m +++ b/transformations/BH_multi_loadOrBin.m @@ -70,8 +70,7 @@ % even sized images. force_odd_dimension = true; [binSize, binShift] = BH_multi_calcBinShift([iHeader.nX, iHeader.nY], samplingRate, force_odd_dimension); - % FIXME - binShift = binShift .* 0; + % Gridding correction for the interpolation in the binning. Not % sure this is quite right, but it looks much better. TODO FIXME [ R ] = BH_multi_gridCoordinates([iHeader.nX,iHeader.nY],'Cartesian','GPU', {'none'},1,1,1); @@ -83,8 +82,7 @@ iProjection = gpuArray(getVolume(tiltObj,[],[],iPrj,'keep')); if (iPrj == 1) - % FIXME: The swapping - bhF = fourierTransformer(iProjection); + bhF = fourierTransformer(iProjection,'OddSizeOversampled'); end iProjection = bhF.invFFT(bhF.fwdFFT(R.*iProjection,0,0,[1e-6,600,samplingRate*pixelSize,pixelSize]),2); From 5dc961b8dc095a343cfc7cdedf3f02554feb8d6a Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 23 Feb 2024 02:11:07 -0500 Subject: [PATCH 043/151] wip --- alignment/BH_templateSearch3d_2.m | 2 +- ctf/BH_ctf_Correct3d.m | 82 ++++++++++++++++++++----------- 2 files changed, 53 insertions(+), 31 deletions(-) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index cc6cb6cf..7aec2d11 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -183,7 +183,7 @@ mapName = sprintf('%s_%d_bin%d',tomoName,tomoIdx,samplingRate); mapExt = '.rec'; -[ recGeom, ~, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName) ); +[ recGeom, ~, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName), mapBackIter); bp_vals(2) = 2.*max(latticeRadius); diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 864250a1..e2bef969 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -316,18 +316,20 @@ iterList{gpuList(iGPU)} = iGPU+(tiltStart-1):nGPUs:nTilts; iterList{gpuList(iGPU)}; end + try - EMC_parpool(nGPUs) +EMC_parpool(nGPUs) catch - delete(gcp('nocreate')) - EMC_parpool(nGPUs) +delete(gcp('nocreate')) +EMC_parpool(nGPUs) end parfor iGPU = 1:nGPUs % for iGPU = 1:nGPUs %%revert for iTilt = iterList{gpuList(iGPU)} - + nTomos = 0; + alreadyMade = 0; iTomoList = {}; % For now, since the tilt geometry is not necessarily updated (it is manual) @@ -342,8 +344,6 @@ % Get all the tomogram names that belong to a given tilt-series. % FIXME: I'm not sure it makes sense to restrict this block to for_subTomo if (recon_for_subTomo) - nTomos = 0; - alreadyMade = 0; for iTomo = 1:length(tomoList) if strcmp(tiltList{iTilt}, subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) iTomoList{nTomos+1} = tomoList{iTomo}; @@ -393,27 +393,28 @@ for iTilt = iterList{gpuList(iGPU)} slab_list = {}; - if (recon_for_subTomo) - nTomos = masterTM.mapBackGeometry.(tiltList{iTilt}).nTomos; - iCoords = masterTM.mapBackGeometry.(tiltList{iTilt}).coords; - % FIXME - else - if (recon_for_tomoCPR) - nTomos = 1; - else - % templaterch - nTomos = nTomosPerTilt{iTilt}; - iCoords = recGeom{iTilt}; - end - end + % if (recon_for_subTomo) + % nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; + % iCoords = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).coords; + % % FIXME + % else + % if (recon_for_tomoCPR) + % nTomos = 1; + % else + % % templaterch + % nTomos = nTomosPerTilt{iTilt}; + % iCoords = tiltRecGeom{iTilt}; + % end + % end - if (recWithoutMat && ~loadSubTomoMeta) || ~recWithoutMat - targetSizeY = diff(floor(iCoords(:,2:3)),1,2)+1; - iCoords = iCoords ./ samplingRate; - iCoords(:,1:4) = floor(iCoords(:,1:4)); - iCoords(:,3) = iCoords(:,3) - (diff(floor(iCoords(:,2:3)),1,2)+1 - floor(targetSizeY./samplingRate)); - end - iTomoList = cell(nTomos,1); + + % if (recon_for_subTomo || recon_for_templateMatching)) + % targetSizeY = diff(floor(iCoords(:,2:3)),1,2)+1; + % iCoords = iCoords ./ samplingRate; + % iCoords(:,1:4) = floor(iCoords(:,1:4)); + % iCoords(:,3) = iCoords(:,3) - (diff(floor(iCoords(:,2:3)),1,2)+1 - floor(targetSizeY./samplingRate)); + % end + % iTomoList = cell(nTomos,1); @@ -454,7 +455,27 @@ end end - + if (recon_for_templateMatching) + % templaterch + nTomos = nTomosPerTilt{iTilt}; + % tiltRecGeom is a cell with each value being a cell returned by multi_recGeom + iCoords = tiltRecGeom{iTilt}; + % iCoords will be a cell indexed by each tomo wwith a struct .tomoCoords + else + nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; + % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; + iCoords = cell(nTomos,1); + for iCoordIdx = 1:nTomos + iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(iTomoList{iCoordIdx}); + % FIXME + end + % else + % if (recon_for_tomoCPR) + % nTomos = 1; + % else + + % end + end if samplingRate > 1 @@ -612,19 +633,20 @@ if (recon_for_tomoCPR) TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{1}),1); + TA = TA(:,4); end if (recon_for_subTomo) TA = sortrows(subTomoMeta.tiltGeometry.(sprintf('%s_%d',tiltList{iTilt},thisTomo)),1); + TA = TA(:,4); end - TA = TA(:,4); + if (recon_for_templateMatching) if (mapBackIter) % FIXME: I don't think this block should work, it should only be the tilt angles! error('THis block should not be reached.') - TA = load(sprintf('%smapBack%d/%s_ali%d_ctf.tlt',CWD,mapBackIter,tiltList{iTilt},... - mapBackIter)); + TA = load(sprintf('%smapBack%d/%s_ali%d_ctf.tlt',CWD,mapBackIter,tiltList{iTilt}, mapBackIter)); else TA = load(sprintf('%sfixedStacks/%s.tlt',CWD,tiltList{iTilt})); end From eed2649ff16f7120e312a80de0e93beff16eef96 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 23 Feb 2024 09:05:08 -0500 Subject: [PATCH 044/151] Fix indexing in new check on TS name vs TOMO name in mapBacks --- synthetic/BH_synthetic_mapBack.m | 2 +- synthetic/BH_to_cisTEM_mapBack.m | 4 ++-- testScripts/EMC_tmpDir.sh | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 1747a6cd..73b7aa05 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -253,7 +253,7 @@ tilt_binned_filename = sprintf('%scache/%s_bin%d%s', CWD, tltName, samplingRate, tltExt); fn = fieldnames(subTomoMeta.mapBackGeometry.tomoName); for iTomo = 1:numel(fn) - if (subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName == tiltNameList{iTiltSeries}) + if strcmp(subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName, tiltNameList{iTiltSeries}) % This is dumb, fix it to be explicit. if (subTomoMeta.mapBackGeometry.tomoCoords.(fn{iTomo}).is_active) tomoList{tomoIDX} = fn{iTomo}; diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index eb1fee00..0c9537cf 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -182,7 +182,7 @@ tomoIDX = 1; fn = fieldnames(subTomoMeta.mapBackGeometry.tomoName); for iTomo = 1:numel(fn) - if (subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName == tiltNameList{iTiltSeries}) + if strcmp(subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName, tiltNameList{iTiltSeries}) % This is dumb, fix it to be explicit. if (subTomoMeta.mapBackGeometry.tomoCoords.(fn{iTomo}).is_active) tomoList{tomoIDX} = fn{iTomo}; @@ -391,7 +391,7 @@ tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconGeometry = masterTM.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... reconGeometry.dY_specimen_to_tomo, ... reconGeometry.dZ_specimen_to_tomo]; diff --git a/testScripts/EMC_tmpDir.sh b/testScripts/EMC_tmpDir.sh index d4261f5e..f6e618ff 100644 --- a/testScripts/EMC_tmpDir.sh +++ b/testScripts/EMC_tmpDir.sh @@ -26,6 +26,7 @@ while read target; do if [[ $this_mem && $this_mem -gt $MAX_MEM ]] ; then MAX_MEM=${this_mem} MAX_FS=${target} + echo $MAX_MEM fi fi From b8de36330f356b1bc8e3a19d8e8ab41dc56113e0 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 23 Feb 2024 09:53:13 -0500 Subject: [PATCH 045/151] Changes to make to_cisTEM always use disk not ramdisk to avoid OOM. the output is clearly broken with the recent changes. My first guess is not handling sampling properly with the new coord structs. --- metaData/BH_geometryInitialize.m | 2 +- metaData/EMC_setup_tmp_cache.m | 6 +++--- synthetic/BH_to_cisTEM_mapBack.m | 17 +++++++---------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index ab1934a8..aa67004f 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -121,7 +121,7 @@ % If we are in the working directory, following template matching, there should % be the director convmap, holding convolution maps, model files etc. -if ~isdir('convmap') +if ~isfolder('convmap') error('Did not find directory named ') end diff --git a/metaData/EMC_setup_tmp_cache.m b/metaData/EMC_setup_tmp_cache.m index 077f4b09..ccfcbcc0 100644 --- a/metaData/EMC_setup_tmp_cache.m +++ b/metaData/EMC_setup_tmp_cache.m @@ -15,7 +15,7 @@ end else % We are using an existing cache directory, we need to make sure it exists - if isdir(existing_tmpCache) + if isfolder(existing_tmpCache) tmpCache = existing_tmpCache; else error('The existing tmpCache %s does not exist',existing_tmpCache); @@ -32,7 +32,7 @@ if isempty(name) % We must be in the local project directory, add this little check - if ~isdir('fixedStacks') + if ~isfolder('fixedStacks') % Should work for soft links too error('The fixedStacks directory does not exist in the current directory, %s',pwd); end @@ -52,7 +52,7 @@ case 'ctf3d' tmpCache = fullfile(tmpCache, 'ctf3d'); system(sprintf('mkdir -p %s', tmpCache)); - if ~isdir(tmpCache) + if ~isfolder(tmpCache) error('The tmpCache %s does not exist',tmpCache); end case 'tomoCPR' diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 0c9537cf..36e97673 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -72,15 +72,12 @@ flgInvertTiltAngles = 0; -if (skip_to_the_end_and_run) - % The fractional runs have already copied everything to cache/mapback%d, - % so override the tmpCache. - tmpCache = ''; -else - tmpCache= emc.('fastScratchDisk'); -end +% These stacks can get very unwieldy to we won't use the ramdisk even if it is asked for, +% additionally we'll save stacks as we go which has the unfortunate side effect of +% doubling the amount of disk space needed. The "Add stack" command might be a viable option, though there is +% some risk of data corruption. +[tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache('', fullfile(pwd,'cache'), 'cisTEM', false); -[tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(tmpCache, '', 'cisTEM', true); nGPUs = emc.('nGPUs'); pInfo = parcluster(); @@ -783,8 +780,8 @@ fprintf(fh,'%d\n', iCell); fclose(fh); -system(sprintf('cat %s >> %s',newstack_file,newstack_file_with_n_stacks)); -system(sprintf('newstack -FileOfInputs %s %s.mrc > /dev/null',newstack_file_with_n_stacks,output_prefix)); +system(sprintf('cat %s >> %s', newstack_file, newstack_file_with_n_stacks)); +system(sprintf('newstack -FileOfInputs %s %s.mrc > /dev/null', newstack_file_with_n_stacks, output_prefix)); % SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',output_prefix),pixelSize); From e13bcb147757c5417d18778018d4875fab167974 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 23 Feb 2024 11:20:56 -0500 Subject: [PATCH 046/151] Fixed local file ommission and reverted matrix definition, but still broken. --- coordinates/eulerSearch.m | 14 +++--- synthetic/BH_to_cisTEM_mapBack.m | 80 ++++++++++++++++++-------------- 2 files changed, 52 insertions(+), 42 deletions(-) diff --git a/coordinates/eulerSearch.m b/coordinates/eulerSearch.m index f58d5836..5cd30aa4 100644 --- a/coordinates/eulerSearch.m +++ b/coordinates/eulerSearch.m @@ -36,13 +36,13 @@ methods function [obj] = eulerSearch(wanted_symmetry_symbol, ... - wanted_theta_max,... - wanted_theta_step,... - wanted_psi_max,... - wanted_psi_step,... - wanted_resolution_limit,... - wanted_parameters_to_keep,... - wanted_random_start_angle) + wanted_theta_max,... + wanted_theta_step,... + wanted_psi_max,... + wanted_psi_step,... + wanted_resolution_limit,... + wanted_parameters_to_keep,... + wanted_random_start_angle) if (wanted_theta_max < 0) wanted_theta_max = abs(wanted_theta_max); diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 36e97673..feadcc48 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -97,7 +97,7 @@ calcCTF = emc.('tomo_cpr_defocus_refine'); -[tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); +[tilt_series_filenames, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); if (multi_node_run && ~skip_to_the_end_and_run) @@ -157,47 +157,49 @@ end if (useFixedNotAliStack) - tilt_filename = sprintf('%sfixedStacks/%s.fixed',CWD,tiltNameList{iTiltSeries}); + tilt_filestem = tilt_series_filenames{iTiltSeries}; + tilt_filepath = sprintf('%sfixedStacks/%s.fixed', CWD, tilt_series_filenames{iTiltSeries}); else - tilt_filename = sprintf('%saliStacks/%s_ali%d.fixed',CWD,tiltNameList{iTiltSeries},mapBackIter+1); + tilt_filestem = sprintf('%s_ali%d',tilt_series_filenames{iTiltSeries},mapBackIter+1); + tilt_filepath = sprintf('%saliStacks/%s_ali%d.fixed', CWD, tilt_series_filenames{iTiltSeries}, mapBackIter+1); end - tilt_filename = sprintf('%saliStacks/%s_ali%d.fixed', CWD, tiltNameList{iTiltSeries}, mapBackIter + 1); - mapBackRePrjSize = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize'); - % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); - nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos - if nTomograms == 0 + % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tilt_series_filenames{iTiltSeries}); + n_tomos_this_tilt_series = subTomoMeta.mapBackGeometry.(tilt_series_filenames{iTiltSeries}).nTomos; + if n_tomos_this_tilt_series == 0 % No points were saved after template matching so skip this tilt series % altogether. continue end - skip_this_tilt_series_because_it_is_empty = false(nTomograms,1); + skip_this_tilt_series_because_it_is_empty = false(n_tomos_this_tilt_series,1); % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); tomoList = {}; - tomoIDX = 1; + n_active_tomos = 0; fn = fieldnames(subTomoMeta.mapBackGeometry.tomoName); for iTomo = 1:numel(fn) - if strcmp(subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName, tiltNameList{iTiltSeries}) + if strcmp(subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName, tilt_series_filenames{iTiltSeries}) % This is dumb, fix it to be explicit. if (subTomoMeta.mapBackGeometry.tomoCoords.(fn{iTomo}).is_active) - tomoList{tomoIDX} = fn{iTomo}; + tomoList{n_active_tomos+1} = fn{iTomo}; % Only increment if values found. - tomoIDX = tomoIDX + 1; + n_active_tomos = n_active_tomos + 1; end end end - - [~,tiltBaseName,~] = fileparts(tilt_filename); - mbOUT{2} = tiltBaseName; - + + if (n_active_tomos == 0) + continue; + end + + mbOUT{2} = tilt_filestem; if (mapBackIter) - localFile = sprintf('%s/%s_ali%d_ctf.local', CWD,mapBackIter,tiltNameList{iTiltSeries},mapBackIter); + localFile = sprintf('%smapBack%d/%s_ali%d_ctf.local', CWD,mapBackIter,tilt_series_filenames{iTiltSeries},mapBackIter); else - localFile = sprintf('%sfixedStacks/%s.local',CWD,tiltNameList{iTiltSeries}); + localFile = sprintf('%sfixedStacks/%s.local',CWD,tilt_series_filenames{iTiltSeries}); end - + if exist(localFile,'file') fprintf('Found local file %s\n', localFile); else @@ -269,12 +271,12 @@ maxZ = 0; % The - tiltHeader = getHeader(MRCImage(tilt_filename, 0)); + tiltHeader = getHeader(MRCImage(tilt_filepath, 0)); tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{1}).tiltName; [ maxZ ] = emc_get_max_specimen_NZ(subTomoMeta.mapBackGeometry.tomoName, ... subTomoMeta.mapBackGeometry.tomoCoords, ... tomoList, ... - nTomograms, ... + n_active_tomos, ... 1); fprintf('combining thickness and shift, found a maxZ of %d\n',maxZ); @@ -302,7 +304,7 @@ % aggressivley downweight outliers in the alignment nFidsTotal = 0; fidIDX = 0; - for iTomo = 1:nTomograms + for iTomo = 1:n_active_tomos TLT = tiltGeometry.(tomoList{iTomo}); @@ -373,13 +375,12 @@ end positionList = geometry.(tomoList{iTomo}); - tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; - tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); - tiltHeader = getHeader(MRCImage(tilt_filename,0)); + tiltHeader = getHeader(MRCImage(tilt_filepath,0)); fullTiltSizeXandY = [tiltHeader.nX,tiltHeader.nY]; @@ -433,7 +434,9 @@ % imod is indexing from zero zCoord = iPrj_nat; % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) - rTilt = BH_defineMatrix(TLT(iPrj_nat,4),'TILT','fwdVector') ; + rTilt = BH_defineMatrix([0,TLT(iPrj_nat,4),0],'SPIDER','inv'); + % rTilt = BH_defineMatrix(TLT(iPrj_nat,4),'TILT','fwdVector') ; + prjCoords = rTilt*subtomo_origin_wrt_tilt_origin'; @@ -476,7 +479,7 @@ end if (localFile) - lastLine1 = sprintf('LOCALFILE %s', localFile) + lastLine1 = sprintf('LOCALFILE %s', localFile); % Used if GPU fails cpuLastLine = lastLine1; else @@ -519,7 +522,7 @@ '%s\n',... '%s\n',... '%s\n',... - 'EOF'],tilt_filename, mbOUT{1:2}, maxZ, ... + 'EOF'],tilt_filepath, mbOUT{1:2}, maxZ, ... mbOUT{1:2},... mbOUT{1:2},... pixel_size./10, ... % Ang --> nm @@ -549,9 +552,14 @@ system(sprintf('imodtrans -2 %s %s/%s.fid %s/%s.invfid > /dev/null', iXFName_inv, mbOUT{1:2},mbOUT{1:2})); - system(sprintf(['model2point -contour -zero ',... - '%s/%s.invfid %s/%s.coordPrj > /dev/null'],... - mbOUT{1:2}, mbOUT{1:2})) + base_cmd = sprintf(['model2point -contour -zero ',... + '%s/%s.invfid %s/%s.coordPrj '],... + mbOUT{1:2}, mbOUT{1:2}) + errmsg = system(sprintf('%s > /dev/null',base_cmd)); + if (errmsg) + system(sprintf('%s',base_cmd)); + error('Failed to run model2point'); + end else system(sprintf(['model2point -contour -zero ',... '%s/%s.fid %s/%s.coordPrj > /dev/null'],... @@ -646,7 +654,7 @@ fullXform = load(iXFName_inv); end - STACK = single(getVolume(MRCImage(tilt_filename))); + STACK = single(getVolume(MRCImage(tilt_filepath))); for iPrj = 1:nPrjs @@ -694,9 +702,11 @@ rotFull = rTilt*[RF(1), RF(2), 0; RF(3), RF(4), 0; 0, 0, 1]*reshape(wrkPar(iFid,7:15),3,3); else % This gives us Rz*Ry - rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','fwdVector'); + % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','fwdVector'); + + % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); + rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','fwdVector'); - % this fives Rz*Ry*e3*e2*e1 * interpolant would rotate the particle by Rz*Ry*e1*e2*e3 rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); end From 4336ae1ef5de2b63486a58e25024fee7b23bc899 Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 26 Feb 2024 16:28:54 -0500 Subject: [PATCH 047/151] hate --- ctf/BH_ctf_Correct3d.m | 22 ++++++------- logicals/BH_isWindowValid.m | 2 +- metaData/BH_geometryInitialize.m | 4 ++- metaData/BH_parseParameterFile.m | 8 ++--- mexFiles/mexXform3d.cu | 56 +++++++++++++------------------- synthetic/BH_to_cisTEM_mapBack.m | 40 ++++++++++++++--------- 6 files changed, 65 insertions(+), 67 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index e2bef969..6fd98bbf 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -317,15 +317,15 @@ iterList{gpuList(iGPU)}; end -try -EMC_parpool(nGPUs) -catch -delete(gcp('nocreate')) -EMC_parpool(nGPUs) -end - -parfor iGPU = 1:nGPUs -% for iGPU = 1:nGPUs %%revert +% try +% EMC_parpool(nGPUs) +% catch +% delete(gcp('nocreate')) +% EMC_parpool(nGPUs) +% end + +% parfor iGPU = 1:nGPUs +for iGPU = 1:nGPUs %%revert for iTilt = iterList{gpuList(iGPU)} nTomos = 0; @@ -384,8 +384,8 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. -parfor iGPU = 1:nGPUs -% for iGPU = 1:nGPUs %%revert +% parfor iGPU = 1:nGPUs +for iGPU = 1:nGPUs %%revert % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); diff --git a/logicals/BH_isWindowValid.m b/logicals/BH_isWindowValid.m index c44d3920..eb33e506 100755 --- a/logicals/BH_isWindowValid.m +++ b/logicals/BH_isWindowValid.m @@ -48,7 +48,7 @@ % Get the down to the nearest integer position, and save any fractional shift -winCenter = floor(CENTER); +winCenter = floor(CENTER+0.5); deltaWinCenter = CENTER - winCenter; diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index aa67004f..22ad8a4c 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -167,7 +167,7 @@ end subTomoMeta.('mapBackGeometry').(tiltName).('nTomos') = nTomos; subTomoMeta.('mapBackGeometry').(tiltName).('tomoCprRePrjSize') = 512; - + stm_tomoList = cell(nTomos,1); for iTomo = 1:nTomos if (doImport) @@ -183,6 +183,7 @@ tomoName = sprintf('%s_%d', tiltName,tomoIdx); fileInfo{n_tomos_added,1} = tiltName; fileInfo{n_tomos_added,2} = tomoName; + stm_tomoList{iTomo} = tomoName; fileInfo{n_tomos_added,3} = sprintf('%s_%d_bin%d',tiltName, tomoIdx, dupSampling); @@ -203,6 +204,7 @@ subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dY_specimen_to_tomo') = recGeom{tomoIdx}.dY_specimen_to_tomo; subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dZ_specimen_to_tomo') = recGeom{tomoIdx}.dZ_specimen_to_tomo; end + subTomoMeta.('mapBackGeometry').(tiltName).('tomoList') = stm_tomoList; end % end of loop over stacks % For now, just assuming all of the maps are in the same place and have the same diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index bad1d9ee..4bd09c73 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -399,25 +399,25 @@ if isfield(emc, 'flgPostShift') EMC_assert_numeric(emc.flgPostShift, 2); else - emc.flgPostShift = [-1.5,1.0]; + emc.flgPostShift = 0.*[-1.5,1.0]; end if isfield(emc, 'prjVectorShift') EMC_assert_numeric(emc.prjVectorShift, 3); else - emc.prjVectorShift = [0.5,0.5,0.5]; + emc.prjVectorShift = 0.*[0.5,0.5,0.5]; end if isfield(emc,'pixelShift') EMC_assert_numeric(emc.pixelShift, 1); else - emc.pixelShift = -1; + emc.pixelShift = 0.*-1; end if isfield(emc, 'pixelMultiplier') EMC_assert_numeric(emc.pixelMultiplier, 1); else - emc.pixelMultiplier = 1; + emc.pixelMultiplier = -1; end if isfield(emc, 'tomoCPR_random_subset') diff --git a/mexFiles/mexXform3d.cu b/mexFiles/mexXform3d.cu index 09c1d4be..02802fc0 100644 --- a/mexFiles/mexXform3d.cu +++ b/mexFiles/mexXform3d.cu @@ -42,23 +42,25 @@ __global__ void transformKernel_FWD(cudaTextureObject_t thisTexObj, float u,v,w,tu,tv,tw; - if (doFwdXform) - { + if (doFwdXform) { // First, put the origin at the center, rotate, then shift - u = (float)x - (float)(dims.x/2) + shifts.x; - v = (float)y - (float)(dims.y/2) + shifts.y; - w = (float)z - (float)(dims.z/2) + shifts.z; + // Effectively, we rotate the interpolant about the current origin, but then need to shift it + // by dx in the new coordinate system. + // R * (x - c) - R * dx + // R * (x - c - dx) = R ( x - (c -dx)) + u = float(x - dims.x/2) - shifts.x; + v = float(y - dims.y/2) - shifts.y; + w = float(z - dims.z/2) - shifts.z; tu = u*rm_1.x + v*rm_1.y + w*rm_1.z + size_shift.x; tv = u*rm_2.x + v*rm_2.y + w*rm_2.z + size_shift.y; tw = u*rm_3.x + v*rm_3.y + w*rm_3.z + size_shift.z; } - else - { + else { // First, put the origin at the center, shift, then rotate - u = (float)x - (float)(dims.x/2); - v = (float)y - (float)(dims.y/2); - w = (float)z - (float)(dims.z/2); + u = float(x - dims.x/2); + v = float(y - dims.y/2); + w = float(z - dims.z/2); tu = u*rm_1.x + v*rm_1.y + w*rm_1.z + shifts.x + size_shift.x; tv = u*rm_2.x + v*rm_2.y + w*rm_2.z + shifts.y + size_shift.y; tw = u*rm_3.x + v*rm_3.y + w*rm_3.z + shifts.z + size_shift.z; @@ -68,22 +70,8 @@ __global__ void transformKernel_FWD(cudaTextureObject_t thisTexObj, tu /= (float)dims.x; tv /= (float)dims.y; tw /= (float)dims.z; - // Since we are - tu += 0.5f; - tv += 0.5f; - tw += 0.5f; - -// if (tu < 0 | tv < 0 | tw < 0 | tu >= 1 - 1/(float)dims.x | tv >= 1 - 1/(float)dims.y | tw >= 1 - 1/(float)dims.z) -// { - -// outputData[ (z*dims.y + y) * dims.x + x ] = extrapVal; -// } -// else -// { -// outputData[ (z*dims.y + y) * dims.x + x ] = tex3D(thisTexObj, tu, tv, tw); -// } - // cudaAddressModeBorder returns 0.0 for out of bounds reads. - outputData[ (z*dims.y + y) * dims.x + x ] = tex3D(thisTexObj, tu, tv, tw); + + outputData[ (z*dims.y + y) * dims.x + x ] = tex3D(thisTexObj, tu + 0.5f, tv + 0.5f, tw + 0.5f); } @@ -157,15 +145,15 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) float *ts = (float *) mxGetData(prhs[3]); doFwdXform = (bool *) mxGetData(prhs[4]); - if (*doFwdXform) - { - // transpose matrix - shifts = make_float3(-ts[0],-ts[1],-ts[2]); - } - else - { + // if (*doFwdXform) + // { + // // transpose matrix + // shifts = make_float3(-ts[0],-ts[1],-ts[2]); + // } + // else + // { shifts = make_float3(ts[0],ts[1],ts[2]); - } + // } rm_1 = make_float3(angles[0],angles[3],angles[6]); rm_2 = make_float3(angles[1],angles[4],angles[7]); diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index feadcc48..21d6ba85 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -290,7 +290,7 @@ TLT = tiltGeometry.(tomoList{1}); - iRawTltName = sprintf('%s/%s_align.rawtlt',mbOUT{1:2}) + iRawTltName = sprintf('%s/%s_align.rawtlt',mbOUT{1:2}); iTiltFile = fopen(iRawTltName, 'w'); rawTLT = sortrows(TLT(:,[1,4]),1); fprintf(iTiltFile,'%f\n',rawTLT(:,2)'); @@ -390,7 +390,7 @@ tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); - tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... + tomo_origin_wrt_tilt_origin = [ reconGeometry.dX_specimen_to_tomo, ... reconGeometry.dY_specimen_to_tomo, ... reconGeometry.dZ_specimen_to_tomo]; tomo_origin_in_tomo_frame = emc_get_origin_index([reconGeometry.NX, ... @@ -434,8 +434,8 @@ % imod is indexing from zero zCoord = iPrj_nat; % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) - rTilt = BH_defineMatrix([0,TLT(iPrj_nat,4),0],'SPIDER','inv'); - % rTilt = BH_defineMatrix(TLT(iPrj_nat,4),'TILT','fwdVector') ; + % rTilt = BH_defineMatrix([0,TLT(iPrj_nat,4),0],'SPIDER','inv'); + rTilt = BH_defineMatrix(TLT(iPrj_nat,4),'TILT','fwdVector') ; prjCoords = rTilt*subtomo_origin_wrt_tilt_origin'; @@ -468,11 +468,11 @@ fclose(coordOUT); fclose(coordSTART); - p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 ',... + p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 -ImageForCoordinates %s ',... '%s/%s.coord %s/%s.3dfid'], ... - mbOUT{1:2},mbOUT{1:2}); + tilt_filepath, mbOUT{1:2},mbOUT{1:2}); system(p2m); - + taStr = [sprintf('%f',rawTLT(1,2))]; for iTa = 2:length(rawTLT(:,2)) taStr = [taStr sprintf(',%f',rawTLT(iTa,2))]; @@ -567,7 +567,6 @@ end - try fidList = load(sprintf('%s/%s.coordPrj',mbOUT{1:2})); catch @@ -594,7 +593,6 @@ % Give every instance of each fiducial a unique identifier. fidList = [1:size(fidList,1);fidList']'; - particlePad = 2.0; tileRadius = floor(particlePad.*particle_radius); tileSize = BH_multi_iterator((2.*tileRadius).*[1,1],'fourier2d'); @@ -671,7 +669,7 @@ wrkFid = fidList(wrkPrjIDX,:); wrkPar = parList(wrkPrjIDX,:); wrkDefAngTilt = defList(wrkPrjIDX,[7,6,5]); % Confirming with David but this should include the local adjustments to tilt/in-plane angle - + for iFid = 1:size(wrkFid,1) @@ -687,7 +685,18 @@ sx = pixelX - floor(pixelX); sy = pixelY - floor(pixelY); - + + % With a pixel running from -0.5 to 0.5, the center of the pixel is at 0.0, to get the index we need to do this + % pixelX = floor(x_coord + 0.5); + % pixelY = floor(y_coord + 0.5); + + % x_start = pixelX - tileOrigin(1) + 1; + % y_start = pixelY - tileOrigin(2) + 1; + + % % The remainder is the result of the windowing operation + % sx = x_coord - pixelX; + % sy = y_coord - pixelY; + particle_was_skipped = false; if ( x_start > 0 && y_start > 0 && x_start + tileSize(1) - 1 < sTX && y_start + tileSize(2) - 1 < sTY ) output_particle_stack(:,:,iGpuDataCounter) = STACK(x_start:x_start+tileSize(1)-1,y_start:y_start+tileSize(2)-1,TLT(iPrj,1)); @@ -705,7 +714,7 @@ % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','fwdVector'); % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); - rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','fwdVector'); + rTilt = BH_defineMatrix([-wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','fwdVector'); rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); end @@ -740,9 +749,8 @@ particleGroup = wrkPar(iFid,3); preExposure = wrkPar(iFid,16); totalExposure = wrkPar(iFid,17); - pixelMultiplier = 0; - xShift = pixelMultiplier*sx*pixelSize; - yShift = pixelMultiplier*sy*pixelSize; + xShift = emc.pixelMultiplier*sx*pixelSize; + yShift = emc.pixelMultiplier*sy*pixelSize; % df1 = ( wrkPar(iFid,4) + wrkPar(iFid,5)) * 10; % df2 = ( wrkPar(iFid,4) - wrkPar(iFid,5)) * 10; @@ -837,7 +845,7 @@ 'No\n', ...Dump intermediate arrays (merge later) [No] : 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : - '%2.2d\n', ...Max. threads to use for calculation [36] : + '%d\n', ...Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); From aba9f0cf9278de9de822e2db30744a6aa8a7bab0 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 28 Feb 2024 11:42:27 -0500 Subject: [PATCH 048/151] wip --- @MRCImage/SAVE_IMG.m | 9 +- @MRCImage/getModeString.m | 4 + @MRCImage/getVolume.m | 9 ++ @MRCImage/loadVolume.m | 9 +- @MRCImage/private/getModeBytes.m | 14 +- @MRCImage/private/readHeader.m | 18 +-- .../private/setVolumeAndHeaderFromVolume.m | 43 +++--- @MRCImage/private/writeHeader.m | 29 ++-- @MRCImage/save.m | 18 ++- @MRCImage/showHeader.m | 15 +- ctf/BH_ctf_Correct3d.m | 95 +++++++------ metaData/BH_skipClassAlignment.m | 2 +- metaData/SAVE_IMG.m | 9 +- mexFiles/mexFFT.cu | 2 - statistics/BH_clusterPub.m | 19 ++- synthetic/BH_synthetic_mapBack.m | 93 +++++++------ testScripts/emClarity.m | 11 +- testScripts/recScript.m | 130 ++++++++++++++++++ 18 files changed, 371 insertions(+), 158 deletions(-) create mode 100644 testScripts/recScript.m diff --git a/@MRCImage/SAVE_IMG.m b/@MRCImage/SAVE_IMG.m index 5b16da71..d273361f 100644 --- a/@MRCImage/SAVE_IMG.m +++ b/@MRCImage/SAVE_IMG.m @@ -38,7 +38,7 @@ if nargin > 1 newFilename = varargin{2}; - mRCImage = close(mRCImage); + mRCImage = close(mRCImage); mRCImage.filename = newFilename; % Check to see if the file exists @@ -93,7 +93,12 @@ else flgComplex = false; end - + % if isa(mRCImage.volume, 'half') + % mrcImage.volume = typecast(mRCImage.volume, 'uint16'); + % end + if strcmp(modeStr, 'half') + modeStr = 'uint16'; + end nElements = numel(mRCImage.volume); if mRCImage.header.mode == 0 && getWriteBytesAsSigned(mRCImage) diff --git a/@MRCImage/getModeString.m b/@MRCImage/getModeString.m index d010e9d1..fbd56a10 100644 --- a/@MRCImage/getModeString.m +++ b/@MRCImage/getModeString.m @@ -35,6 +35,10 @@ modeString = 'int16*2'; % used for complex short ints case 4 modeString = 'float32*2'; % used for complex floating point +case 6 + modeString = 'uint16'; + case 12 + modeString = 'half'; % 16-bit floating point otherwise PEETError('Unsupported MRCImage mode %d!', mRCImage.header.mode); end diff --git a/@MRCImage/getVolume.m b/@MRCImage/getVolume.m index 4db188a3..aee8bd78 100644 --- a/@MRCImage/getVolume.m +++ b/@MRCImage/getVolume.m @@ -163,6 +163,11 @@ else flgComplex = 0; end +% fread doesn't yet recognize "half" +% read as uint16 and then typecast to half later +if strcmp(modeStr, 'half') + modeStr = 'uint16'; +end % Changed counters and index variables to be clear to me BAH 2017-11-22 % l --> nSlice, k --> iSlice @@ -294,5 +299,9 @@ mRCImage.fid = []; end +if mRCImage.header.mode == 12 + vol = half.typecast(vol); +end + end diff --git a/@MRCImage/loadVolume.m b/@MRCImage/loadVolume.m index e4cb3e09..3ad1d269 100644 --- a/@MRCImage/loadVolume.m +++ b/@MRCImage/loadVolume.m @@ -31,6 +31,11 @@ nVoxels = mRCImage.header.nX * mRCImage.header.nY * mRCImage.header.nZ; modeStr = getModeString(mRCImage); +% fread doesn't yet recognize "half" +% read as uint16 and then typecast to half later +if strcmp(modeStr, 'half') + modeStr = 'uint16'; +end if strcmp(modeStr, 'int16*2') || strcmp(modeStr, 'float32*2') % handle reading complex volume modeStr = modeStr(1 : end - 2); @@ -62,7 +67,9 @@ mRCImage.flgVolume = 1; - +if mRCImage.header.mode == 12 + mRCImage.volume = half.typecast(mRCImage.volume); +end mRCImage.volume = reshape(mRCImage.volume, ... mRCImage.header.nX, ... mRCImage.header.nY, ... diff --git a/@MRCImage/private/getModeBytes.m b/@MRCImage/private/getModeBytes.m index e9c670b4..d5282f2c 100644 --- a/@MRCImage/private/getModeBytes.m +++ b/@MRCImage/private/getModeBytes.m @@ -27,15 +27,19 @@ switch mRCImage.header.mode case 0 - nBytes = 1; + nBytes = 1; % byte case 1 - nBytes = 2; + nBytes = 2; % int16 case 2 - nBytes = 4; + nBytes = 4; % single case 3 - nBytes = 4; + nBytes = 4; % complex int16 case 4 - nBytes = 8; + nBytes = 8; % complex single + case 6 + nBytes = 2; % uint16 + case 12 + nBytes = 2; % half (16-bit float) otherwise nBytes = -1; end diff --git a/@MRCImage/private/readHeader.m b/@MRCImage/private/readHeader.m index 9c54f7c9..74658910 100644 --- a/@MRCImage/private/readHeader.m +++ b/@MRCImage/private/readHeader.m @@ -59,6 +59,12 @@ [fname, perm, fileEndianFormat] = fopen(mRCImage.fid); %#ok if strcmp('ieee-be', fileEndianFormat) == 1 mRCImage.endianFormat = 'ieee-le'; +elseif strcmp('ieee-be.l64', fileEndianFormat) == 1 + mRCImage.endianFormat = 'ieee-le.l64'; + elseif strcmp('ieee-le', fileEndianFormat) == 1 + mRCImage.endianFormat = 'ieee-be'; + elseif strcmp('ieee-le.l64', fileEndianFormat) == 1 + mRCImage.endianFormat = 'ieee-be.l64'; else mRCImage.endianFormat = 'ieee-be'; end @@ -104,12 +110,8 @@ mRCImage.header.minDensity = fread(mRCImage.fid, 1, 'float32'); mRCImage.header.maxDensity = fread(mRCImage.fid, 1, 'float32'); mRCImage.header.meanDensity = fread(mRCImage.fid, 1, 'float32'); -mRCImage.header.spaceGroup = fread(mRCImage.fid, 1, 'int16'); -mRCImage.header.nSymmetryBytes = fread(mRCImage.fid, 1, 'int16'); -if debug - fprintf(debugFD, 'nSymmetry bytes %d\n', mRCImage.header.nSymmetryBytes); -end - +mRCImage.header.spaceGroup = fread(mRCImage.fid, 1, 'int32'); +% nBytesExtended is called nsymbt in the 2014 MRC Standard mRCImage.header.nBytesExtended = fread(mRCImage.fid, 1, 'int32'); if debug fprintf(debugFD, 'nBytesExtended %d\n', mRCImage.header.nBytesExtended); @@ -117,7 +119,7 @@ % MRC EXTRA section mRCImage.header.creatorID = fread(mRCImage.fid, 1, 'int16'); -junk = fread(mRCImage.fid, 30, 'uchar'); %#ok +mRCImage.header.extraInfo1 = fread(mRCImage.fid, 30, 'uchar'); mRCImage.header.nBytesPerSection = fread(mRCImage.fid, 1, 'int16'); if debug fprintf(debugFD, 'nBytesPerSection %d\n', mRCImage.header.nBytesPerSection); @@ -126,7 +128,7 @@ if debug fprintf(debugFD, 'serialEMType %d\n', mRCImage.header.serialEMType); end -junk = fread(mRCImage.fid, 20, 'uchar'); %#ok +mRCImage.header.extraInfo2 = fread(mRCImage.fid, 20, 'uchar'); % IMOD stamp / flags for deciding whether to read / write mode 0 files % as signed or unsigned bytes (added in PEET 1.8.0) diff --git a/@MRCImage/private/setVolumeAndHeaderFromVolume.m b/@MRCImage/private/setVolumeAndHeaderFromVolume.m index 6ba7ae79..5c5279e7 100644 --- a/@MRCImage/private/setVolumeAndHeaderFromVolume.m +++ b/@MRCImage/private/setVolumeAndHeaderFromVolume.m @@ -19,38 +19,45 @@ if isa(volume, 'uint8') mRCImage.header.mode = 0; - mRCImage.volume = volume; -end -if isa(volume, 'int16') +elseif isa(volume, 'int16') if isreal(volume) mRCImage.header.mode = 1; else mRCImage.header.mode = 3; end - mRCImage.volume = volume; -end -if isa(volume, 'single') +elseif isa(volume, 'single') if isreal(volume) mRCImage.header.mode = 2; else mRCImage.header.mode = 4; end - mRCImage.volume = volume; -end -if isa(volume, 'double') +elseif isa(volume, 'double') if ~isreal(volume) - PEETError('Double precision complex images are not supported!'); + PEETError('Double precision complex volumes are not supported!'); end mRCImage.header.mode = 2; - mRCImage.volume = single(volume); +elseif isa(volume, 'uint16') + if isreal(volume) + mRCImage.header.mode = 6; + else + PEETError('Complex uint16 volumes are not supported!'); + end +elseif isa(volume, 'half') + if isreal(volume) + mRCImage.header.mode = 12; + else + PEETError('Complex half-precision volumes are not supported!'); + end +else + PEETError('Volume must be uint8, int16, unit16, half, single, or double!'); end -% Added BAH - setting mRCImage.volume is just a pointer UNTIL we work with it. -% The volume will be duplicated in memory later, so clear "volume" in order -% to avoid wasting large chunks of memory. Change subsequent references from -% "volume" to "mRCImage.volume" -clear volume - + +if isa(volume, 'double') + mRCImage.volume = single(volume); % Store float*64 as float*32 +else + mRCImage.volume = volume; +end % TODO: is this the correct/best way to set the values mRCImage.header.nX = size(mRCImage.volume, 1); mRCImage.header.nY = size(mRCImage.volume, 2); @@ -79,8 +86,10 @@ mRCImage.header.imodFlags = writeBytesAsSigned; mRCImage.header.creatorID = int16(0); +mRCImage.header.extraInfo1 = char(zeros(1, 30, 'uint8')); mRCImage.header.nBytesPerSection = int16(0); mRCImage.header.serialEMType = int16(0); +mRCImage.header.extraInfo2 = char(zeros(1, 20, 'uint8')); mRCImage.header.idtype = 0; mRCImage.header.lens = 0; mRCImage.header.ndl = 0; diff --git a/@MRCImage/private/writeHeader.m b/@MRCImage/private/writeHeader.m index 2108769b..35a8149e 100644 --- a/@MRCImage/private/writeHeader.m +++ b/@MRCImage/private/writeHeader.m @@ -51,7 +51,7 @@ status = fseek(mRCImage.fid, 0, 'bof'); if status disp('Could not move the file pointer to the begining '); - PEETError('Could not seek to beginning of file id %d!', mRCImage.fid); + PEETError('Could not seek to beginning of file id %d', mRCImage.fid); end % Write out the dimensions of the data @@ -78,20 +78,19 @@ writeAndCheck(mRCImage.fid, mRCImage.header.maxDensity, 'float32'); writeAndCheck(mRCImage.fid, mRCImage.header.meanDensity, 'float32'); -writeAndCheck(mRCImage.fid, mRCImage.header.spaceGroup, 'int16'); -writeAndCheck(mRCImage.fid, mRCImage.header.nSymmetryBytes, 'int16'); +writeAndCheck(mRCImage.fid, mRCImage.header.spaceGroup, 'int32'); writeAndCheck(mRCImage.fid, mRCImage.header.nBytesExtended, 'int32'); % MRC EXTRA section writeAndCheck(mRCImage.fid, mRCImage.header.creatorID, 'int16'); -writeAndCheck(mRCImage.fid, char(zeros(1, 30)), 'uchar'); +writeAndCheck(mRCImage.fid, mRCImage.header.extraInfo1, 'uchar'); writeAndCheck(mRCImage.fid, mRCImage.header.nBytesPerSection, 'int16'); writeAndCheck(mRCImage.fid, mRCImage.header.serialEMType, 'int16'); -writeAndCheck(mRCImage.fid, char(zeros(1, 20)), 'uchar'); +writeAndCheck(mRCImage.fid, mRCImage.header.extraInfo2, 'uchar'); mRCImage.header.imodStamp = defaultIMODStamp(); writeAndCheck(mRCImage.fid, mRCImage.header.imodStamp, 'int32'); -if getWriteBytesAsSigned(mRCImage); +if getWriteBytesAsSigned(mRCImage) mRCImage.header.imodFlags = ... int32(bitor(uint32(mRCImage.header.imodFlags), 1)); end @@ -126,12 +125,12 @@ end % If there's room, add a label indicating writing by PEET -%if mRCImage.header.nLabels < 10 -% msg = ['Written by PEET / MatTomo ' datestr(now)]; -% writeAndCheck(mRCImage.fid, msg, 'uchar'); -% writeAndCheck(mRCImage.fid, char(blanks(80 - length(msg))), 'uchar'); -% mRCImage.header.nLabels = mRCImage.header.nLabels + 1; -%end +if mRCImage.header.nLabels < 10 +msg = ['Written by PEET / MatTomo ' datestr(now)]; +writeAndCheck(mRCImage.fid, msg, 'uchar'); +writeAndCheck(mRCImage.fid, char(blanks(80 - length(msg))), 'uchar'); +mRCImage.header.nLabels = mRCImage.header.nLabels + 1; +end % Use blank messages for the remainder for iJunk = mRCImage.header.nLabels+1:10 @@ -146,7 +145,11 @@ % Simple error checking write function writeAndCheck(fid, matrix, precision) nElements = numel(matrix); - count = fwrite(fid, matrix, precision); + if strcmp(precision, 'half') + count = fwrite(fid, matrix, 'uint16'); + else + count = fwrite(fid, matrix, precision); + end if count ~= nElements error('Matrix contains %d elements, but only wrote %d',nElements, count); end diff --git a/@MRCImage/save.m b/@MRCImage/save.m index 7ed2d0d1..7bbd3727 100644 --- a/@MRCImage/save.m +++ b/@MRCImage/save.m @@ -47,7 +47,16 @@ % Write out the volume if it is not already on the disk if mRCImage.flgVolume - modeStr = getModeString(mRCImage); + modeStr = getModeString(mRCImage); + if strcmp(modeStr, 'half') + %PEETError('Sorry, writing half-precision files is not supported!') + + % fwrite doesn't yet recognize "half" + % typecast to uint16 before writing, then typecast back later + modeStr = 'uint16'; + mRCImage.volume = typecast(mRCImage.volume, 'uint16'); + + end if strcmp(modeStr, 'int16*2') || strcmp(modeStr, 'float32*2') modeStr = modeStr(1 : end - 2); flgComplex = true; @@ -91,10 +100,17 @@ else % normal (not complex) data count = fwrite(mRCImage.fid, mRCImage.volume, modeStr); if count ~= nElements + if mRCImage.header.mode == 12 + mRCImage.volume = typecast(mRCImage.volume, 'half'); + end fprintf('Matrix contains %d but only wrote %d elements\n', ... nElements, count); PEETError('Failed writing matrix!'); end end end +if mRCImage.header.mode == 12 + mRCImage.volume = half.typecast(mRCImage.volume); +end + close(mRCImage); diff --git a/@MRCImage/showHeader.m b/@MRCImage/showHeader.m index 0262e1d4..fe49f21e 100644 --- a/@MRCImage/showHeader.m +++ b/@MRCImage/showHeader.m @@ -58,10 +58,15 @@ function showHeader(mRCImage) fprintf('rms density:\t\t\t%d\n', mRCImage.header.densityRMS); fprintf('space group:\t\t\t%d\n', mRCImage.header.spaceGroup); -fprintf('# symmetry bytes:\t\t%d\n', mRCImage.header.nSymmetryBytes); - fprintf('# extended header bytes:\t%d\n', mRCImage.header.nBytesExtended); fprintf('creator ID:\t\t\t%d\n', mRCImage.header.creatorID); + +if strcmp(mRCImage.header.extraInfo1, char(zeros(1, 30, 'uint8'))) + fprintf('extended header info1:'); + fprintf('%x ', mRCImage.header.extraInfo1); + fprintf('\n'); +end + fprintf('Extended header bytes/section:\t%d\n', ... mRCImage.header.nBytesPerSection); fprintf('Serial EM data type:\t\t%d\n', mRCImage.header.serialEMType); @@ -70,6 +75,12 @@ function showHeader(mRCImage) fprintf('IMOD flags: \t\t\t%d\n', mRCImage.header.imodFlags); end +if strcmp(mRCImage.header.extraInfo2, char(zeros(1, 20, 'uint8'))) + fprintf('extended header info2:'); + fprintf('%x ', mRCImage.header.extraInfo2); + fprintf('\n'); +end + fprintf('X origin:\t\t\t%d\n', mRCImage.header.xOrigin); fprintf('Y origin:\t\t\t%d\n', mRCImage.header.yOrigin); fprintf('Z origin:\t\t\t%d\n', mRCImage.header.zOrigin); diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 6fd98bbf..d4b81fd5 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -317,15 +317,15 @@ iterList{gpuList(iGPU)}; end -% try -% EMC_parpool(nGPUs) -% catch -% delete(gcp('nocreate')) -% EMC_parpool(nGPUs) -% end - -% parfor iGPU = 1:nGPUs -for iGPU = 1:nGPUs %%revert +try +EMC_parpool(nGPUs) +catch +delete(gcp('nocreate')) +EMC_parpool(nGPUs) +end + +parfor iGPU = 1:nGPUs +% for iGPU = 1:nGPUs %%revert for iTilt = iterList{gpuList(iGPU)} nTomos = 0; @@ -384,8 +384,8 @@ % All data is handled through disk i/o so everything unique created in the % parfor is also destroyed there as well. -% parfor iGPU = 1:nGPUs -for iGPU = 1:nGPUs %%revert +parfor iGPU = 1:nGPUs +% for iGPU = 1:nGPUs %%revert % for iGPU = 1:nGPUs gpuDevice(gpuList(iGPU)); @@ -462,19 +462,19 @@ iCoords = tiltRecGeom{iTilt}; % iCoords will be a cell indexed by each tomo wwith a struct .tomoCoords else - nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; - % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; - iCoords = cell(nTomos,1); - for iCoordIdx = 1:nTomos - iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(iTomoList{iCoordIdx}); - % FIXME + if (recon_for_tomoCPR) + nTomos = 1; + iCoords = {subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{1})}; + else + nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; + % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; + iCoords = cell(nTomos,1); + for iCoordIdx = 1:nTomos + iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(iTomoList{iCoordIdx}); + % FIXME + end end - % else - % if (recon_for_tomoCPR) - % nTomos = 1; - % else - % end end @@ -494,7 +494,7 @@ maskedStack = single(getVolume(MRCImage(inputStack))); if (recon_for_subTomo) - [ ~, specimen_NX_nm, tomoIdx, ~ ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... + [ ~, specimen_NZ_nm, tomoIdx, ~ ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... 0,1); @@ -504,8 +504,9 @@ NY = size(maskedStack,2); % NY = size(maskedStack,2)-1; - NZ = floor(reconstructionParameters(1)) - specimen_NX_nm = NZ; + NZ = floor(reconstructionParameters(1)); + specimen_NZ_nm = NZ * emc.pixel_size_angstroms / 10; + % iCoords = [NX,0,NY-1,NZ,0,0]; tomoIdx = 1; % FIXME @@ -518,7 +519,7 @@ iCoords{iCoordIdx}.tomoCoords.NZ = NZ * samplingRate; end else - [ ~, specimen_NX_nm, tomoIdx, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... + [ ~, specimen_NZ_nm, tomoIdx, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... 0,1); @@ -527,7 +528,7 @@ if ( flg2dCTF || recon_for_tomoCPR) n_slabs_to_reconstruct = 1; - ctf3dDepth = specimen_NX_nm * 10 ^ -9; + ctf3dDepth = specimen_NZ_nm * 10 ^ -9; else % TODO: for very thick specimen, this may be preventing the avg from getting to high enough % resolution to be useful. So far, this is only optimized on in vitro samples. @@ -536,7 +537,7 @@ [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... TLT(1,17),TLT(1,18),abs(TLT(1,15)), ... 2048, TLT(1,19), ... - resTarget,specimen_NX_nm*10, ... + resTarget,specimen_NZ_nm*10, ... dampeningMax,CYCLE); fprintf('\n\nCalculated a ctfDepth of %2.2f nm for %s\n\n',ctf3dDepth*10^9,tiltList{iTilt}); if (ctf3dDepth > emc.max_ctf3dDepth) @@ -547,7 +548,7 @@ % the mean defocus determination, although this could be corrected using % knowledge of particle positions given assurance that particles are the % primary source of signal (and not carbon for example). - n_slabs_to_reconstruct = ceil(specimen_NX_nm/(ctf3dDepth*10^9)); + n_slabs_to_reconstruct = ceil(specimen_NZ_nm/(ctf3dDepth*10^9)); % max odd number n_slabs_to_reconstruct = n_slabs_to_reconstruct + ~mod(n_slabs_to_reconstruct,2); end @@ -560,7 +561,7 @@ if (recon_for_subTomo) - [ avgZ, specimen_NX_nm, tomoIdx, surfaceFit ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... + [ avgZ, specimen_NZ_nm, tomoIdx, surfaceFit ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... iTomoList,nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... slab_list, 0); @@ -602,7 +603,7 @@ [ correctedStack ] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... - specimen_NX_nm*10/emc.pixel_size_angstroms,flgDampenAliasedFrequencies,... + specimen_NZ_nm*10/emc.pixel_size_angstroms,flgDampenAliasedFrequencies,... preCombDefocus,samplingRate,... applyExposureFilter,surfaceFit,... useSurfaceFit,invertDose,... @@ -671,13 +672,13 @@ end % round down and then we'll add any extra needed to the final chunk - tiltChunkSize = floor(iCoords{thisTomo}.NY ./ samplingRate ./ emc.n_tilt_workers); + tiltChunkSize = floor(iCoords{iT}.NY ./ samplingRate ./ emc.n_tilt_workers); % This shoulid never happen, but to be safe - if (emc.n_tilt_workers > floor(iCoords{thisTomo}.NY ./ samplingRate)) + if (emc.n_tilt_workers > floor(iCoords{iT}.NY ./ samplingRate)) error('n_tilt_workers is greater than the number of slices in the tilt series'); end - y_i = floor(iCoords{thisTomo}.y_i ./ samplingRate); - y_f = floor(iCoords{thisTomo}.y_f ./ samplingRate); + y_i = floor(iCoords{iT}.y_i ./ samplingRate); + y_f = floor(iCoords{iT}.y_f ./ samplingRate); tiltChunks = y_i:tiltChunkSize:y_f; @@ -691,8 +692,7 @@ tiltChunks(1) = 0; n_slices_in_Y = tiltChunks(end) - tiltChunks(1) + 1; end - - + rCMD = sprintf(['tilt %s %s -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... super_sample, ... @@ -701,9 +701,9 @@ reconName, ... rawTLT, ... gpuList(iGPU), ... - floor(iCoords{thisTomo}.NX ./ samplingRate),... % WIDTH = NX + floor(iCoords{iT}.NX ./ samplingRate),... % WIDTH = NX floor(round(slab_list{iT}(iSection,5))), ... % THICKNESS = NZ - iCoords{thisTomo}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X + iCoords{iT}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X slab_list{iT}(iSection,6)); @@ -973,7 +973,7 @@ function [correctedStack] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... avgZ,TLT,pixel_size_angstroms,maskedStack,... - specimen_NX_nm,flgDampenAliasedFrequencies,... + specimen_NZ_nm,flgDampenAliasedFrequencies,... preCombDefocus,samplingRate,... applyExposureFilter,surfaceFit,... useSurfaceFit,invertDose, ... @@ -1051,8 +1051,7 @@ if (filterProjectionsForTomoCPRBackground ~= 0) bpFilter = BH_bandpass3d(fastFTSize,0, 0, filterProjectionsForTomoCPRBackground, 'GPU',pixel_size_angstroms); - fprintf('Filtering input projections to %f angstroms with %f pixel size\n',bpFilter,pixel_size_angstroms); - + % fprintf('Filtering input projections to %f angstroms with %f pixel size\n',bpFilter,pixel_size_angstroms); else bpFilter = 1; end @@ -1060,7 +1059,7 @@ for iPrj = 1:nPrjs - maxEval = cosd(TLT(iPrj,4)).*(d1/2) + specimen_NX_nm./2*abs(sind(TLT(iPrj,4))); + maxEval = cosd(TLT(iPrj,4)).*(d1/2) + specimen_NZ_nm./2*abs(sind(TLT(iPrj,4))); oX = emc_get_origin_index(d1); oY = emc_get_origin_index(d2); iEvalMask = floor(oX-maxEval):ceil(oX+maxEval); @@ -1182,7 +1181,7 @@ clear tile Hqz end -function [avgZ, specimen_NX_nm, tomoIdx,surfaceFit] = calcAvgZ(subTomoMeta, ... +function [avgZ, specimen_NZ_nm, tomoIdx,surfaceFit] = calcAvgZ(subTomoMeta, ... iCoords, ... tiltName, ... tomoList,... @@ -1206,14 +1205,14 @@ val_to_pass = 'dummy'; end -[ specimen_NX_nm, tomoIdx ] = emc_get_max_specimen_NZ(val_to_pass, ... +[ specimen_NZ_pixels, tomoIdx ] = emc_get_max_specimen_NZ(val_to_pass, ... iCoords, ... tomoList, ... nTomos, ... samplingRate); -specimen_NX_nm = specimen_NX_nm .* pixel_size_angstroms ./ 10; -fprintf('combining the thickness and shift on tilt %s, found a specimen_NX_nm %3.3f nm\n',tiltName,specimen_NX_nm); +specimen_NZ_nm = specimen_NZ_pixels .* pixel_size_angstroms ./ 10; +fprintf('combining the thickness and shift on tilt %s, found a specimen_NZ_nm %3.3f nm\n',tiltName,specimen_NZ_nm); if (calcMaxZ) return; @@ -1259,7 +1258,7 @@ % X in the Y frame means a vector from the Y lower left to the X origin % X origin wrt Y origin is a vector from the origin of Y to the X origin - reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iT}); tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... reconGeometry.dY_specimen_to_tomo, ... reconGeometry.dZ_specimen_to_tomo]; diff --git a/metaData/BH_skipClassAlignment.m b/metaData/BH_skipClassAlignment.m index 9bea8149..d6377bcd 100755 --- a/metaData/BH_skipClassAlignment.m +++ b/metaData/BH_skipClassAlignment.m @@ -39,7 +39,7 @@ if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - if (emc.multi_reference_alignment) + if (emc.multi_reference_alignment || emc.classification) if (emc.classification) subTomoMeta.(cycleNumber).('RawAlign') = subTomoMeta.(cycleNumber).('ClusterClsGeom'); else diff --git a/metaData/SAVE_IMG.m b/metaData/SAVE_IMG.m index 1765c652..5a162d6d 100644 --- a/metaData/SAVE_IMG.m +++ b/metaData/SAVE_IMG.m @@ -15,11 +15,12 @@ mRCImage = MRCImage(gather(vol)); -mRCImage.header.minDensity = imgMin; -mRCImage.header.maxDensity = imgMax; +% These are all expected to be singles +mRCImage.header.minDensity = single(imgMin); +mRCImage.header.maxDensity = single(imgMax); -mRCImage.header.meanDensity = imgMean; -mRCImage.header.densityRMS = imgRMS; +mRCImage.header.meanDensity = single(imgMean); +mRCImage.header.densityRMS = single(imgRMS); % mRCImage.writeHeader(mRCImage); Changed SAVE_IMG to write out the header no matter what. switch length(varargin) diff --git a/mexFiles/mexFFT.cu b/mexFiles/mexFFT.cu index 004d247c..f6f073bb 100644 --- a/mexFiles/mexFFT.cu +++ b/mexFiles/mexFFT.cu @@ -208,8 +208,6 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) mexMakeArrayPersistent(plhs[2]); - - if ( ! fwd_xform ) { mexErrMsgIdAndTxt("The cufft plan must be created on a forward transform.","womp"); } diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index a66eed73..b12f1671 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -10,8 +10,6 @@ emc = BH_parseParameterFile(PARAMETER_FILE); -% Only used in PCA pub right now -test_multi_ref_diffmap=true; %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of @@ -24,7 +22,6 @@ end -nRows = emc.n_scale_spaces; featureVector = cell(2,1); if flgGold featureVector{1,1} = emc.('Pca_coeffs_odd'); @@ -34,10 +31,7 @@ featureVector{1,1} end -nFeatures = size(featureVector{1,1}); -if (nFeatures(1) ~= nRows) - error('There should be a set of indices for each pca_scale_spaces, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') -end +nFeatures = size(featureVector{1,1}) nCores = BH_multi_parallelWorkers(emc.('nCpuCores')); @@ -128,9 +122,12 @@ end %%% experimental part of pcaMS - nScaleSpace = size(featureVector{iGold},1) - featureVector{1} - nFeatures = zeros(1,nScaleSpace) + nScaleSpace = size(coeffsUNTRIMMED,1); + nFeatures = zeros(1,nScaleSpace); + if (nFeatures(1) ~= nScaleSpace) + error('There should be a set of indices for each pca_scale_spaces, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') + end + % if length(relativeScale) ~= nScaleSpace % error('relativeScale has %d elements for %d scaleSpaces', ... @@ -143,7 +140,7 @@ nFeatures(iScale) = nnz(featureVector{iGold}(iScale,:)); end coeffMat = zeros(sum(nFeatures), nJ, 'single'); - nAdded = 0 + nAdded = 0; for iScale = 1:nScaleSpace fV = featureVector{iGold}(iScale,:); fV = sort(fV(fV~=0)) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 73b7aa05..53c9b3b7 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -18,7 +18,9 @@ buildTomo=1;% % % % % % % - +% If false, this is faster, simplifies the code and permits defocus estimation +% This will likely be removed in favor of deleting all the blockes under its control +use_background_estimate = false; % Default true, we don't need this after projection delete_background_estimate = true; @@ -246,22 +248,11 @@ skip_this_tilt_series_because_it_is_empty = false(nTomograms,1); % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); - tomoList = {}; - tomoIDX = 1; + tomoList = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).tomoList; tilt_filename = sprintf('%saliStacks/%s_ali%d.fixed', CWD, tiltNameList{iTiltSeries}, mapBackIter + 1); [~, tltName, tltExt] = fileparts(tilt_filename); tilt_binned_filename = sprintf('%scache/%s_bin%d%s', CWD, tltName, samplingRate, tltExt); - fn = fieldnames(subTomoMeta.mapBackGeometry.tomoName); - for iTomo = 1:numel(fn) - if strcmp(subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName, tiltNameList{iTiltSeries}) - % This is dumb, fix it to be explicit. - if (subTomoMeta.mapBackGeometry.tomoCoords.(fn{iTomo}).is_active) - tomoList{tomoIDX} = fn{iTomo}; - % Only increment if values found. - tomoIDX = tomoIDX + 1; - end - end - end + [~,tiltBaseName,~] = fileparts(tilt_filename); mbOUT{3} = tiltBaseName; @@ -424,38 +415,41 @@ sprintf('[%d,%d]',maxZ,samplingRate); tiltNameList{iTiltSeries}; - - backgroundName = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec',CWD,tiltNameList{iTiltSeries},1, samplingRate); - fprintf('In tomocpr, using background estimate %s\n\n',backgroundName); - send_backgroundLowPassResolution = 28; + if (use_background_estimate) + backgroundName = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec',CWD,tiltNameList{iTiltSeries},1, samplingRate); + fprintf('In tomocpr, using background estimate %s\n\n',backgroundName); + send_backgroundLowPassResolution = 28; - % TODO: investigate deviations from the default, which is to shut off the phakePhasePlate and to use a backgroundLowPassResolution of 28 - % Default false, we don't apply this filter - % if enabled, it currently only saves the filtered background estimate for visualization in addition to the normal version - % if (emc.save_mapback_classes) - % BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3, tmpCache); - % end - % FIXME: calling like this does not use the surface fit for the background - send_phakePhasePlateOption = [0,0]; - BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution, tmpCache); - - % re-initialize the parpool for each tilt series to free up mem. - delete(gcp('nocreate')) - EMC_parpool(nWorkers); - + % TODO: investigate deviations from the default, which is to shut off the phakePhasePlate and to use a backgroundLowPassResolution of 28 + % Default false, we don't apply this filter + % if enabled, it currently only saves the filtered background estimate for visualization in addition to the normal version + % if (emc.save_mapback_classes) + % BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3, tmpCache); + % end + % FIXME: calling like this does not use the surface fit for the background + send_phakePhasePlateOption = [0,0]; + BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution, tmpCache); + + % re-initialize the parpool for each tilt series to free up mem. + delete(gcp('nocreate')) + EMC_parpool(nWorkers); - avgTomo{1} = getVolume(MRCImage(backgroundName)); - if (delete_background_estimate) - system(sprintf('rm -f %s',backgroundName)); - end + avgTomo{1} = getVolume(MRCImage(backgroundName)); + avgTomo{1} = avgTomo{1} ./ (rmsScale*rms(avgTomo{1}(:))); + if (delete_background_estimate) + system(sprintf('rm -f %s',backgroundName)); + end + else + avgTomo{1} = zeros(reconstruction_size, 'single'); + end % if (use_background_estimate) for iRef = 1:nRefs refVol{1}{iRef} = gpuArray(refVol{1}{iRef}); refVol{2}{iRef} = gpuArray(refVol{2}{iRef}); particleMask{iRef} = gpuArray(particleMask{iRef}); end - avgTomo{1} = avgTomo{1} ./ (rmsScale*rms(avgTomo{1}(:))); + if (emc.save_mapback_classes) @@ -467,6 +461,10 @@ coordSTART = fopen(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3}),'w'); if (emc.save_mapback_classes) coordCLASS = fopen(sprintf('%smapBack%d/%s.coord_class',mbOUT{1:3}),'w'); + coordCLASS_perTomo = cell(nTomograms,1); + for iCoordClassPerTomo = 1:nTomograms + coordClass_perTomo{iCoordClassPerTomo} = fopen(sprintf('%smapBack%d/%s.coord_class',mbOUT{1:2},tomoList{iCoordClassPerTomo}),'w'); + end end defOUT = fopen(sprintf('%smapBack%d/%s.defAng',mbOUT{1:3}),'w'); end @@ -518,7 +516,6 @@ positionList = geometry.(tomoList{iTomo}); tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; - tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); @@ -536,10 +533,9 @@ sTY = floor(tiltHeader.nY ); iTLT = floor(tiltHeader.nZ); - tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconGeometry = masterTM.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... reconGeometry.dY_specimen_to_tomo, ... reconGeometry.dZ_specimen_to_tomo]; @@ -636,6 +632,7 @@ iMaskResamp = BH_resample3d(particleMask{iRefIDX},subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'); iAvgResamp = gather(iMaskResamp.*iAvgResamp); + if (emc.save_mapback_classes) if ~(emc.save_mapback_classes) @@ -674,7 +671,9 @@ 1+padVAL(1,2):end-padVAL(2,2),... 1+padVAL(1,3):end-padVAL(2,3)); catch - fprintf('Warning, subTomo %d appears to be out of bounds in mapBack?\n'); + fprintf('Warning, subTomo %d appears to be out of bounds in mapBack?\n', iSubTomo); + indVAL + size(avgTomo{1}) continue end @@ -688,6 +687,10 @@ % Save a non-rotated model with each class on its own object for visualization if (emc.save_mapback_classes) fprintf(coordCLASS,'%d 1 %0.4f %0.4f %0.4f\n', iClassIDX, subtomo_origin_wrt_tilt_origin'./samplingRate + binned_specimen_origin_in_specimen_frame'- emc.prjVectorShift'); + % Also save one for each tomo + fprintf(coordClass_perTomo{iTomo},'%d 1 %0.4f %0.4f %0.4f\n', iClassIDX, subtomo_origin_in_tomo_frame'./samplingRate - emc.prjVectorShift'); + + end for iPrj = 1:nPrjs @@ -737,8 +740,14 @@ '%smapBack%d/%s.coord_class %smapBack%d/%s_classIdx.3dfid'], ... mbOUT{1:3},mbOUT{1:3}); system(p2m); + for iCoordClassPerTomo = 1:nTomograms + fclose(coordClass_perTomo{iCoordClassPerTomo}); + p2m = sprintf(['point2model -sphere 6 -thick 6 -scat ',... + '%smapBack%d/%s.coord_class %smapBack%d/%s_classIdx.3dfid'], ... + mbOUT{1:2},tomoList{iCoordClassPerTomo},mbOUT{1:2},tomoList{iCoordClassPerTomo}); + system(p2m); + end end - p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 ',... '%smapBack%d/%s.coord %smapBack%d/%s.3dfid'], ... mbOUT{1:3},mbOUT{1:3}); diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 93ce6ba2..e70d178f 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -160,7 +160,8 @@ multiGPUs = 0; case 'cleanTemplateSearch' multiGPUs = 0; - + case 'segment' + multiGPUs = 0; otherwise emc = emC_testParse(varargin{2}); end @@ -187,6 +188,7 @@ '\nhelp - this message\n',... '\n\t\t for more details, emClarity help\n',... '\ncheck - system check for dependencies\n',... + '\nsegment - define subregions to reconstruct\n',... '\ninit - create a new project from template matching results.\n',... '\nautoAlign - align tilt-serie\n',... '\navg - average subtomograms\n',... @@ -224,6 +226,13 @@ else BH_checkInstall(getenv('BH_CHECKINSTALL')) end + case 'segment' + if emcProgramHelp || ... + length(varargin) ~= 2 + fprintf(['\nUsage: emClarity segment build (make bin10 tomos)\n\tor\n\temClarify segment recon (convert model files)\n']); + else + recScript(varargin{2}); + end case 'init' if emcProgramHelp || ... length(varargin) < 2 && length(varargin)> 5 diff --git a/testScripts/recScript.m b/testScripts/recScript.m new file mode 100644 index 00000000..6a285a96 --- /dev/null +++ b/testScripts/recScript.m @@ -0,0 +1,130 @@ +function [] = recScript(wanted_op) + +modBin=10; % this could be changed, but works well in most cases and is ill advised. +modThick=300; % this could be changed, particularly if your sample is thicker + % than 3000 pixels. But then it is probably a bad candidate for + % high resolution tomography anyhow ( assuming your pixel size > 1Ang) + +if strcmpi(wanted_op, 'build') + % make a bin10 directory and create bin 10 tomos for evaluation. + mkdir('bin10'); + + if ~isfolder('aliStacks') + error('No aliStacks folder found. Please run the alignment script first.'); + end + + stack_names = dir('aliStacks/*.fixed'); + stack_names = {stack_names.name}; + nParProcesses = 4; + try + EMC_parpool(nParProcesses+1) + catch + delete(gcp('nocreate')) + EMC_parpool(nParProcesses+1) + end + + parfor iStack = 1:length(stack_names) + fprintf('Working on stack %s\n', stack_names{iStack}); + bin_cmd = sprintf('newstack -bin %d aliStacks/%s bin10/%s_bin10.fixed', modBin, stack_names{iStack}, stack_names{iStack}(1:end-11)); + rec_cmd = sprintf('tilt -input bin10/%s_bin10.fixed -output bin10/%s_bin10.rec -TILTFILE fixedStacks/%s.tlt -RADIAL 0.15,0.05 -UseGPU 0 -THICKNESS %d -RotateBy90', ... + stack_names{iStack}(1:end-11), stack_names{iStack}(1:end-11), stack_names{iStack}(1:end-11), modThick); + errmsg = system(sprintf('%s > /dev/null', bin_cmd)); + if errmsg + fprintf('Error binning the stack'); + system(sprintf('%s', bin_cmd)); + error('Error binning the stack'); + end + errmsg = system(sprintf('%s > /dev/null', rec_cmd)); + if errmsg + fprintf('Error reconstructing the stack'); + system(sprintf('%s', rec_cmd)); + error('Error reconstructing the stack'); + end + end +elseif strcmpi(wanted_op, 'recon') + + mkdir('recon'); + + if ~isfolder('bin10') + error('No bin10 folder found. Please run recScript(build) first.'); + end + + mod_names = dir('bin10/*.mod'); + mod_names = {mod_names.name}; + if isempty(mod_names) + error('No mod files found in the bin10 folder'); + end + + for iMod = 1:length(mod_names) + mod_basename = mod_names{iMod}(1:end-10); + if ~isfile(sprintf('aliStacks/%s_ali1.fixed', mod_basename)) + error('Did not find the aligned, fixed stack at %s_ali1.fixed', mod_basename); + end + header = getHeader(MRCImage(sprintf('aliStacks/%s_ali1.fixed', mod_basename), 0)); + + if ~isfolder('fixedStacks') + error('No fixedStacks folder found. Please run the alignment script first.'); + end + + if isfile(sprintf('fixedStacks/%s.local', mod_basename)) + local_file = sprintf('-LOCALFILE fixedStacks/%s.local', mod_basename); + else + local_file = ''; + end + + cmd = sprintf('model2point -contour bin10/%s bin10/mod.tmp', mod_names{iMod}); + msg = system(sprintf('%s > /dev/null', cmd)); + if msg + fprintf('Error running model2point'); + system(sprintf('%s', cmd)); + error('Error running model2point'); + end + mod_f = importdata('bin10/mod.tmp'); + mod_f = sortrows(mod_f, 1); + if mod(size(mod_f,1), 6) ~= 0 + error('The number of contours is not a multiple of 6 %s', mod_names{iMod}); + end + rmpath('bin10/mod.tmp'); + numParticles = size(mod_f, 1)/6; + fid = fopen(sprintf('recon/%s_recon.coords', mod_basename), 'w'); + fprintf(fid, '%s\n', mod_basename); + fprintf(fid, '%d\n', numParticles); + + for iRegion = 1:numParticles + xLt = mod_f((iRegion-1)*6+1, 2)*modBin; + xHt = mod_f((iRegion-1)*6+2, 2)*modBin; + yLt = mod_f((iRegion-1)*6+3, 3)*modBin; + yHt = mod_f((iRegion-1)*6+4, 3)*modBin; + zLt = mod_f((iRegion-1)*6+5, 4)*modBin; + zHt = mod_f((iRegion-1)*6+6, 4)*modBin; + + xL = min(xLt, xHt); + xH = max(xLt, xHt); + yL = min(yLt, yHt); + yH = max(yLt, yHt); + zL = min(zLt, zHt); + zH = max(zLt, zHt); + + width = int32(xH-xL); + height = int32(yH-yL); + yLow = int32(yL); + yTop = int32(yLow + height - 1); + oX = int32((xL + (width/2.0)) - header.nX/2.0); + thickness = int32(sqrt((zH - zL)^2)); + oZ = int32((zL+thickness/2.0) - (modThick*modBin)/2.0); + oY = int32(yL + height/2.0 - header.nY/2.0); + + fprintf(fid, '%d\n%d\n%d\n%d\n%d\n%d\n', width, height, thickness, oX, oY, oZ); + + end + end + +else + error('Invalid operation. Please use "build" or "recon"'); +end + + + + + +end \ No newline at end of file From 3430f21e03c122660c872adaf479a263fede5537 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 28 Feb 2024 14:04:36 -0500 Subject: [PATCH 049/151] wip --- statistics/BH_clusterPub.m | 4 +--- transformations/BH_average3d.m | 5 +++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index b12f1671..534217b3 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -124,9 +124,7 @@ %%% experimental part of pcaMS nScaleSpace = size(coeffsUNTRIMMED,1); nFeatures = zeros(1,nScaleSpace); - if (nFeatures(1) ~= nScaleSpace) - error('There should be a set of indices for each pca_scale_spaces, is Pca_coeffis using ; vs , to ensure a matrix vs vector?') - end + % if length(relativeScale) ~= nScaleSpace diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 723f437a..bb8964fb 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -1526,6 +1526,11 @@ end iOdd = iRef; iEve = iRef; + + % Just in case we want to average without the shape mask after classification + if strcmpi(STAGEofALIGNMENT, 'Cluster') + mskParams{5} = emc.fsc_shape_mask; + end if (flgFinalAvg) % negative to combine but NOT apply the xform to the odd set From 42eb9407ccaac6b2a1bb051459729cc584133043 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 28 Feb 2024 15:42:16 -0500 Subject: [PATCH 050/151] Added mex function to convert to halfprecision and typecast to a uint16 array to fp16 can be saved. Added a complementary OPEN_IMG function to convert the file to given precision after loading) --- metaData/OPEN_IMG.m | 14 ++++++++++++++ metaData/SAVE_IMG.m | 15 +++++++++++++++ mexFiles/include/core_headers.cuh | 2 ++ mexFiles/mexCompile.m | 2 +- 4 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 metaData/OPEN_IMG.m diff --git a/metaData/OPEN_IMG.m b/metaData/OPEN_IMG.m new file mode 100644 index 00000000..cdf3900e --- /dev/null +++ b/metaData/OPEN_IMG.m @@ -0,0 +1,14 @@ +function [ vol ] = OPEN_IMG(precision, filename, varargin) + + vol = getVolume(MRCImage(filename, precision), varargin{:}); + if strcmp(precision, 'single') + vol = single(vol); + elseif strcmp(precision, 'double') + vol = double(vol); + elseif strcmp(precision, 'half') + vol = half(vol); + else + error('Unknown precision'); + end + +end \ No newline at end of file diff --git a/metaData/SAVE_IMG.m b/metaData/SAVE_IMG.m index 5a162d6d..84e774e3 100644 --- a/metaData/SAVE_IMG.m +++ b/metaData/SAVE_IMG.m @@ -21,6 +21,21 @@ mRCImage.header.meanDensity = single(imgMean); mRCImage.header.densityRMS = single(imgRMS); + +if (length(varargin) > 0) + if isa(varargin{1}, 'cell') + if strcmp(varargin{1}{2}, 'half') + mRCImage.header.mode = 12; + tmp_vol = zeros(size(vol), 'uint16'); + mexFP16(single(vol), tmp_vol); + mRCImage.volume = tmp_vol; + varargin{1} = varargin{1}{1}; + else + error('Unknown option for SAVE_IMG, should be {filename, half}'); + end + end +end + % mRCImage.writeHeader(mRCImage); Changed SAVE_IMG to write out the header no matter what. switch length(varargin) diff --git a/mexFiles/include/core_headers.cuh b/mexFiles/include/core_headers.cuh index 6dc5f612..47c50dac 100644 --- a/mexFiles/include/core_headers.cuh +++ b/mexFiles/include/core_headers.cuh @@ -3,6 +3,8 @@ #include "gpu/mxGPUArray.h" #include "matrix.h" +#include "ieee-754-half/half.hpp" + // Cuda includes #include #include diff --git a/mexFiles/mexCompile.m b/mexFiles/mexCompile.m index 9bc41862..64324841 100644 --- a/mexFiles/mexCompile.m +++ b/mexFiles/mexCompile.m @@ -16,7 +16,7 @@ if nargin > 0 mexFILE = varargin; else - mexFILE = {'mexCTF','mexFFT','mexXform3d','mexSF3D'}; + mexFILE = {'mexCTF','mexFFT','mexXform3d','mexSF3D', 'mexFP16'}; % mexFILE = {'mexXform3d'}; end % --extra-device-vectorization From d498fb43a66439ab0ae74937c4819133a1d9a248 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 29 Feb 2024 07:58:37 -0500 Subject: [PATCH 051/151] MPS: changed ctf3d to be able to run multiple processes per gpu and fixed re-ordering in multi_parallelJobs to make sure alignment work was spread out more evenly. --- alignment/BH_alignRaw3d_v2.m | 31 ++++++++++++--------- ctf/BH_ctf_Correct3d.m | 48 ++++++++++++++++++-------------- logicals/BH_multi_parallelJobs.m | 27 ++++++++++++++---- 3 files changed, 67 insertions(+), 39 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 13b174be..75b42496 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -185,8 +185,11 @@ BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); - -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, sizeCalc(1), emc.nCpuCores); +if (flgStartThird) + [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, sizeCalc(1), emc.nCpuCores, [cycle_numerator,cycle_denominator]); +else + [ nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, sizeCalc(1), emc.nCpuCores); +end if ( flgReverseOrder ) % Flip the order for reverse processing on a second machine. This will also disable saving of % of the metadata so there aren't conflicts. @@ -194,19 +197,21 @@ iterList{iParProc} = flip(iterList{iParProc}); end -elseif ( flgStartThird ) - - for iParProc = 1:nParProcesses - % Note the use of floor is more like ceiling here (rounds away from - % zero) - nParts = ceil(length(iterList{iParProc}) ./ cycle_denominator); - fIDX = 1+(cycle_numerator - 1)*nParts; - lIDX = min(cycle_numerator*nParts,length(iterList{iParProc})); - iterList{iParProc} = iterList{iParProc}(fIDX:lIDX); - end +% elseif ( flgStartThird ) + +% % Divide up the list in two parts, +% % for iParProc = 1:nParProcesses +% % % Note the use of floor is more like ceiling here (rounds away from +% % % zero) +% % % FIXME: when combined with sorted list this is not doing what it should be. +% % nParts = ceil(length(iterList{iParProc}) ./ cycle_denominator); +% % fIDX = 1+(cycle_numerator - 1)*nParts; +% % lIDX = min(cycle_numerator*nParts,length(iterList{iParProc})); +% % iterList{iParProc} = iterList{iParProc}(fIDX:lIDX) +% % end -else +% else end diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index d4b81fd5..d250d861 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -312,22 +312,20 @@ iterList = cell(nGPUs,1); % If there is only one tilt, things break in a weird way nGPUs = min(nGPUs, nTilts); -for iGPU = 1:nGPUs - iterList{gpuList(iGPU)} = iGPU+(tiltStart-1):nGPUs:nTilts; - iterList{gpuList(iGPU)}; -end + +[ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores); try -EMC_parpool(nGPUs) + EMC_parpool(nParProcesses) catch -delete(gcp('nocreate')) -EMC_parpool(nGPUs) + delete(gcp('nocreate')) + EMC_parpool(nParProcesses) end -parfor iGPU = 1:nGPUs +parfor iParProc = 1:nParProcesses % for iGPU = 1:nGPUs %%revert - - for iTilt = iterList{gpuList(iGPU)} + iGPU = mod(iParProc,nGPUs); + for iTilt = iterList{iParProc} nTomos = 0; alreadyMade = 0; iTomoList = {}; @@ -337,7 +335,6 @@ % not present. TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); TLT = load(TLTNAME); - % fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); @@ -383,14 +380,15 @@ end % All data is handled through disk i/o so everything unique created in the -% parfor is also destroyed there as well. -parfor iGPU = 1:nGPUs +parfor iParProc = 1:nParProcesses + % for iGPU = 1:nGPUs %%revert + iGPU = mod(iParProc,nGPUs); % for iGPU = 1:nGPUs %%revert % for iGPU = 1:nGPUs - gpuDevice(gpuList(iGPU)); + gpuDevice(iGPU+1); % Loop over each tilt - for iTilt = iterList{gpuList(iGPU)} + for iTilt = iterList{iParProc} slab_list = {}; % if (recon_for_subTomo) @@ -420,7 +418,7 @@ TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt}, mapBackIter + 1 ); TLT = load(TLTNAME); - fprintf('iGPU %d and iTilt %d using TLT %s\n', iGPU, iTilt, TLTNAME); + fprintf('iParProc %d and iTilt %d using TLT %s\n', iParProc, iTilt, TLTNAME); iTomoList = {}; @@ -700,7 +698,7 @@ outputStack, ... reconName, ... rawTLT, ... - gpuList(iGPU), ... + iGPU, ... floor(iCoords{iT}.NX ./ samplingRate),... % WIDTH = NX floor(round(slab_list{iT}(iSection,5))), ... % THICKNESS = NZ iCoords{iT}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X @@ -1251,6 +1249,7 @@ zFull{iSection} = []; end +use_subtomo_z_positions = true; for iT = 1:nTomos @@ -1272,11 +1271,14 @@ % shouldn't be any removed particles at this stage but later there would be. subtomo_origin_z_in_tomo_frame = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,13); - % We should not get to this point if all subtomograms have been removed. + % We may get here if we have split a data set into several small classes so skip the centering on average if needed if isempty(subtomo_origin_z_in_tomo_frame) - error('No subtomograms found for %s',iTomoName); + use_subtomo_z_positions = false; + fprintf('No subtomograms found for %s',iTomoName); + continue; end + % shift from lower left to centered and include the tomos offset from the subtomo_origin_wrt_specimen_origin = subtomo_origin_z_in_tomo_frame - tomo_origin_in_tomo_frame(3) + tomo_origin_wrt_tilt_origin(3); subtomo_origin_wrt_specimen_origin = subtomo_origin_wrt_specimen_origin ./ samplingRate; @@ -1309,12 +1311,16 @@ end % loop over tomos -avgZ = totalZ / nSubTomos*pixel_size_angstroms / 10*10^-9; +if (nSubTomos == 0) + avgZ = 0; +else + avgZ = totalZ / nSubTomos*pixel_size_angstroms / 10*10^-9; +end for iSection = 1:n_slabs_to_reconstruct - if length(xFull{iSection}) >= 6 + if (use_subtomo_z_positions && length(xFull{iSection}) >= 6) surfaceFit{iSection} = fit([xFull{iSection}, yFull{iSection}],zFull{iSection},'poly22','Robust','on'); else surfaceFit{iSection} = 0; diff --git a/logicals/BH_multi_parallelJobs.m b/logicals/BH_multi_parallelJobs.m index 5ad4a1d6..3ca9ceb0 100644 --- a/logicals/BH_multi_parallelJobs.m +++ b/logicals/BH_multi_parallelJobs.m @@ -1,4 +1,4 @@ -function [nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, calcSize, flgAvg) +function [nParProcesses, iterList] = BH_multi_parallelJobs(nTomograms, nGPUs, calcSize, flgAvg, varargin) % Rough Scaling of processes by mem available. Assuming all gpus are @@ -21,6 +21,14 @@ else scaleMem = 2.2; end +do_split = false; +if nargin > 4 + split_jobs = varargin{1}; + do_split = true; + if (numel(split_jobs) ~= 2) + error('split_jobs must be a 2 element vector'); + end +end fprintf('found totalMem on GPU1 %3.3e, nWorkers %d, so scaling nProcs by %2.2f\n',totMem,pInfo.NumWorkers,scaleMem); % need to add restrainst on parpool size, and also try to balance based on number of tomograms. % even better o explicitly depend on memory available ( this will allow smaller cards to be used ) @@ -66,13 +74,22 @@ nParProcesses = min(nParProcesses,flgAvg); end -nIters = min(nParProcesses,nTomograms); +nWorkersPerGPU = min(nParProcesses,nTomograms); % fprintf('Using %d workers in %d batches\n',nParProcesses,(1+nTomograms)./nParProcesses); % Divide the tomograms up over each gpu -iterList = cell(nIters,1); -for iParProc = 1:nIters - iterList{iParProc} = iParProc:nIters:nTomograms; +if (do_split) + gpu_id = split_jobs(1); + n_gpus = split_jobs(2); + iterList = cell(nWorkersPerGPU,1); + for iParProc = 0:nWorkersPerGPU-1 + iterList{iParProc+1} = gpu_id + n_gpus*iParProc:nWorkersPerGPU*n_gpus:nTomograms; + end +else + iterList = cell(nWorkersPerGPU,1); + for iParProc = 1:nWorkersPerGPU + iterList{iParProc} = iParProc:nWorkersPerGPU:nTomograms; + end end end From d9c2707b9ddf60ae04cc4e274be89cdca22db5b8 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 29 Feb 2024 14:13:59 -0500 Subject: [PATCH 052/151] Switch all get volumes to the new OPEN_IMG funciton to handle FP16) --- alignment/BH_alignRaw3d_v2.m | 25 +++---------------- alignment/BH_fitBeads.m | 2 +- alignment/BH_runAutoAlign.m | 4 +-- ctf/BH_ctf_Correct3d.m | 12 ++++----- ctf/BH_ctf_Estimate.m | 2 +- ctf/BH_ctf_Refine2.m | 2 +- ctf/BH_ctf_Updatefft.m | 2 +- ctf/BH_runCtfFind.m | 2 +- metaData/OPEN_IMG.m | 7 +++++- statistics/BH_multi_loadAndMaskStack.m | 2 +- statistics/BH_pcaPub.m | 11 ++++---- statistics/BH_rocCurve.m | 5 +--- synthetic/BH_synthetic_mapBack.m | 15 +++++++---- synthetic/BH_to_cisTEM_mapBack.m | 2 +- testScripts/emClarity.m | 2 +- testScripts/recScript.m | 4 +-- transformations/BH_average3d.m | 4 +-- transformations/BH_montage4d.m | 4 +-- transformations/BH_montage4d_halfGrid.m | 4 +-- transformations/BH_multi_loadOrBin.m | 8 +++--- transformations/BH_multi_loadOrBuild.m | 2 +- transformations/BH_reScale3d.m | 2 +- transformations/BH_resample2d.m | 6 ++--- transformations/BH_resample3d.m | 2 +- transformations/BH_unStackMontage4d.m | 2 +- .../BH_unStackMontage4d_halfGrid.m | 2 +- 26 files changed, 62 insertions(+), 73 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 75b42496..ec035832 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -196,23 +196,6 @@ for iParProc = 1:nParProcesses iterList{iParProc} = flip(iterList{iParProc}); end - -% elseif ( flgStartThird ) - -% % Divide up the list in two parts, -% % for iParProc = 1:nParProcesses -% % % Note the use of floor is more like ceiling here (rounds away from -% % % zero) -% % % FIXME: when combined with sorted list this is not doing what it should be. -% % nParts = ceil(length(iterList{iParProc}) ./ cycle_denominator); -% % fIDX = 1+(cycle_numerator - 1)*nParts; -% % lIDX = min(cycle_numerator*nParts,length(iterList{iParProc})); -% % iterList{iParProc} = iterList{iParProc}(fIDX:lIDX) -% % end - - -% else - end if any(peakSearch > maskRadius) @@ -818,7 +801,7 @@ % first!!! TODO add a flag to check this. try particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iparticle = gpuArray(getVolume(MRCImage(particleOUT_name),[indVAL(1,1),indVAL(2,1)], ... + iparticle = gpuArray(OPEN_IMG('single',particleOUT_name,[indVAL(1,1),indVAL(2,1)], ... [indVAL(1,2),indVAL(2,2)], ... [indVAL(1,3),indVAL(2,3)],'keep')); catch @@ -829,9 +812,9 @@ end else - iparticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); + iparticle = gpuArray(OPEN_IMG('single', volumeData, [indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); end [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... padVAL(2,1:3), 'GPU', 'singleTaper'); diff --git a/alignment/BH_fitBeads.m b/alignment/BH_fitBeads.m index 95083208..62528ee8 100644 --- a/alignment/BH_fitBeads.m +++ b/alignment/BH_fitBeads.m @@ -21,7 +21,7 @@ edgeVal = 4; beadVal = -4; -input_ts = gpuArray(getVolume(MRCImage(img_name))); +input_ts = gpuArray(OPEN_IMG('single', img_name)); system(sprintf('model2point -contour %s %s.txt',input_name,input_name)); input_pos= gpuArray(load(sprintf('%s.txt',input_name))); output_pos = zeros(size(input_pos),'single'); diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index 1a1da359..46462708 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -100,7 +100,7 @@ fprintf("Stack in is %s\n",stackIN); inputMRC = MRCImage(stackIN,0); -inputStack = single(getVolume(inputMRC)); +inputStack = OPEN_IMG('single', inputMRC); skip_tilts_logical = []; if (skip_tilts) @@ -199,7 +199,7 @@ % Once we've done this, we want to work as if this is how the stack % came off the scope. system(sprintf('newstack -fromone -secs %d -rotate 90 fixedStacks/%s.fixed %s >/dev/null',iPrj,baseName,tmpFile)); - rotStack(:,:,iPrj) = getVolume(MRCImage(sprintf('%s',tmpFile))); + rotStack(:,:,iPrj) = OPEN_IMG('single', sprintf('%s',tmpFile)); system(sprintf('rm %s',tmpFile)); end diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index d250d861..00f6e3db 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -489,7 +489,7 @@ tiltList{iTilt},mapBackIter+1); end - maskedStack = single(getVolume(MRCImage(inputStack))); + maskedStack = OPEN_IMG('single', inputStack); if (recon_for_subTomo) [ ~, specimen_NZ_nm, tomoIdx, ~ ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... @@ -618,7 +618,7 @@ outputStack = sprintf('%s/%s_ali%d_%d.fixed', tmpCache,tiltList{iTilt},mapBackIter+1,iSection); - SAVE_IMG(correctedStack,outputStack,emc.pixel_size_angstroms); + SAVE_IMG(correctedStack,{outputStack, 'half'}, emc.pixel_size_angstroms); correctedStack = []; % Loop over tomos reconstructing section and appending a file to @@ -691,7 +691,7 @@ n_slices_in_Y = tiltChunks(end) - tiltChunks(1) + 1; end - rCMD = sprintf(['tilt %s %s -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... + rCMD = sprintf(['tilt %s %s -MODE 12 -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... super_sample, ... expand_lines, ... @@ -745,12 +745,12 @@ % Z coords (y in this orientation) are decreasing into the % monitor. For symmetrical padding this doesn't matter, but keep % in mind. /dev/null - trimCMD = sprintf('trimvol -rx -y %d,%d %s.TMPPAD %s > /dev/null ' , ... + trimCMD = sprintf('trimvol -mode 12 -rx -y %d,%d %s.TMPPAD %s > /dev/null ' , ... 1,floor(round(slab_list{iT}(iSection,5))),reconName,reconName); [msg,~]= system(trimCMD); if (msg) fprintf('%d from trimCMD\n',msg) - trimCMDPrintError = sprintf('trimvol -rx -y %d,%d %s.TMPPAD %s', ... + trimCMDPrintError = sprintf('trimvol -mode 12 -rx -y %d,%d %s.TMPPAD %s', ... 1,floor(round(slab_list{iT}(iSection,5))),reconName,reconName); system(trimCMDPrintError); error('error during trimvol'); @@ -820,7 +820,7 @@ end fclose(recombineCMD); pause(1); - recCMD = sprintf('newstack -fromone -FileOfInputs %s -output %s\n', file_of_outputs, reconNameFull); + recCMD = sprintf('newstack -mode 12 -fromone -FileOfInputs %s -output %s\n', file_of_outputs, reconNameFull); [err_msg, ~] = system(sprintf('%s > /dev/null ',recCMD)); %/dev/null if (err_msg) diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 86b3f744..b8d38ea4 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -400,7 +400,7 @@ % of the odd output here we can just read it in this way, unlike super res. iProjection = ... - single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],TLT(i,23),'keep')); + OPEN_IMG('single', iMrcObj,[1+osX,d1],[1+osY,d2],TLT(i,23),'keep'); iProjection = real(ifftn(fftn(iProjection).* BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'))); diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index 2b53705e..ac84601a 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -116,7 +116,7 @@ for iStack = 1%stacksFound - STACK = single(getVolume(MRCImage(INPUT_CELL{iStack,2}))); + STACK = OPEN_IMG('single',INPUT_CELL{iStack,2}); % The pixel size should be previously set correctly, but if it is not, then we % must maintain whatever is there in case beads are to be erased. The model % used for this process depends on the pixel size in the header when it was diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index aa77c119..1945defd 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -346,7 +346,7 @@ % of the odd output here we can just read it in this way, unlike super res. iProjection = ... - single(getVolume(iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep')); + OPEN_IMG('single', iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep'); iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); diff --git a/ctf/BH_runCtfFind.m b/ctf/BH_runCtfFind.m index a9a35b10..48be404f 100644 --- a/ctf/BH_runCtfFind.m +++ b/ctf/BH_runCtfFind.m @@ -14,7 +14,7 @@ ctfFindPath = getenv('EMC_CTFFIND'); fprintf('%s\n',ctfFindPath);% split the stack up -fullStack = getVolume(MRCImage(stackName)); +fullStack = OPEN_IMG('single', stackName); [d1,d2,d3] = size(fullStack); % FIXME d1 assumed to equal d2 Add check in saving for iPrj = 1:d3 diff --git a/metaData/OPEN_IMG.m b/metaData/OPEN_IMG.m index cdf3900e..08bfd851 100644 --- a/metaData/OPEN_IMG.m +++ b/metaData/OPEN_IMG.m @@ -1,6 +1,11 @@ function [ vol ] = OPEN_IMG(precision, filename, varargin) - vol = getVolume(MRCImage(filename, precision), varargin{:}); + if isa(filename, 'MRCImage') + vol = getVolume(filename, varargin{:}); + else + vol = getVolume(MRCImage(filename), varargin{:}); + end + if strcmp(precision, 'single') vol = single(vol); elseif strcmp(precision, 'double') diff --git a/statistics/BH_multi_loadAndMaskStack.m b/statistics/BH_multi_loadAndMaskStack.m index 0c6402ff..18dce659 100644 --- a/statistics/BH_multi_loadAndMaskStack.m +++ b/statistics/BH_multi_loadAndMaskStack.m @@ -63,7 +63,7 @@ else inputStack = sprintf('aliStacks/%s_ali%d%s.fixed',STACK,mapBackIter+1,suffix); end - STACK = single(getVolume(MRCImage(inputStack))); + STACK = OPEN_IMG('single',inputStack); end [d1,d2,d3] = size(STACK); diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index fca277bb..802f3791 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -344,7 +344,7 @@ end if (flgLoadMask) && (iGold == 1) fprintf('\n\nLoading external mask\n'); - externalMask = getVolume(MRCImage(sprintf('%s-pcaMask',subTomoMeta.(cycleNumber).(imgNAME){1}))); + externalMask = OPEN_IMG('single',sprintf('%s-pcaMask',subTomoMeta.(cycleNumber).(imgNAME){1})); end end @@ -424,7 +424,7 @@ outputPrefix, eigsFound, halfSet, iScale); prevVarianceMaps.(sprintf('h%d',iGold)).(sprintf('s%d',iScale)) = ... - getVolume(MRCImage(fname)).^flgStdDev; + OPEN_IMG('single', fname).^flgStdDev; end clear v coeffs eigsFound idxList end @@ -432,8 +432,7 @@ if (PREVIOUS_PCA) - volumeMask = gpuArray(getVolume(MRCImage( ... - sprintf('%s_pcaVolMask.mrc',outputPrefix)))); + volumeMask = gpuArray(OPEN_IMG('single', sprintf('%s_pcaVolMask.mrc',outputPrefix))); else if (emc.Pca_constrain_symmetry) @@ -782,14 +781,14 @@ if (emc.flgCutOutVolumes) particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak+1); - iParticle = gpuArray(getVolume(MRCImage(particleOUT_name),... + iParticle = gpuArray(OPEN_IMG('single',particleOUT_name,... [indVAL(1,1),indVAL(2,1)], ... [indVAL(1,2),indVAL(2,2)], ... [indVAL(1,3),indVAL(2,3)],'keep')); else - iParticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... + iParticle = gpuArray(OPEN_IMG('single', volumeData,[indVAL(1,1),indVAL(2,1)], ... [indVAL(1,2),indVAL(2,2)], ... [indVAL(1,3),indVAL(2,3)],'keep')); end diff --git a/statistics/BH_rocCurve.m b/statistics/BH_rocCurve.m index 345d0023..5b284e2d 100644 --- a/statistics/BH_rocCurve.m +++ b/statistics/BH_rocCurve.m @@ -18,11 +18,8 @@ error('did not understand C1 or O from your symmetry %s\n',symmetry) end -% 89411 % for 2.5, 1.5 -% 163863 % for 2 deg 1.25 deg -% 178451 % for 2.5 0.75 -mip = getVolume(MRCImage(mip_name)); +mip = OPEN_IMG('single',mip_name); % Search peaks first [d1,d2] = size(mip); ox = floor(d1/2)+1; diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 53c9b3b7..22f61bf9 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -143,6 +143,11 @@ [tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); +% if (multi_node_run) +% [ nParProcesses, iterList] = BH_multi_parallelJobs(nTiltSeries, nGPUs, sizeCalc(1), emc.nCpuCores, [cycle_numerator,cycle_denominator]); +% else +% [ nParProcesses, iterList] = BH_multi_parallelJobs(nTiltSeries, nGPUs, sizeCalc(1), emc.nCpuCores); +% end if (multi_node_run && ~skip_to_the_end_and_run) nParts = ceil(nTiltSeries ./ cycle_denominator); @@ -435,7 +440,7 @@ EMC_parpool(nWorkers); - avgTomo{1} = getVolume(MRCImage(backgroundName)); + avgTomo{1} = OPEN_IMG('single',backgroundName); avgTomo{1} = avgTomo{1} ./ (rmsScale*rms(avgTomo{1}(:))); if (delete_background_estimate) system(sprintf('rm -f %s',backgroundName)); @@ -556,7 +561,7 @@ % % TODO need to update this. % if (emc.save_mapback_classes) - % colorMap = single(getVolume(MRCImage(COLOR_MAP))); + % colorMap = OPEN_IMG('single',COLOR_MAP); % % should be the same size as the average % if any(size(refVol{1})-size(colorMap)) @@ -1186,7 +1191,7 @@ tic while toc < 300 try - dataPrj = single(getVolume(iMrcObj,[1,sTX],[1,sTY],iPrj,'keep')); + dataPrj = OPEN_IMG('single', iMrcObj,[1,sTX],[1,sTY],iPrj,'keep'); break catch pause(1e-1) @@ -1204,7 +1209,7 @@ tic while toc < 300 try - refPrj = single(getVolume(iMrcObjRef,[1,sTX],[1,sTY],iPrj,'keep')); + refPrj = OPEN_IMG('single', iMrcObjRef,[1,sTX],[1,sTY],iPrj,'keep'); break catch pause(1e-1) @@ -1222,7 +1227,7 @@ tic while toc < 300 try - samplingMask = gpuArray(single(getVolume(iMrcObjSamplingMask,[],[],iPrj,'keep'))); + samplingMask = gpuArray(OPEN_IMG('single', iMrcObjSamplingMask,[],[],iPrj,'keep')); break catch pause(1e-1) diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 21d6ba85..0ceed133 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -652,7 +652,7 @@ fullXform = load(iXFName_inv); end - STACK = single(getVolume(MRCImage(tilt_filepath))); + STACK = OPEN_IMG('single',tilt_filepath); for iPrj = 1:nPrjs diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index e70d178f..2d024476 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -314,7 +314,7 @@ else switch length(varargin) case 5 - maskVol = getVolume(MRCImage(varargin{3})); + maskVol = OPEN_IMG('single', varargin{3}); pixelSize = EMC_str2double(varargin{5}); maskVol = BH_mask3d(maskVol,EMC_str2double(varargin{5}),'',''); SAVE_IMG(MRCImage(gather(maskVol)),varargin{4},pixelSize); diff --git a/testScripts/recScript.m b/testScripts/recScript.m index 6a285a96..3a1f73fd 100644 --- a/testScripts/recScript.m +++ b/testScripts/recScript.m @@ -25,8 +25,8 @@ parfor iStack = 1:length(stack_names) fprintf('Working on stack %s\n', stack_names{iStack}); - bin_cmd = sprintf('newstack -bin %d aliStacks/%s bin10/%s_bin10.fixed', modBin, stack_names{iStack}, stack_names{iStack}(1:end-11)); - rec_cmd = sprintf('tilt -input bin10/%s_bin10.fixed -output bin10/%s_bin10.rec -TILTFILE fixedStacks/%s.tlt -RADIAL 0.15,0.05 -UseGPU 0 -THICKNESS %d -RotateBy90', ... + bin_cmd = sprintf('newstack -mode 12 -bin %d aliStacks/%s bin10/%s_bin10.fixed', modBin, stack_names{iStack}, stack_names{iStack}(1:end-11)); + rec_cmd = sprintf('tilt -MODE 12 -input bin10/%s_bin10.fixed -output bin10/%s_bin10.rec -TILTFILE fixedStacks/%s.tlt -RADIAL 0.15,0.05 -UseGPU 0 -THICKNESS %d -RotateBy90', ... stack_names{iStack}(1:end-11), stack_names{iStack}(1:end-11), stack_names{iStack}(1:end-11), modThick); errmsg = system(sprintf('%s > /dev/null', bin_cmd)); if errmsg diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index bb8964fb..ca38aaaf 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -908,7 +908,7 @@ if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) try particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iParticle = gpuArray(getVolume(MRCImage(particleOUT_name),... + iParticle = gpuArray(OPEN_IMG('single', particleOUT_name,... [indVAL(1,1),indVAL(2,1)], ... [indVAL(1,2),indVAL(2,2)], ... [indVAL(1,3),indVAL(2,3)],'keep')); @@ -918,7 +918,7 @@ end else - iParticle = gpuArray(getVolume(volumeData,[indVAL(1,1),indVAL(2,1)], ... + iParticle = gpuArray(OPEN_IMG('single', volumeData,[indVAL(1,1),indVAL(2,1)], ... [indVAL(1,2),indVAL(2,2)], ... [indVAL(1,3),indVAL(2,3)],'keep')); end diff --git a/transformations/BH_montage4d.m b/transformations/BH_montage4d.m index ecdd56c1..077a3676 100755 --- a/transformations/BH_montage4d.m +++ b/transformations/BH_montage4d.m @@ -36,7 +36,7 @@ if ischar(IMAGES{1}) - img1 = getVolume(MRCImage(IMAGES{1})); + img1 = OPEN_IMG('single', IMAGES{1}); else img1 = IMAGES{1}; @@ -61,7 +61,7 @@ img = IMAGES{iIMG}; else - img = getVolume(MRCImage(IMAGES{iIMG})); + img = OPEN_IMG('single', IMAGES{iIMG}); end diff --git a/transformations/BH_montage4d_halfGrid.m b/transformations/BH_montage4d_halfGrid.m index 9901214b..85781498 100644 --- a/transformations/BH_montage4d_halfGrid.m +++ b/transformations/BH_montage4d_halfGrid.m @@ -39,7 +39,7 @@ if ischar(IMAGES{1}) - img1 = getVolume(MRCImage(IMAGES{1})); + img1 = OPEN_IMG('single', IMAGES{1)); else img1 = IMAGES{1}; end @@ -58,7 +58,7 @@ IMAGES{iIMG} = []; else - MONTAGE(1 + (iIMG-1)*dx:dx + (iIMG-1)*dx,:,:) = getVolume(MRCImage(IMAGES{iIMG})); + MONTAGE(1 + (iIMG-1)*dx:dx + (iIMG-1)*dx,:,:) = OPEN_IMG('single', IMAGES{iIMG}); end diff --git a/transformations/BH_multi_loadOrBin.m b/transformations/BH_multi_loadOrBin.m index 5d38bb5b..36dd2018 100755 --- a/transformations/BH_multi_loadOrBin.m +++ b/transformations/BH_multi_loadOrBin.m @@ -79,7 +79,7 @@ binSize = [binSize,iHeader.nZ]; newStack = zeros(binSize,'single'); for iPrj = 1:binSize(3) - iProjection = gpuArray(getVolume(tiltObj,[],[],iPrj,'keep')); + iProjection = gpuArray(OPEN_IMG('single',tiltObj,[],[],iPrj,'keep')); if (iPrj == 1) bhF = fourierTransformer(iProjection,'OddSizeOversampled'); @@ -92,7 +92,7 @@ newStack(:,:,iPrj) = gather(iProjection); end - SAVE_IMG(newStack,outputName,iPixelHeader,iOriginHeader); + SAVE_IMG(newStack,{outputName,'half'},iPixelHeader,iOriginHeader); clear newStack bpFilt iProjection % system(sprintf('newstack -shrink %d -antialias 6 %s cache/%s_bin%d%s > /dev/null', ... % samplingRate,input_tilt_series_filename, imgName, samplingRate,imgExt)); @@ -116,7 +116,7 @@ m = MRCImage(sprintf(... 'cache/%s_bin%d%s', imgName, samplingRate,imgExt)); fprintf('Loaded the MRCImage\n'); - IMG_OUT =getVolume(m); + IMG_OUT = OPEN_IMG('single', m); fprintf('Loaded the volume\n'); IMG_OUT = single(IMG_OUT); fprintf('Volume --> single\n'); @@ -133,7 +133,7 @@ % but throw a warning. fprintf('\n\nYou requested a sampling of -1 Nonsense!! loading anyway.\n\n'); - IMG_OUT = single(getVolume(MRCImage(input_tilt_series_filename))); + IMG_OUT = OPEN_IMG('single', input_tilt_series_filename); end diff --git a/transformations/BH_multi_loadOrBuild.m b/transformations/BH_multi_loadOrBuild.m index 56d3519e..66a8ca9a 100755 --- a/transformations/BH_multi_loadOrBuild.m +++ b/transformations/BH_multi_loadOrBuild.m @@ -68,7 +68,7 @@ m = MRCImage(sprintf('%s', recon),0); % fprintf('Loaded the MRCImage\n'); if ( flgLoad ) - IMG_OUT = single(getVolume(m)); + IMG_OUT = OPEN_IMG('single', m); % fprintf('Loaded the volume\n'); else IMG_OUT = m; diff --git a/transformations/BH_reScale3d.m b/transformations/BH_reScale3d.m index f4374f17..3cda2246 100755 --- a/transformations/BH_reScale3d.m +++ b/transformations/BH_reScale3d.m @@ -40,7 +40,7 @@ else pixelSize = pixelSizeX; end - inputVol = {getVolume(mrcImage')}; + inputVol = {OPEN_IMG('single', mrcImage')}; writeOut = true; outPutArray = false; diff --git a/transformations/BH_resample2d.m b/transformations/BH_resample2d.m index 6c1e09f8..77b33fe1 100755 --- a/transformations/BH_resample2d.m +++ b/transformations/BH_resample2d.m @@ -1,6 +1,6 @@ function [ TRANS_IMAGE ] = BH_resample2d( IMAGE, ANGLES, SHIFTS, ... - CONVENTION, METHOD, DIRECTION, ... - MAG, SIZEOUT, varargin) + CONVENTION, METHOD, DIRECTION, ... + MAG, SIZEOUT, varargin) %Transform an image in 3d. % % @@ -87,7 +87,7 @@ if ischar(IMAGE) % Read in the image - stackIN = getVolume(MRCImage(IMAGE),[],[],[]); + stackIN = OPEN_IMG('single',IMAGE,[],[],[]); else stackIN = IMAGE; clear IMAGE end diff --git a/transformations/BH_resample3d.m b/transformations/BH_resample3d.m index d0c75659..60d2701c 100755 --- a/transformations/BH_resample3d.m +++ b/transformations/BH_resample3d.m @@ -80,7 +80,7 @@ if ischar(IMAGE) % Read in the image - IMAGE = getVolume(MRCImage(IMAGE)); + IMAGE = OPEN_IMG('single', IMAGE); end flgComplex = 0; diff --git a/transformations/BH_unStackMontage4d.m b/transformations/BH_unStackMontage4d.m index dbf91944..e62df85c 100755 --- a/transformations/BH_unStackMontage4d.m +++ b/transformations/BH_unStackMontage4d.m @@ -33,7 +33,7 @@ -montage = getVolume(MRCImage(NAME)); +montage = OPEN_IMG('single', MRCImage(NAME); ind = LOCATIONS; diff --git a/transformations/BH_unStackMontage4d_halfGrid.m b/transformations/BH_unStackMontage4d_halfGrid.m index c0a62d62..2267477e 100644 --- a/transformations/BH_unStackMontage4d_halfGrid.m +++ b/transformations/BH_unStackMontage4d_halfGrid.m @@ -33,7 +33,7 @@ -montage = getVolume(MRCImage(NAME)); +montage = OPEN_IMG('single', NAME); nVolumes = numel(IMAGES); From c2e6a008fd754eb27b525ed246e5f2d7ecf1e064 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 29 Feb 2024 16:27:59 -0500 Subject: [PATCH 053/151] half-precision working from emC segment to emC ctf3d templateSearch --- @MRCImage/getVolume.m | 28 +- @MRCImage/{OPEN_IMG.m => open_img.m} | 0 alignment/BH_templateSearch3d_2.m | 2 +- coordinates/BH_multi_recGeom.m | 27 +- ctf/BH_ctf_Correct3d.m | 463 ++++++++++------------ metaData/BH_geometryInitialize.m | 4 +- metaData/emc_check_for_valid_image_file.m | 19 + synthetic/BH_synthetic_mapBack.m | 2 +- synthetic/BH_to_cisTEM_mapBack.m | 2 +- transformations/emc_get_max_specimen_NZ.m | 26 +- 10 files changed, 263 insertions(+), 310 deletions(-) rename @MRCImage/{OPEN_IMG.m => open_img.m} (100%) create mode 100644 metaData/emc_check_for_valid_image_file.m diff --git a/@MRCImage/getVolume.m b/@MRCImage/getVolume.m index aee8bd78..292a9f3a 100644 --- a/@MRCImage/getVolume.m +++ b/@MRCImage/getVolume.m @@ -192,6 +192,11 @@ wordLength = (flgComplex*1 + 1) .* [nImageElements]; flgReCast = 1; + +readX = length(iIndex); +readY = length(jIndex); +imgSize = readX*readY; + switch mode % Make a string to tell fread how long each "value is. BAH 2017-11-22 case 0 @@ -204,7 +209,7 @@ case 1 - precisionString = 'int16'; + precisionString = 'int16'; nToRead = sprintf('%d*int16=>int16',wordLength); nToSkip = 2*nPixelsBetween * (flgComplex*1 + 1); % Allocate the output matrix - NOTE: always single precision at end @@ -220,11 +225,15 @@ % 2x as fast to recast the whole array, than to either read in as % single or to recast each slice. case 6 - precisionString = 'uint16'; + precisionString = 'uint16'; nToRead = sprintf('%d*uint16=>uint16',wordLength); nToSkip = 2*nPixelsBetween * (flgComplex*1 + 1); - otherwise + case 12 + precisionString = 'uint16'; + nToRead = sprintf('%d*uint16=>uint16',wordLength); + nToSkip = 2*nPixelsBetween * (flgComplex*1 + 1); + otherwise error('did not recognize mode value %d\n', mode) end @@ -239,9 +248,6 @@ vol = zeros(length(iIndex), length(jIndex), length(kIndex), precisionString); end -readX = length(iIndex); -readY = length(jIndex); -imgSize = readX*readY; nSlice = 1; for iSlice = kIndex @@ -284,7 +290,11 @@ if ( flgComplex ) vol = complex(single(vol{1}),single(vol{2})); else - vol = single(vol); + if mRCImage.header.mode == 12 + vol = half.typecast(vol); + else + vol = single(vol); + end end else if ( flgComplex ) @@ -299,9 +309,7 @@ mRCImage.fid = []; end -if mRCImage.header.mode == 12 - vol = half.typecast(vol); -end + end diff --git a/@MRCImage/OPEN_IMG.m b/@MRCImage/open_img.m similarity index 100% rename from @MRCImage/OPEN_IMG.m rename to @MRCImage/open_img.m diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 7aec2d11..04837deb 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -183,7 +183,7 @@ mapName = sprintf('%s_%d_bin%d',tomoName,tomoIdx,samplingRate); mapExt = '.rec'; -[ recGeom, ~, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName), mapBackIter); +% [ recGeom, ~, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName), mapBackIter); bp_vals(2) = 2.*max(latticeRadius); diff --git a/coordinates/BH_multi_recGeom.m b/coordinates/BH_multi_recGeom.m index 9827cfd0..208e361c 100755 --- a/coordinates/BH_multi_recGeom.m +++ b/coordinates/BH_multi_recGeom.m @@ -1,4 +1,4 @@ -function [ recGeom, tiltName, nTomos, tilt_geometry ] = BH_multi_recGeom( reconCoordName, mapBackIter ) +function [ recGeom, tiltName, tomoList, tilt_geometry ] = BH_multi_recGeom( reconCoordName, mapBackIter ) %UNTITLED Summary of this function goes here % Detailed explanation goes here @@ -48,6 +48,7 @@ % Tomos may be ignored when cleaning template matching results, or later if set to be ignored % in geometryAnalysis or if there are zero sub-tomos left. recGeom = cell(nTomos,1); +tiltList = cell(nTomos,1); for iTomo = 1:nTomos read_in_Coords = recCoords(1 + (iTomo-1)*6: 6 + (iTomo-1)*6); % This is not necessarily correct, e.,g you could have 4 bin10 tomos from one tilt, and not keep any model points @@ -72,17 +73,19 @@ tomoCoords = struct(); - tomoCoords.('y_i') = (read_in_Coords(2)); - tomoCoords.('y_f') = (read_in_Coords(3)); - tomoCoords.('NX') = (read_in_Coords(1)); - tomoCoords.('NY') = (read_in_Coords(3) - read_in_Coords( 2) + 1); - tomoCoords.('NZ') = (read_in_Coords(4)); - tomoCoords.('dX_specimen_to_tomo') = read_in_Coords(5); - tomoCoords.('dY_specimen_to_tomo') = ... - (emc_get_origin_index(tomoCoords.('NY')) ... - + read_in_Coords(2)) ... % origin of the tomogram in the full tilt projection - - emc_get_origin_index(tilt_geometry(1,21)); + tomoCoords.('is_active') = true; + tomoCoords.('NX') = read_in_Coords(1); + tomoCoords.('NY') = read_in_Coords(2); + tomoCoords.('NZ') = read_in_Coords(3); + tomoCoords.('dX_specimen_to_tomo') = read_in_Coords(4); + tomoCoords.('dY_specimen_to_tomo') = read_in_Coords(5); tomoCoords.('dZ_specimen_to_tomo') = read_in_Coords(6); + tomoCoords.('tilt_NX') = tilt_geometry(1,20); + tomoCoords.('tilt_NY') = tilt_geometry(1,21); + y_i = floor(emc_get_origin_index(tilt_geometry(1,21)) + tomoCoords.('dY_specimen_to_tomo') - tomoCoords.('NY') ./ 2); + y_f = y_i + floor(tomoCoords.('NY')) - 1; + tomoCoords.('y_i') = y_i; + tomoCoords.('y_f') = y_f; % Check that NX, NY, NZ are all positive if tomoCoords.('NX') <= 0 @@ -95,8 +98,8 @@ error('NZ is not positive for %s', tomoName); end - recGeom{iTomo} = tomoCoords; + tomoList{iTomo} = tomoName; end diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 00f6e3db..b22a74b7 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -254,28 +254,18 @@ nTomosPerTilt = 0; recGeom = 0; tiltRecGeom = 0; +tomoList = {}; +nTomos= 0; if (recon_for_subTomo) - [tiltList,nTilts] = BH_returnIncludedTilts(subTomoMeta.mapBackGeometry); - tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); - + [tiltList, nTilts] = BH_returnIncludedTilts(subTomoMeta.mapBackGeometry); else if (recon_for_tomoCPR) tiltList{1} = varargin{2}; nTilts = 1; - % We just need on - %e valid subtomot - iTry = 1; - tomoList{1} = ''; - while iTry < 25 - if (isfield(subTomoMeta.mapBackGeometry.tomoName,sprintf('%s_%d',tiltList{1},iTry))) - tomoList{1} = sprintf('%s_%d',tiltList{1},iTry); - break; - end - iTry = iTry + 1; - end - if isempty(tomoList{1}) - error('Did not find a valid tomogram in the searchspace ->25'); + if ~isfield(subTomoMeta.mapBackGeometry.(tiltList{1}).(tomoList)) + error('Did not find any tomograms for tilt-series %s',tiltList{1}); end + tomoList = subTomoMeta.mapBackGeometry.(tiltList{1}).(tomoList); else % TODO set up a check on the recon folder to get what is needed for % templateSearch @@ -285,24 +275,15 @@ error('Did not find any tomogram coordinates in recon/TS*.coords'); end tiltList = cell(nTilts,1); - nTomosTotal = 0; - nTomosPerTilt = cell(nTilts,1); tiltRecGeom = cell(nTilts,1); + tiltTomoList = cell(nTilts,1); for iStack = 1:nTilts - [ tiltRecGeom{iStack}, tiltName, nTomosPossible, tilt_geometry] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); - nTomosTotal = nTomosTotal + nTomosPossible; - nTomosPerTilt{iStack} = nTomosPossible; + % Since we are calling this for templateSearch nTomosPossible == nTomos + % After template matching, there may be inactive tomos, but we'll have the same amount + [ tiltRecGeom{iStack}, tiltName, tiltTomoList{iStack}, tilt_geometry] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); tiltList{iStack} = tiltName; end - - tomoList = cell(nTomosTotal,1); - nTomosAdd = 0; - for iStack = 1:nTilts - for iTomo = 1:nTomosPerTilt{iStack} - tomoList{nTomosAdd+1} = sprintf('%s_%d',tiltName,iTomo); - nTomosAdd = nTomosAdd +1; - end - end + end end @@ -315,20 +296,19 @@ [ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores); -try - EMC_parpool(nParProcesses) -catch - delete(gcp('nocreate')) - EMC_parpool(nParProcesses) -end +% try +% EMC_parpool(nParProcesses) +% catch +% delete(gcp('nocreate')) +% EMC_parpool(nParProcesses) +% end -parfor iParProc = 1:nParProcesses -% for iGPU = 1:nGPUs %%revert - iGPU = mod(iParProc,nGPUs); +% parfor iParProc = 1:nParProcesses +for iParProc = 1:nParProcesses %%revert + % iGPU = mod(iParProc,nGPUs); for iTilt = iterList{iParProc} nTomos = 0; alreadyMade = 0; - iTomoList = {}; % For now, since the tilt geometry is not necessarily updated (it is manual) % in the subTomoMeta, check that newer (possible perTilt refined) data is @@ -336,28 +316,37 @@ TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); TLT = load(TLTNAME); - - % Get all the tomogram names that belong to a given tilt-series. % FIXME: I'm not sure it makes sense to restrict this block to for_subTomo - if (recon_for_subTomo) - for iTomo = 1:length(tomoList) - if strcmp(tiltList{iTilt}, subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) - iTomoList{nTomos+1} = tomoList{iTomo}; - nTomos = nTomos + 1; - end + if (recon_for_subTomo || recon_for_templateMatching) + if (recon_for_subTomo) + % List of all possible tomos, some may be "in-active" since this is post-template matching + tomoList = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).(tomoList); + else + % List of all possible tomos, all are "active" since this is pre-template matching + tomoList = tiltTomoList{iTilt}; + end + nTomos = length(tomoList); + if (bh_global_turn_on_phase_plate(1)) + filtered = '_filtered'; + else + filtered = ''; + end + + for iTomo = 1:nTomos % The order of tomo num could be off but only if all are present do we % skip. - if (bh_global_turn_on_phase_plate(1)) - filtered = '_filtered'; - else - filtered = ''; - end - checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', ... - tiltList{iTilt},iTomo,samplingRate,filtered); + checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', tiltList{iTilt}, iTomo, samplingRate, filtered); if exist(checkRecon, 'file') - fprintf('found %s to already exits\n',checkRecon); - alreadyMade = alreadyMade +1; + try + % Could have a corrupt file + testread = MRCImage(checkRecon,0); + fprintf('found %s to already exits\n',checkRecon); + alreadyMade = alreadyMade + 1 + catch + fprintf('found %s to already exits but it is corrupt\n',checkRecon); + system(sprintf('rm %s',checkRecon)); + end end end @@ -368,20 +357,21 @@ end end - - - preBinStacks(TLT, tiltList{iTilt}, mapBackIter,1,... - samplingRate,... - PosControl2d,... - tiltWeight,... - flgMedianFilter); + preBinStacks(TLT, ... + tiltList{iTilt}, ... + mapBackIter,... + 1,... + samplingRate,... + PosControl2d,... + tiltWeight,... + flgMedianFilter); end end % All data is handled through disk i/o so everything unique created in the -parfor iParProc = 1:nParProcesses - % for iGPU = 1:nGPUs %%revert +% parfor iParProc = 1:nParProcesses + for iParProc = 1:nParProcesses %%revert iGPU = mod(iParProc,nGPUs); % for iGPU = 1:nGPUs %%revert @@ -391,85 +381,45 @@ for iTilt = iterList{iParProc} slab_list = {}; - % if (recon_for_subTomo) - % nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; - % iCoords = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).coords; - % % FIXME - % else - % if (recon_for_tomoCPR) - % nTomos = 1; - % else - % % templaterch - % nTomos = nTomosPerTilt{iTilt}; - % iCoords = tiltRecGeom{iTilt}; - % end - % end - - - % if (recon_for_subTomo || recon_for_templateMatching)) - % targetSizeY = diff(floor(iCoords(:,2:3)),1,2)+1; - % iCoords = iCoords ./ samplingRate; - % iCoords(:,1:4) = floor(iCoords(:,1:4)); - % iCoords(:,3) = iCoords(:,3) - (diff(floor(iCoords(:,2:3)),1,2)+1 - floor(targetSizeY./samplingRate)); - % end - % iTomoList = cell(nTomos,1); - - - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt}, mapBackIter + 1 ); TLT = load(TLTNAME); fprintf('iParProc %d and iTilt %d using TLT %s\n', iParProc, iTilt, TLTNAME); - - iTomoList = {}; - if (recon_for_subTomo) - % Get all the tomogram names that belong to a given tilt-series. - nTomos = 0; - alreadyMade = 0; - for iTomo = 1:length(tomoList) - if strcmp(tiltList{iTilt},subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName) - iTomoList{nTomos+1} = tomoList{iTomo}; - nTomos = nTomos + 1; - end - % The order of tomo num could be off but only if all are present do we - % skip. - if (bh_global_turn_on_phase_plate(1)) - filtered = '_filtered'; - else - filtered = ''; - end - checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', ... - tiltList{iTilt},iTomo,samplingRate,filtered); - if exist(checkRecon, 'file') - fprintf('found %s to already exits\n',checkRecon); - alreadyMade = alreadyMade +1; - end - - end - - if alreadyMade == nTomos - fprintf('All tomos 1-%d found to exist for tilt-series %s\n',nTomos,tiltList{iTilt}); - continue + if (recon_for_subTomo || recon_for_templateMatching) + if (recon_for_subTomo) + % List of all possible tomos, some may be "in-active" since this is post-template matching + tomoList = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).(tomoList); + else + % List of all possible tomos, all are "active" since this is pre-template matching + tomoList = tiltTomoList{iTilt}; end + % else tomoCPR, nTilts = 1 and tomoList is already set end + nTomos = length(tomoList); + iCoords = cell(nTomos,1); if (recon_for_templateMatching) - % templaterch - nTomos = nTomosPerTilt{iTilt}; % tiltRecGeom is a cell with each value being a cell returned by multi_recGeom - iCoords = tiltRecGeom{iTilt}; - % iCoords will be a cell indexed by each tomo wwith a struct .tomoCoords + % each element of this cell is a struct tomoCoords, we effectively create an anonymous struct + % accessed through iCoords. + for iCoordIdx = 1:nTomos + % each element of this cell is a struct tomoCoords + iCoords{iCoordIdx} = tiltRecGeom{iTilt}{iCoordIdx}; + end else if (recon_for_tomoCPR) - nTomos = 1; - iCoords = {subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{1})}; + % Get a copy of the tomoCoords (all tomodata will be deleted and only the tilt info kept for tomoCPR) + % place in a cell for consistency + + iCoords{1} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{1}); + iCoords{1}.is_active = 1; + iCoords{1}.dX_specimen_to_tomo = 0; + iCoords{1}.dY_specimen_to_tomo = 0; + iCoords{1}.dZ_specimen_to_tomo = 0; else - nTomos = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).nTomos; - % subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.tomoCoords.dX_specimen_to_tomo; - iCoords = cell(nTomos,1); for iCoordIdx = 1:nTomos - iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(iTomoList{iCoordIdx}); - % FIXME + % each element of this cell is a struct tomoCoords + iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iCoordIdx}); end end @@ -477,23 +427,20 @@ if samplingRate > 1 - fullStack = sprintf('%aliStacks/%s_ali%d.fixed', ... - tiltList{iTilt},mapBackIter+1); - inputStack = sprintf('cache/%s_ali%d_bin%d.fixed',... - tiltList{iTilt},mapBackIter+1,samplingRate); - if ~exist(inputStack, 'file') + fullStack = sprintf('%aliStacks/%s_ali%d.fixed', tiltList{iTilt}, mapBackIter + 1); + inputStack = sprintf('cache/%s_ali%d_bin%d.fixed', tiltList{iTilt}, mapBackIter + 1, samplingRate); + if ~emc_check_for_valid_image_file(inputStack) BH_multi_loadOrBin(fullStack, samplingRate, 2, false); end else - inputStack = sprintf('aliStacks/%s_ali%d.fixed',... - tiltList{iTilt},mapBackIter+1); + inputStack = sprintf('aliStacks/%s_ali%d.fixed', tiltList{iTilt}, mapBackIter + 1); end maskedStack = OPEN_IMG('single', inputStack); if (recon_for_subTomo) - [ ~, specimen_NZ_nm, tomoIdx, ~ ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, emc.pixel_size_angstroms, ... + [ ~, specimen_NZ_nm, ~ ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... + tomoList, nTomos, emc.pixel_size_angstroms, ... samplingRate, cycleNumber,... 0,1); else @@ -505,22 +452,19 @@ NZ = floor(reconstructionParameters(1)); specimen_NZ_nm = NZ * emc.pixel_size_angstroms / 10; - % iCoords = [NX,0,NY-1,NZ,0,0]; - tomoIdx = 1; - % FIXME + if (nTomos ~= 1) + error('For tomoCPR, only one tomo can be reconstructed at a time'); + end for iCoordIdx = 1:nTomos - iCoords{iCoordIdx}.tomoCoords.dX_specimen_to_tomo = 0; - iCoords{iCoordIdx}.tomoCoords.dY_specimen_to_tomo = 0; - iCoords{iCoordIdx}.tomoCoords.dZ_specimen_to_tomo = 0; iCoords{iCoordIdx}.tomoCoords.NX = NX * samplingRate; iCoords{iCoordIdx}.tomoCoords.NY = NY * samplingRate; iCoords{iCoordIdx}.tomoCoords.NZ = NZ * samplingRate; end else - [ ~, specimen_NZ_nm, tomoIdx, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, emc.pixel_size_angstroms, ... - samplingRate, cycleNumber,... - 0,1); + [ ~, specimen_NZ_nm, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... + tomoList,nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + 0,1); end end @@ -555,14 +499,14 @@ % For each tomo create a list of slices that are to be reconstructed % for every section section. - [ slab_list ] = calc_slab_boundaries(iCoords, tomoIdx, emc.pixel_size_angstroms, n_slabs_to_reconstruct, tiltList{iTilt}, ctf3dDepth, samplingRate); + [ slab_list ] = calc_slab_boundaries(iCoords, tomoList, emc.pixel_size_angstroms, n_slabs_to_reconstruct, tiltList{iTilt}, ctf3dDepth, samplingRate); if (recon_for_subTomo) - [ avgZ, specimen_NZ_nm, tomoIdx, surfaceFit ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... - iTomoList,nTomos, emc.pixel_size_angstroms, ... - samplingRate, cycleNumber,... - slab_list, 0); + [ avgZ, specimen_NZ_nm, surfaceFit ] = calcAvgZ(subTomoMeta,iCoords,tiltList{iTilt}, ... + tomoList,nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + slab_list, 0); else avgZ = 0; surfaceFit = 0; @@ -582,10 +526,10 @@ defFitFull = ''; preCombDefocus = 0; if (mapBackIter) - defFitFull = sprintf('mapBack%d/%s_ali%d_ctf.defFidFull',mapBackIter, tiltList{iTilt},mapBackIter); + defFitFull = sprintf('mapBack%d/%s_ali%d_ctf.defFidFull',mapBackIter, tiltList{iTilt}, mapBackIter); if exist(defFitFull,'file') preCombDefocus = load(defFitFull); - fprintf('3dCTF using pre calc combined per tilt defocus %s\n',defFitFull); + fprintf('3dCTF using pre calc combined per tilt defocus %s\n', defFitFull); end end @@ -617,18 +561,15 @@ end - outputStack = sprintf('%s/%s_ali%d_%d.fixed', tmpCache,tiltList{iTilt},mapBackIter+1,iSection); - SAVE_IMG(correctedStack,{outputStack, 'half'}, emc.pixel_size_angstroms); + outputStack = sprintf('%s/%s_ali%d_%d.fixed', tmpCache, tiltList{iTilt}, mapBackIter+1, iSection); + SAVE_IMG(correctedStack, {outputStack, 'half'}, emc.pixel_size_angstroms); correctedStack = []; % Loop over tomos reconstructing section and appending a file to - for iT = 1:nTomos - - thisTomo = tomoIdx(iT); + for iTomo = 1:nTomos - if (slab_list{iT}(iSection,1)) - reconName = sprintf('%s/%s_ali%d_%d_%d.rec', ... - tmpCache,tiltList{iTilt},mapBackIter+1,thisTomo,iSection); + if (slab_list{iTomo}(iSection,1)) + reconName = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, iTomo, iSection); if (recon_for_tomoCPR) TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{1}),1); @@ -636,7 +577,7 @@ end if (recon_for_subTomo) - TA = sortrows(subTomoMeta.tiltGeometry.(sprintf('%s_%d',tiltList{iTilt},thisTomo)),1); + TA = sortrows(subTomoMeta.tiltGeometry.(sprintf('%s_%d',tiltList{iTilt},iTomo)),1); TA = TA(:,4); end @@ -651,7 +592,7 @@ end end - rawTLT = sprintf('cache/%s_%d.rawtlt',tiltList{iTilt},thisTomo); + rawTLT = sprintf('cache/%s_%d.rawtlt',tiltList{iTilt},iTomo); rawTLT_file = fopen(rawTLT, 'w'); fprintf(rawTLT_file,'%f\n', TA'); fclose(rawTLT_file); @@ -670,13 +611,14 @@ end % round down and then we'll add any extra needed to the final chunk - tiltChunkSize = floor(iCoords{iT}.NY ./ samplingRate ./ emc.n_tilt_workers); + tiltChunkSize = floor(iCoords{iTomo}.NY ./ samplingRate ./ emc.n_tilt_workers); % This shoulid never happen, but to be safe - if (emc.n_tilt_workers > floor(iCoords{iT}.NY ./ samplingRate)) + if (emc.n_tilt_workers > floor(iCoords{iTomo}.NY ./ samplingRate)) error('n_tilt_workers is greater than the number of slices in the tilt series'); end - y_i = floor(iCoords{iT}.y_i ./ samplingRate); - y_f = floor(iCoords{iT}.y_f ./ samplingRate); + + y_i = floor(iCoords{iTomo}.y_i ./ samplingRate); + y_f = floor(iCoords{iTomo}.y_f ./ samplingRate); tiltChunks = y_i:tiltChunkSize:y_f; @@ -699,10 +641,10 @@ reconName, ... rawTLT, ... iGPU, ... - floor(iCoords{iT}.NX ./ samplingRate),... % WIDTH = NX - floor(round(slab_list{iT}(iSection,5))), ... % THICKNESS = NZ - iCoords{iT}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X - slab_list{iT}(iSection,6)); + floor(iCoords{iTomo}.NX ./ samplingRate),... % WIDTH = NX + floor(round(slab_list{iTomo}(iSection,5))), ... % THICKNESS = NZ + iCoords{iTomo}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X + slab_list{iTomo}(iSection,6)); reconScaling = 1; @@ -746,12 +688,12 @@ % monitor. For symmetrical padding this doesn't matter, but keep % in mind. /dev/null trimCMD = sprintf('trimvol -mode 12 -rx -y %d,%d %s.TMPPAD %s > /dev/null ' , ... - 1,floor(round(slab_list{iT}(iSection,5))),reconName,reconName); + 1,floor(round(slab_list{iTomo}(iSection,5))),reconName,reconName); [msg,~]= system(trimCMD); if (msg) fprintf('%d from trimCMD\n',msg) trimCMDPrintError = sprintf('trimvol -mode 12 -rx -y %d,%d %s.TMPPAD %s', ... - 1,floor(round(slab_list{iT}(iSection,5))),reconName,reconName); + 1,floor(round(slab_list{iTomo}(iSection,5))),reconName,reconName); system(trimCMDPrintError); error('error during trimvol'); end @@ -769,19 +711,15 @@ evalMask = []; maskedStack = []; - for iT = 1:nTomos - thisTomo = tomoIdx(iT); + for iTomo = 1:nTomos % Note that bh_global_turn_on_phase_plate could be true for any of the recon_for_stage bools, so it must % be checked first. if (bh_global_turn_on_phase_plate(1)) - reconNameFull = sprintf('cache/%s_%d_bin%d_filtered.rec', ... - tiltList{iTilt},thisTomo,samplingRate); + reconNameFull = sprintf('cache/%s_%d_bin%d_filtered.rec', tiltList{iTilt}, iTomo, samplingRate); elseif recon_for_tomoCPR - reconNameFull = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec', ... - CWD,tiltList{iTilt},thisTomo,samplingRate); + reconNameFull = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec', CWD,tiltList{iTilt},iTomo,samplingRate); else - reconNameFull = sprintf('cache/%s_%d_bin%d.rec', ... - tiltList{iTilt},thisTomo,samplingRate); + reconNameFull = sprintf('cache/%s_%d_bin%d.rec', tiltList{iTilt},iTomo,samplingRate); end fprintf('in ctf3d reconNameFull is %s\n\n',reconNameFull); @@ -789,13 +727,13 @@ % Get the total number of sections for this tomo n_total_sections = 0; for iSection = 1:n_slabs_to_reconstruct - if(slab_list{iT}(iSection,1)) + if(slab_list{iTomo}(iSection,1)) n_total_sections = n_total_sections + 1; end end if (n_total_sections == 0) - fprintf('no sections for tomo %d\n',thisTomo); + fprintf('no sections for tomo %d\n',iTomo); continue end @@ -811,11 +749,11 @@ % end % for iSection = 1:n_slabs_to_reconstruct for iSection = slab_order - if (slab_list{iT}(iSection,1)) - this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, thisTomo, iSection); + if (slab_list{iTomo}(iSection,1)) + this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, iTomo, iSection); cleanup3 = sprintf('%s %s',cleanup3,this_slab); fprintf(recombineCMD, '%s\n', this_slab); - fprintf(recombineCMD, '1-%d\n',floor(round(slab_list{iT}(iSection,5)))); + fprintf(recombineCMD, '1-%d\n',floor(round(slab_list{iTomo}(iSection,5)))); end end fclose(recombineCMD); @@ -850,11 +788,14 @@ end -function [] = preBinStacks(TLT, STACK_PRFX, mapBackIter,usableArea,... - samplingRate,... - PosControl2d,... - tiltWeight,... - flgMedianFilter) +function [] = preBinStacks(TLT, ... + STACK_PRFX, ... + mapBackIter, ... + usableArea,... + samplingRate,... + PosControl2d,... + tiltWeight,... + flgMedianFilter) @@ -866,12 +807,10 @@ suffix = ''; end - -fullStack = sprintf('%sStacks/%s_ali%d%s.fixed', ... - prefix,STACK_PRFX,mapBackIter+1,suffix); -inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed',... - STACK_PRFX,mapBackIter+1,suffix,samplingRate); -if ~exist(inputStack, 'file') +% TODO: this could all be in loadOrBin +fullStack = sprintf('%sStacks/%s_ali%d%s.fixed', prefix,STACK_PRFX, mapBackIter+1, suffix); +inputStack = sprintf('cache/%s_ali%d%s_bin%d.fixed', STACK_PRFX, mapBackIter+1, suffix, samplingRate); +if ~emc_check_for_valid_image_file(inputStack) BH_multi_loadOrBin(fullStack, samplingRate, 2, false); end @@ -879,10 +818,10 @@ end -function [ slab_list ] = calc_slab_boundaries(iCoords, tomoIdx, pixel_size_angstroms, n_slabs_to_reconstruct, tiltName, ctf_3d_depth_si, samplingRate, use_inverted_newstack) +function [ slab_list ] = calc_slab_boundaries(iCoords, tomoList, pixel_size_angstroms, n_slabs_to_reconstruct, tiltName, ctf_3d_depth_si, samplingRate, use_inverted_newstack) %%% This function is to produce a list of z indices, starting from 1, to pass to imod for tilt based reconstruction -nTomos = length(tomoIdx); +nTomos = length(tomoList); slab_list = cell(nTomos,1); for iTomo = 1:nTomos % min and max in absolute pixels min and max from 1:nZrecon @@ -895,16 +834,23 @@ slab_size_pixels = slab_size_pixels + ~mod(slab_size_pixels,2); oS = emc_get_origin_index(slab_size_pixels); -for iT = 1:length(tomoIdx) - iTomo = tomoIdx(iT); +for iTomo = 1:nTomos % Origin + originshift - tomo_origin_wrt_tilt_origin = iCoords{iT}.dZ_specimen_to_tomo ./ samplingRate; - tomo_origin_in_tomo_frame = emc_get_origin_index(iCoords{iT}.NZ ./ samplingRate); + + if ~(iCoords{iTomo}.is_active) + % Nothing to do, the first column in this row of slab_list is already zero, but we can set it + % explicitly in case the code changes in the future + slab_list{iTomo}(:,1) = 0; + continue; + end + + tomo_origin_wrt_tilt_origin = iCoords{iTomo}.dZ_specimen_to_tomo ./ samplingRate; + tomo_origin_in_tomo_frame = emc_get_origin_index(iCoords{iTomo}.NZ ./ samplingRate); fraction_origin_shift = tomo_origin_wrt_tilt_origin - round(tomo_origin_wrt_tilt_origin); tomogram_lower_bound = floor((tomo_origin_wrt_tilt_origin - tomo_origin_in_tomo_frame)); - recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + ceil(iCoords{iT}.NZ./samplingRate) - 1; + recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + ceil(iCoords{iTomo}.NZ./samplingRate) - 1; % For each slab see if this tomogram has any sections in it for iSlab = 1:n_slabs_to_reconstruct @@ -917,13 +863,13 @@ is_in_range = ismember(recon_range_z_in_specimen_frame, slab_range); valid_indices = recon_range_z_in_specimen_frame(is_in_range); - slab_list{iT}(iSlab,5) = length(valid_indices); - if (slab_list{iT}(iSlab,5) > 0) - slab_list{iT}(iSlab,1) = 1; + slab_list{iTomo}(iSlab,5) = length(valid_indices); + if (slab_list{iTomo}(iSlab,5) > 0) + slab_list{iTomo}(iSlab,1) = 1; else continue; end - valid_region_origin = emc_get_origin_index(slab_list{iT}(iSlab,5)); + valid_region_origin = emc_get_origin_index(slab_list{iTomo}(iSlab,5)); % This is a vector from the origin of the sample to the origin of the slab. % The shift passed to imod-tilt moves the reconstructed area in the opposite sense. % All slabs need to be shifted to the specimen origin (0) from tilts perspective, and then the are assembled into the final volume. @@ -932,33 +878,33 @@ % I know ... this is a shit show. dZ_for_reconstructed_slab = (valid_indices(valid_region_origin) + fraction_origin_shift); - slab_list{iT}(iSlab,6) = dZ_for_reconstructed_slab; %dZ + slab_list{iTomo}(iSlab,6) = dZ_for_reconstructed_slab; %dZ end % Check to ensure we don't have any tiny slabs leftover, if so, merge them into a neighboring slab - biggest_slab = max(slab_list{iT}(:,5)); + biggest_slab = max(slab_list{iTomo}(:,5)); for iSlab = 1:n_slabs_to_reconstruct - if (slab_list{iT}(iSlab,1) && slab_list{iT}(iSlab, 5) / biggest_slab < 0.1) - if (iSlab > 1 && slab_list{iT}(iSlab-1,1)) - delta = slab_list{iT}(iSlab,5); - slab_list{iT}(iSlab-1,5) = slab_list{iT}(iSlab-1,5) + delta; - slab_list{iT}(iSlab,1) = 0; + if (slab_list{iTomo}(iSlab,1) && slab_list{iTomo}(iSlab, 5) / biggest_slab < 0.1) + if (iSlab > 1 && slab_list{iTomo}(iSlab-1,1)) + delta = slab_list{iTomo}(iSlab,5); + slab_list{iTomo}(iSlab-1,5) = slab_list{iTomo}(iSlab-1,5) + delta; + slab_list{iTomo}(iSlab,1) = 0; % we are adding slices from above the specimen in Z so the z shift is positive - slab_list{iT}(iSlab-1,6) = (slab_list{iT}(iSlab-1,6) + delta); - elseif (iSlab < n_slabs_to_reconstruct && slab_list{iT}(iSlab+1,1)) - delta = slab_list{iT}(iSlab,5); - slab_list{iT}(iSlab+1,5) = slab_list{iT}(iSlab+1,5) + slab_list{iT}(iSlab,5); - slab_list{iT}(iSlab,1) = 0; + slab_list{iTomo}(iSlab-1,6) = (slab_list{iTomo}(iSlab-1,6) + delta); + elseif (iSlab < n_slabs_to_reconstruct && slab_list{iTomo}(iSlab+1,1)) + delta = slab_list{iTomo}(iSlab,5); + slab_list{iTomo}(iSlab+1,5) = slab_list{iTomo}(iSlab+1,5) + slab_list{iTomo}(iSlab,5); + slab_list{iTomo}(iSlab,1) = 0; % we are adding slices from below the specimen in Z so the z shift is negative - slab_list{iT}(iSlab+1,6) = (slab_list{iT}(iSlab+1,6) - delta); + slab_list{iTomo}(iSlab+1,6) = (slab_list{iTomo}(iSlab+1,6) - delta); end end end % TroubleShoot tSHT = fopen(sprintf('.tblSht_%s_i%d.txt',tiltName,iTomo),'w'); - fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords{iT}.NX, iCoords{iT}.NY, iCoords{iT}.NZ, iCoords{iT}.dX_specimen_to_tomo, iCoords{iT}.dY_specimen_to_tomo, iCoords{iT}.dZ_specimen_to_tomo); - fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', slab_list{iT}'); + fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords{iTomo}.NX, iCoords{iTomo}.NY, iCoords{iTomo}.NZ, iCoords{iTomo}.dX_specimen_to_tomo, iCoords{iTomo}.dY_specimen_to_tomo, iCoords{iTomo}.dZ_specimen_to_tomo); + fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', slab_list{iTomo}'); fclose(tSHT); end % end loop over tomos @@ -1179,16 +1125,16 @@ clear tile Hqz end -function [avgZ, specimen_NZ_nm, tomoIdx,surfaceFit] = calcAvgZ(subTomoMeta, ... - iCoords, ... - tiltName, ... - tomoList,... - nTomos, ... - pixel_size_angstroms,... - samplingRate, ... - cycleNumber,... - slab_list, ... - calcMaxZ) +function [avgZ, specimen_NZ_nm, surfaceFit] = calcAvgZ(subTomoMeta, ... + iCoords, ... + tiltName, ... + tomoList,... + nTomos, ... + pixel_size_angstroms,... + samplingRate, ... + cycleNumber,... + slab_list, ... + calcMaxZ) % Calculate the maximum extensions in Z and then how many separate sections % need to be corrected. @@ -1197,20 +1143,15 @@ avgZ = 0; -if isa(subTomoMeta,'struct') - val_to_pass = subTomoMeta.mapBackGeometry.tomoName; -else - val_to_pass = 'dummy'; -end -[ specimen_NZ_pixels, tomoIdx ] = emc_get_max_specimen_NZ(val_to_pass, ... - iCoords, ... - tomoList, ... - nTomos, ... - samplingRate); +[ specimen_NZ_pixels ] = emc_get_max_specimen_NZ( ... + iCoords, ... + tomoList, ... + nTomos, ... + samplingRate); specimen_NZ_nm = specimen_NZ_pixels .* pixel_size_angstroms ./ 10; -fprintf('combining the thickness and shift on tilt %s, found a specimen_NZ_nm %3.3f nm\n',tiltName,specimen_NZ_nm); +fprintf('combining the thickness and shift on tilt %s, found a specimen_NZ_nm %3.3f nm\n', tiltName, specimen_NZ_nm); if (calcMaxZ) return; @@ -1251,13 +1192,14 @@ use_subtomo_z_positions = true; -for iT = 1:nTomos - - iTomo = tomoIdx(iT); +for iTomo = 1:nTomos + if ~(subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}).is_active) + continue; + end % X in the Y frame means a vector from the Y lower left to the X origin % X origin wrt Y origin is a vector from the origin of Y to the X origin - reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iT}); + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); tomo_origin_wrt_tilt_origin = [reconGeometry.dX_specimen_to_tomo, ... reconGeometry.dY_specimen_to_tomo, ... reconGeometry.dZ_specimen_to_tomo]; @@ -1265,16 +1207,15 @@ reconGeometry.NY, ... reconGeometry.NZ]); - iTomoName = sprintf('%s_%d',tiltName,iTomo); % Get the z-coordinates of the origin for all included subtomograms relative to the lower left of the tomogram % shouldn't be any removed particles at this stage but later there would be. - subtomo_origin_z_in_tomo_frame = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,13); + subtomo_origin_z_in_tomo_frame = initGeom.(tomoList{iTomo})(initGeom.(tomoList{iTomo})(:,26)~=-9999,13); % We may get here if we have split a data set into several small classes so skip the centering on average if needed if isempty(subtomo_origin_z_in_tomo_frame) use_subtomo_z_positions = false; - fprintf('No subtomograms found for %s',iTomoName); + fprintf('No subtomograms found for %s', tomoList{iTomo}); continue; end @@ -1284,21 +1225,21 @@ subtomo_origin_wrt_specimen_origin = subtomo_origin_wrt_specimen_origin ./ samplingRate; totalZ = totalZ + sum(subtomo_origin_wrt_specimen_origin); fprintf('%s tomo has %d subTomos with mean Z %3.3f nm\n', ... - iTomoName, length(subtomo_origin_wrt_specimen_origin), ... + tomoList{iTomo}, length(subtomo_origin_wrt_specimen_origin), ... mean(subtomo_origin_wrt_specimen_origin) * pixel_size_angstroms ./ 10); nSubTomos = nSubTomos + length(subtomo_origin_wrt_specimen_origin); for iSection = 1:n_slabs_to_reconstruct - iSecOrigin = slab_list{iT}(iSection,6); - iSecRadius = slab_list{iT}(iSection,5)/2; + iSecOrigin = slab_list{iTomo}(iSection,6); + iSecRadius = slab_list{iTomo}(iSection,5)/2; inSectionIDX = subtomo_origin_wrt_specimen_origin > iSecOrigin - iSecRadius & subtomo_origin_wrt_specimen_origin <= iSecOrigin + iSecRadius; - x = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,11); + x = initGeom.(tomoList{iTomo})(initGeom.(tomoList{iTomo})(:,26)~=-9999,11); x = (x - tomo_origin_in_tomo_frame(1) + tomo_origin_wrt_tilt_origin(1))./samplingRate; - y = initGeom.(iTomoName)(initGeom.(iTomoName)(:,26)~=-9999,12); + y = initGeom.(tomoList{iTomo})(initGeom.(tomoList{iTomo})(:,26)~=-9999,12); y = (y - tomo_origin_in_tomo_frame(2) + tomo_origin_wrt_tilt_origin(2))./samplingRate; diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index 22ad8a4c..0bc53d86 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -149,7 +149,8 @@ % other concern is then linking each tomogram to the parent tilt-series. for iStack = 1:nStacks - [ recGeom, tiltName, nTomosPossible, tilt_geometry ] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); + [ recGeom, tiltName, tmpTomoList, tilt_geometry ] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); + nTomosPossible = length(tmpTomoList); % Initialize if (doImport) @@ -158,6 +159,7 @@ iPath = dir(sprintf('convmap/%s_*.mod',tiltName)); end + nTomos = length(iPath); if nTomos > nTomosPossible error('The number of model files in convmap/*.mod is greater than the number in the recon/*.coords\n'); diff --git a/metaData/emc_check_for_valid_image_file.m b/metaData/emc_check_for_valid_image_file.m new file mode 100644 index 00000000..5d018c02 --- /dev/null +++ b/metaData/emc_check_for_valid_image_file.m @@ -0,0 +1,19 @@ +function [ is_valid ] = emc_check_for_valid_image_file(wanted_filename) + + % Check if the file exists + if isfile(wanted_filename) + try + test_header = MRCImage(wanted_filename, 0); + is_valid = true; + return; + catch + system(['rm ' wanted_filename]); + is_valid = false; + return; + end + else + is_valid = false; + return; + end + +end \ No newline at end of file diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 22f61bf9..acf2295d 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -372,7 +372,7 @@ % The tiltHeader = getHeader(MRCImage(tilt_binned_filename, 0)); tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{1}).tiltName; - [ maxZ ] = emc_get_max_specimen_NZ(subTomoMeta.mapBackGeometry.tomoName, ... + [ maxZ ] = emc_get_max_specimen_NZ( ... subTomoMeta.mapBackGeometry.tomoCoords, ... tomoList, ... nTomograms, ... diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 0ceed133..a94ac8dc 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -273,7 +273,7 @@ % The tiltHeader = getHeader(MRCImage(tilt_filepath, 0)); tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{1}).tiltName; - [ maxZ ] = emc_get_max_specimen_NZ(subTomoMeta.mapBackGeometry.tomoName, ... + [ maxZ ] = emc_get_max_specimen_NZ( ... subTomoMeta.mapBackGeometry.tomoCoords, ... tomoList, ... n_active_tomos, ... diff --git a/transformations/emc_get_max_specimen_NZ.m b/transformations/emc_get_max_specimen_NZ.m index 5fde9fdf..6942f092 100644 --- a/transformations/emc_get_max_specimen_NZ.m +++ b/transformations/emc_get_max_specimen_NZ.m @@ -1,37 +1,17 @@ -function [ max_specimen_nz, tomoIdx ] = emc_get_max_specimen_NZ(subTomoMeta_tomoName, subTomoMeta_coords, tomo_name_list, n_tomograms, samplingRate) +function [ max_specimen_nz ] = emc_get_max_specimen_NZ(subTomoMeta_coords, tomo_name_list, n_tomograms, samplingRate) max_z_value = -inf; min_z_value = inf; - tomoIdx = zeros(n_tomograms,1); - for iTomo = 1:n_tomograms - % subTomoMeta_tomoName = subTomoMeta.mapBackGeometry.tomoName - % subTomoMeta_coords = subTomoMeta.mapBackGeometry.(tiltName).coords - if isa(subTomoMeta_tomoName,'struct') - tomoIdx(iTomo) = subTomoMeta_tomoName.(tomo_name_list{iTomo}).tomoIdx; - else - tomoIdx(iTomo) = iTomo; - end - + for iTomo = 1:n_tomograms % 4 is the unbinned pixel size of the tomogram in Z % 6 is location of the specimen origin in Z relative to the origin of the tomogram if isa(subTomoMeta_coords,'cell') nZ = subTomoMeta_coords{iTomo}.NZ; dZ = subTomoMeta_coords{iTomo}.dZ_specimen_to_tomo; else - - if isfield(subTomoMeta_coords, 'NZ') - nZ = subTomoMeta_coords.NZ; - dZ = subTomoMeta_coords.dZ_specimen_to_tomo; - else - if isfield(subTomoMeta_coords, tomo_name_list{iTomo}) - nZ = subTomoMeta_coords.(tomo_name_list{iTomo}).NZ; - dZ = subTomoMeta_coords.(tomo_name_list{iTomo}).dZ_specimen_to_tomo; - else - error('The field NZ or the field %s is not present in the subTomoMeta_coords', tomo_name_list{iTomo}); - end - end + error('The field NZ or the field %s is not present in the subTomoMeta_coords', tomo_name_list{iTomo}); end nZ = nZ ./ samplingRate; From 81e6437351e624899f224367dbe30c7f43ac5d22 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 29 Feb 2024 18:07:44 -0500 Subject: [PATCH 054/151] Little rat fucking matlab - half was not recognized in the compiled code due to licensing something or the other bullshit. Removed all half and just cast with new emc_halfcast calling mexFP16 under the hood, now bidirectional.) --- @MRCImage/getModeString.m | 2 +- @MRCImage/getVolume.m | 5 ++--- @MRCImage/loadVolume.m | 3 ++- @MRCImage/save.m | 9 +++++---- metaData/OPEN_IMG.m | 4 ++-- metaData/SAVE_IMG.m | 4 +--- mexFiles/mexCompile.m | 2 +- testScripts/emc_halfcast.m | 13 +++++++++++++ transformations/BH_unStackMontage4d.m | 2 +- 9 files changed, 28 insertions(+), 16 deletions(-) create mode 100644 testScripts/emc_halfcast.m diff --git a/@MRCImage/getModeString.m b/@MRCImage/getModeString.m index fbd56a10..a13e00ae 100644 --- a/@MRCImage/getModeString.m +++ b/@MRCImage/getModeString.m @@ -35,7 +35,7 @@ modeString = 'int16*2'; % used for complex short ints case 4 modeString = 'float32*2'; % used for complex floating point -case 6 + case 6 modeString = 'uint16'; case 12 modeString = 'half'; % 16-bit floating point diff --git a/@MRCImage/getVolume.m b/@MRCImage/getVolume.m index 292a9f3a..fc1316a7 100644 --- a/@MRCImage/getVolume.m +++ b/@MRCImage/getVolume.m @@ -286,14 +286,13 @@ % vol(~topHalf) = vol(~topHalf) - 128; %end + if ( flgReCast ) if ( flgComplex ) vol = complex(single(vol{1}),single(vol{2})); else if mRCImage.header.mode == 12 - vol = half.typecast(vol); - else - vol = single(vol); + vol = emc_halfcast(vol); end end else diff --git a/@MRCImage/loadVolume.m b/@MRCImage/loadVolume.m index 3ad1d269..753ceb9b 100644 --- a/@MRCImage/loadVolume.m +++ b/@MRCImage/loadVolume.m @@ -68,7 +68,8 @@ mRCImage.flgVolume = 1; if mRCImage.header.mode == 12 - mRCImage.volume = half.typecast(mRCImage.volume); + % FIXME: should the header mode be changed? + mRCImage.volume = emc_halfcast(mRCImage.volume); end mRCImage.volume = reshape(mRCImage.volume, ... mRCImage.header.nX, ... diff --git a/@MRCImage/save.m b/@MRCImage/save.m index 7bbd3727..d3bde741 100644 --- a/@MRCImage/save.m +++ b/@MRCImage/save.m @@ -100,9 +100,9 @@ else % normal (not complex) data count = fwrite(mRCImage.fid, mRCImage.volume, modeStr); if count ~= nElements - if mRCImage.header.mode == 12 - mRCImage.volume = typecast(mRCImage.volume, 'half'); - end + % if mRCImage.header.mode == 12 + % mRCImage.volume = typecast(mRCImage.volume, 'half'); + % end fprintf('Matrix contains %d but only wrote %d elements\n', ... nElements, count); PEETError('Failed writing matrix!'); @@ -110,7 +110,8 @@ end end if mRCImage.header.mode == 12 - mRCImage.volume = half.typecast(mRCImage.volume); + % FIXME: should the header mode be changed? + mRCImage.volume = emc_halfcast(mRCImage.volume); end close(mRCImage); diff --git a/metaData/OPEN_IMG.m b/metaData/OPEN_IMG.m index 08bfd851..e76a4534 100644 --- a/metaData/OPEN_IMG.m +++ b/metaData/OPEN_IMG.m @@ -1,5 +1,6 @@ function [ vol ] = OPEN_IMG(precision, filename, varargin) + if isa(filename, 'MRCImage') vol = getVolume(filename, varargin{:}); else @@ -10,10 +11,9 @@ vol = single(vol); elseif strcmp(precision, 'double') vol = double(vol); - elseif strcmp(precision, 'half') - vol = half(vol); else error('Unknown precision'); end + end \ No newline at end of file diff --git a/metaData/SAVE_IMG.m b/metaData/SAVE_IMG.m index 84e774e3..66821205 100644 --- a/metaData/SAVE_IMG.m +++ b/metaData/SAVE_IMG.m @@ -26,9 +26,7 @@ if isa(varargin{1}, 'cell') if strcmp(varargin{1}{2}, 'half') mRCImage.header.mode = 12; - tmp_vol = zeros(size(vol), 'uint16'); - mexFP16(single(vol), tmp_vol); - mRCImage.volume = tmp_vol; + mRCImage.volume = emc_halfcast(vol); varargin{1} = varargin{1}{1}; else error('Unknown option for SAVE_IMG, should be {filename, half}'); diff --git a/mexFiles/mexCompile.m b/mexFiles/mexCompile.m index 64324841..a231d070 100644 --- a/mexFiles/mexCompile.m +++ b/mexFiles/mexCompile.m @@ -2,7 +2,7 @@ fprintf("\n\nCompile here\n\n"); mexPATH = '/sa_shared/git/emClarity/mexFiles'; -CUDA_LIB = '-L/usr/local/cuda/lib64 '; ... % NOTE if you leave a space at the end of this string, MATLAB does not parse the option correctly (which wouldn't matter in a normal compile line!) +CUDA_LIB = '-L/usr/local/cuda/lib64'; ... % NOTE if you leave a space at the end of this string, MATLAB does not parse the option correctly (which wouldn't matter in a normal compile line!) getenv('MW_NVCC_PATH') getenv('CUDA_HOME') diff --git a/testScripts/emc_halfcast.m b/testScripts/emc_halfcast.m new file mode 100644 index 00000000..5e2834c6 --- /dev/null +++ b/testScripts/emc_halfcast.m @@ -0,0 +1,13 @@ +function [output_vol] = emc_halfcast(input_vol) + + if isa(input_vol, 'uint16') + output_vol = zeros(size(input_vol), 'single'); + mexFP16(output_vol, input_vol, false); + elseif isa(input_vol, 'single') + output_vol = zeros(size(input_vol), 'uint16'); + mexFP16(input_vol, output_vol, true); + else + error('Unknown precision'); + end + +end \ No newline at end of file diff --git a/transformations/BH_unStackMontage4d.m b/transformations/BH_unStackMontage4d.m index e62df85c..c7b893d4 100755 --- a/transformations/BH_unStackMontage4d.m +++ b/transformations/BH_unStackMontage4d.m @@ -33,7 +33,7 @@ -montage = OPEN_IMG('single', MRCImage(NAME); +montage = OPEN_IMG('single', MRCImage(NAME)); ind = LOCATIONS; From e2c48aa4e011ba0fc92049f84a3ac38952992d21 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 1 Mar 2024 10:21:32 -0500 Subject: [PATCH 055/151] Sorted through all the nitty gritty to get slabs to match. Also test supersample. --- ctf/BH_ctf_Correct3d.m | 105 ++++++++++++++++++++----------- metaData/BH_geometryInitialize.m | 66 +++++++++++-------- 2 files changed, 111 insertions(+), 60 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index b22a74b7..8c2abef3 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -313,7 +313,7 @@ % For now, since the tilt geometry is not necessarily updated (it is manual) % in the subTomoMeta, check that newer (possible perTilt refined) data is % not present. - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt},mapBackIter+1); + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', tiltList{iTilt}, mapBackIter+1); TLT = load(TLTNAME); % Get all the tomogram names that belong to a given tilt-series. @@ -321,7 +321,7 @@ if (recon_for_subTomo || recon_for_templateMatching) if (recon_for_subTomo) % List of all possible tomos, some may be "in-active" since this is post-template matching - tomoList = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).(tomoList); + tomoList = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).tomoList; else % List of all possible tomos, all are "active" since this is pre-template matching tomoList = tiltTomoList{iTilt}; @@ -336,7 +336,7 @@ for iTomo = 1:nTomos % The order of tomo num could be off but only if all are present do we % skip. - checkRecon = sprintf('cache/%s_%d_bin%d%s.rec', tiltList{iTilt}, iTomo, samplingRate, filtered); + checkRecon = sprintf('cache/%s_bin%d%s.rec', tomoList{iTomo}, samplingRate, filtered); if exist(checkRecon, 'file') try % Could have a corrupt file @@ -381,14 +381,14 @@ for iTilt = iterList{iParProc} slab_list = {}; - TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt',tiltList{iTilt}, mapBackIter + 1 ); + TLTNAME = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', tiltList{iTilt}, mapBackIter + 1 ); TLT = load(TLTNAME); fprintf('iParProc %d and iTilt %d using TLT %s\n', iParProc, iTilt, TLTNAME); if (recon_for_subTomo || recon_for_templateMatching) if (recon_for_subTomo) % List of all possible tomos, some may be "in-active" since this is post-template matching - tomoList = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).(tomoList); + tomoList = subTomoMeta.mapBackGeometry.(tiltList{iTilt}).tomoList; else % List of all possible tomos, all are "active" since this is pre-template matching tomoList = tiltTomoList{iTilt}; @@ -569,7 +569,8 @@ for iTomo = 1:nTomos if (slab_list{iTomo}(iSection,1)) - reconName = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, iTomo, iSection); + this_tomo_idx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + reconName = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, this_tomo_idx, iSection); if (recon_for_tomoCPR) TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{1}),1); @@ -577,7 +578,7 @@ end if (recon_for_subTomo) - TA = sortrows(subTomoMeta.tiltGeometry.(sprintf('%s_%d',tiltList{iTilt},iTomo)),1); + TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{iTomo}),1); TA = TA(:,4); end @@ -586,21 +587,21 @@ if (mapBackIter) % FIXME: I don't think this block should work, it should only be the tilt angles! error('THis block should not be reached.') - TA = load(sprintf('%smapBack%d/%s_ali%d_ctf.tlt',CWD,mapBackIter,tiltList{iTilt}, mapBackIter)); + TA = load(sprintf('%smapBack%d/%s_ali%d_ctf.tlt', CWD, mapBackIter, tiltList{iTilt}, mapBackIter)); else - TA = load(sprintf('%sfixedStacks/%s.tlt',CWD,tiltList{iTilt})); + TA = load(sprintf('%sfixedStacks/%s.tlt', CWD, tiltList{iTilt})); end end - rawTLT = sprintf('cache/%s_%d.rawtlt',tiltList{iTilt},iTomo); + rawTLT = sprintf('cache/%s.rawtlt', tomoList{iTomo}); rawTLT_file = fopen(rawTLT, 'w'); fprintf(rawTLT_file,'%f\n', TA'); fclose(rawTLT_file); if (mapBackIter) - LOCAL = sprintf('%smapBack%d/%s_ali%d_ctf.local',CWD,mapBackIter,tiltList{iTilt}, mapBackIter); + LOCAL = sprintf('%smapBack%d/%s_ali%d_ctf.local', CWD, mapBackIter, tiltList{iTilt}, mapBackIter); else - LOCAL = sprintf('%sfixedStacks/%s.local',CWD,tiltList{iTilt}); + LOCAL = sprintf('%sfixedStacks/%s.local', CWD, tiltList{iTilt}); end if exist(LOCAL,'file') @@ -650,9 +651,9 @@ reconScaling = 1; % Explicitly set Radial to Nyquist if (flgLocal) - rCMD = [rCMD sprintf('-LOCALFILE %s -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d',LOCAL,reconScaling)]; + rCMD = [rCMD sprintf('-LOCALFILE %s -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d', LOCAL, reconScaling)]; else - rCMD = [rCMD sprintf('-RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d',reconScaling)]; + rCMD = [rCMD sprintf('-RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d', reconScaling)]; end if isfile(sprintf('%s.sh',reconName)) @@ -661,7 +662,7 @@ recScript = fopen(sprintf('%s.sh',reconName),'w'); fprintf(recScript,'#!/bin/bash\n\n'); - fprintf(recScript,'%s -SLICE -1,-1 -TOTALSLICES %d,%d\n',rCMD,totalSlices); + fprintf(recScript,'%s -SLICE -1,-1 -TOTALSLICES %d,%d\n', rCMD, totalSlices); iShift = 1; for iChunk = 1:length(tiltChunks)-1 @@ -677,23 +678,23 @@ fprintf(recScript,'\n\nwait\n\n'); fclose(recScript); pause(1); - system(sprintf('chmod a=wrx %s.sh',reconName)); + system(sprintf('chmod a=wrx %s.sh', reconName)); - [recError,~] = system(sprintf('%s.sh > /dev/null',reconName)); % /dev/null + [recError,~] = system(sprintf('%s.sh > /dev/null', reconName)); % /dev/null if (recError) system(sprintf('%s.sh',reconName)); - error('\n\nerror during reconstruction %s\n\n',reconName); + error('\n\nerror during reconstruction %s\n\n', reconName); end % Z coords (y in this orientation) are decreasing into the % monitor. For symmetrical padding this doesn't matter, but keep % in mind. /dev/null trimCMD = sprintf('trimvol -mode 12 -rx -y %d,%d %s.TMPPAD %s > /dev/null ' , ... - 1,floor(round(slab_list{iTomo}(iSection,5))),reconName,reconName); + 1,floor(round(slab_list{iTomo}(iSection,5))), reconName, reconName); [msg,~]= system(trimCMD); if (msg) fprintf('%d from trimCMD\n',msg) trimCMDPrintError = sprintf('trimvol -mode 12 -rx -y %d,%d %s.TMPPAD %s', ... - 1,floor(round(slab_list{iTomo}(iSection,5))),reconName,reconName); + 1+slab_list{iTomo}(iSection,3),floor(round(slab_list{iTomo}(iSection,5)))-slab_list{iTomo}(iSection,4), reconName, reconName); system(trimCMDPrintError); error('error during trimvol'); end @@ -715,11 +716,11 @@ % Note that bh_global_turn_on_phase_plate could be true for any of the recon_for_stage bools, so it must % be checked first. if (bh_global_turn_on_phase_plate(1)) - reconNameFull = sprintf('cache/%s_%d_bin%d_filtered.rec', tiltList{iTilt}, iTomo, samplingRate); + reconNameFull = sprintf('cache/%s_bin%d_filtered.rec', tomoList{iTomo}, samplingRate); elseif recon_for_tomoCPR - reconNameFull = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec', CWD,tiltList{iTilt},iTomo,samplingRate); + reconNameFull = sprintf('%scache/%s_bin%d_backgroundEst.rec', CWD, tomoList{iTomo}, samplingRate); else - reconNameFull = sprintf('cache/%s_%d_bin%d.rec', tiltList{iTilt},iTomo,samplingRate); + reconNameFull = sprintf('cache/%s_bin%d.rec', tomoList{iTomo},samplingRate); end fprintf('in ctf3d reconNameFull is %s\n\n',reconNameFull); @@ -750,10 +751,11 @@ % for iSection = 1:n_slabs_to_reconstruct for iSection = slab_order if (slab_list{iTomo}(iSection,1)) - this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, iTomo, iSection); + this_tomo_idx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, this_tomo_idx, iSection); cleanup3 = sprintf('%s %s',cleanup3,this_slab); fprintf(recombineCMD, '%s\n', this_slab); - fprintf(recombineCMD, '1-%d\n',floor(round(slab_list{iTomo}(iSection,5)))); + fprintf(recombineCMD, '%d-%d\n',1+slab_list{iTomo}(iSection,3),floor(round(slab_list{iTomo}(iSection,5)))-slab_list{iTomo}(iSection,4)); end end fclose(recombineCMD); @@ -848,13 +850,14 @@ tomo_origin_in_tomo_frame = emc_get_origin_index(iCoords{iTomo}.NZ ./ samplingRate); fraction_origin_shift = tomo_origin_wrt_tilt_origin - round(tomo_origin_wrt_tilt_origin); - - tomogram_lower_bound = floor((tomo_origin_wrt_tilt_origin - tomo_origin_in_tomo_frame)); - recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + ceil(iCoords{iTomo}.NZ./samplingRate) - 1; + wanted_NZ = ceil(iCoords{iTomo}.NZ./samplingRate); + + tomogram_lower_bound = floor((tomo_origin_wrt_tilt_origin - tomo_origin_in_tomo_frame)) + 1; + recon_range_z_in_specimen_frame = tomogram_lower_bound : tomogram_lower_bound + wanted_NZ - 1; % For each slab see if this tomogram has any sections in it for iSlab = 1:n_slabs_to_reconstruct - - slab_origin_in_specimen_frame = ((n_slabs_to_reconstruct-1)/-2+(iSlab-1)) * slab_size_pixels; + slab_idx = ((n_slabs_to_reconstruct-1)/-2+(iSlab-1)); + slab_origin_in_specimen_frame = slab_idx * slab_size_pixels; slab_lower_bound = slab_origin_in_specimen_frame - oS + 1; slab_upper_bound = slab_origin_in_specimen_frame + oS - 1; @@ -864,6 +867,7 @@ valid_indices = recon_range_z_in_specimen_frame(is_in_range); slab_list{iTomo}(iSlab,5) = length(valid_indices); + if (slab_list{iTomo}(iSlab,5) > 0) slab_list{iTomo}(iSlab,1) = 1; else @@ -876,31 +880,62 @@ % This means a slab at Z > 0 needs to be shifted in the negative direction, which means supplying % a shift that is also > 0, moving the volume "up" in the rotated coordinate system (imod -Z) % I know ... this is a shit show. - dZ_for_reconstructed_slab = (valid_indices(valid_region_origin) + fraction_origin_shift); - + slab_list{iTomo}(iSlab,2) = fraction_origin_shift; + + dZ_for_reconstructed_slab = (valid_indices(valid_region_origin) - fraction_origin_shift); slab_list{iTomo}(iSlab,6) = dZ_for_reconstructed_slab; %dZ end % Check to ensure we don't have any tiny slabs leftover, if so, merge them into a neighboring slab biggest_slab = max(slab_list{iTomo}(:,5)); for iSlab = 1:n_slabs_to_reconstruct - if (slab_list{iTomo}(iSlab,1) && slab_list{iTomo}(iSlab, 5) / biggest_slab < 0.1) + if (slab_list{iTomo}(iSlab,1) && (~mod(slab_list{iTomo}(iSlab, 5),2) || slab_list{iTomo}(iSlab, 5) / biggest_slab < 0.2)) if (iSlab > 1 && slab_list{iTomo}(iSlab-1,1)) delta = slab_list{iTomo}(iSlab,5); slab_list{iTomo}(iSlab-1,5) = slab_list{iTomo}(iSlab-1,5) + delta; slab_list{iTomo}(iSlab,1) = 0; % we are adding slices from above the specimen in Z so the z shift is positive - slab_list{iTomo}(iSlab-1,6) = (slab_list{iTomo}(iSlab-1,6) + delta); + slab_list{iTomo}(iSlab-1,6) = (slab_list{iTomo}(iSlab-1,6) + floor(delta/2)); elseif (iSlab < n_slabs_to_reconstruct && slab_list{iTomo}(iSlab+1,1)) delta = slab_list{iTomo}(iSlab,5); slab_list{iTomo}(iSlab+1,5) = slab_list{iTomo}(iSlab+1,5) + slab_list{iTomo}(iSlab,5); slab_list{iTomo}(iSlab,1) = 0; % we are adding slices from below the specimen in Z so the z shift is negative - slab_list{iTomo}(iSlab+1,6) = (slab_list{iTomo}(iSlab+1,6) - delta); + slab_list{iTomo}(iSlab+1,6) = (slab_list{iTomo}(iSlab+1,6) - floor(delta/2)); end end end + % The only time we should have an even Z dimension now is if there is only one slab. if so, pad the top end for reconstruction and trim it off later + for iSlab = 1:n_slabs_to_reconstruct + slab_idx = ((n_slabs_to_reconstruct-1)/-2+(iSlab-1)); + + if (mod(slab_list{iTomo}(iSlab,5),2) == 0) + slab_list{iTomo}(iSlab,5) = slab_list{iTomo}(iSlab,5) + 1; + if (slab_idx > 0) + slab_list{iTomo}(iSlab,3) = 1; + else + slab_list{iTomo}(iSlab,4) = 1; + end + end + end + + % % If the slab thickness is even, we need to account for the difference in definition of imod origin, this can happen at the boundaries + % % the origin in imod is -0.5 for even images, but we are applying a shift to the image, so we add +0.5 + % for iSlab = 1:n_slabs_to_reconstruct + % if (slab_list{iTomo}(iSlab,1)) + % if (mod(slab_list{iTomo}(iSlab,5),2) == 0) + % % if (slab_list{iTomo}(iSlab,5) < 0) + % % slab_list{iTomo}(iSlab,6) = slab_list{iTomo}(iSlab,6) - 0.5; + % % else + % slab_list{iTomo}(iSlab,6) = slab_list{iTomo}(iSlab,6) - 0.5; + % % end + % end + % end + % slab_list{iTomo}(iSlab,6) = slab_list{iTomo}(iSlab,6) + 1; + % end + + % TroubleShoot tSHT = fopen(sprintf('.tblSht_%s_i%d.txt',tiltName,iTomo),'w'); fprintf(tSHT,'%2.2f %2.2f %2.2f %2.2f %2.2f %2.2f\n', iCoords{iTomo}.NX, iCoords{iTomo}.NY, iCoords{iTomo}.NZ, iCoords{iTomo}.dX_specimen_to_tomo, iCoords{iTomo}.dY_specimen_to_tomo, iCoords{iTomo}.dZ_specimen_to_tomo); diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index 0bc53d86..f32cfe2b 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -170,8 +170,9 @@ subTomoMeta.('mapBackGeometry').(tiltName).('nTomos') = nTomos; subTomoMeta.('mapBackGeometry').(tiltName).('tomoCprRePrjSize') = 512; stm_tomoList = cell(nTomos,1); + + n_skipped = 0; for iTomo = 1:nTomos - if (doImport) modName = strsplit(iPath(iTomo).name,'.csv'); modName = strsplit(modName{1},'_'); @@ -180,32 +181,47 @@ modName = strsplit(iPath(iTomo).name,'_'); tomoIdx = EMC_str2double(modName{end-1}); end - - % We are storing this info to make it available when checking for duplicates - tomoName = sprintf('%s_%d', tiltName,tomoIdx); - fileInfo{n_tomos_added,1} = tiltName; - fileInfo{n_tomos_added,2} = tomoName; + tomoName = sprintf('%s_%d', tiltName, tomoIdx); stm_tomoList{iTomo} = tomoName; - fileInfo{n_tomos_added,3} = sprintf('%s_%d_bin%d',tiltName, tomoIdx, dupSampling); - - subTomoMeta.('tiltGeometry').(fileInfo{n_tomos_added,2}) = tilt_geometry; - n_tomos_added = n_tomos_added + 1; - - % Store a reference to the parent tilt-series for every tomogram - subTomoMeta.('mapBackGeometry').('tomoName').(tomoName).('tiltName') = tiltName; - % Store the tomoIdx for every tomogram, currently used to refer back to recGEom, but I'm going to put this into a struct - subTomoMeta.('mapBackGeometry').('tomoName').(tomoName).('tomoIdx') = tomoIdx; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('is_active') = true; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_i') = recGeom{tomoIdx}.y_i; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_f') = recGeom{tomoIdx}.y_f; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NX') = recGeom{tomoIdx}.NX; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NY') = recGeom{tomoIdx}.NY; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NZ') = recGeom{tomoIdx}.NZ; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = recGeom{tomoIdx}.dX_specimen_to_tomo; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dY_specimen_to_tomo') = recGeom{tomoIdx}.dY_specimen_to_tomo; - subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dZ_specimen_to_tomo') = recGeom{tomoIdx}.dZ_specimen_to_tomo; - end + isMemberList = ismember(tmpTomoList, tomoName); + if sum(isMemberList) > 1 + error('Found more than one match for %s in the recon/*.coords file\n',tomoName); + elseif sum(isMemberList) == 1 + iCoords = recGeom{isMemberList}; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('is_active') = true; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_i') = iCoords.y_i; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('y_f') = iCoords.y_f; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NX') = iCoords.NX; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NY') = iCoords.NY; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('NZ') = iCoords.NZ; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dX_specimen_to_tomo') = iCoords.dX_specimen_to_tomo; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dY_specimen_to_tomo') = iCoords.dY_specimen_to_tomo; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('dZ_specimen_to_tomo') = iCoords.dZ_specimen_to_tomo; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('tilt_NX') = iCoords.tilt_NX; + subTomoMeta.('mapBackGeometry').('tomoCoords').(tomoName).('tilt_NY') = iCoords.tilt_NY; + + % We are storing this info to make it available when checking for duplicates + fileInfo{n_tomos_added,1} = tiltName; + fileInfo{n_tomos_added,2} = tomoName; + fileInfo{n_tomos_added,3} = sprintf('%s_bin%d', tomoName, dupSampling); + + subTomoMeta.('tiltGeometry').(fileInfo{n_tomos_added,2}) = tilt_geometry; + % Store a reference to the parent tilt-series for every tomogram + subTomoMeta.('mapBackGeometry').('tomoName').(tomoName).('tiltName') = tiltName; + % Store the tomoIdx for every tomogram, currently used to refer back to recGEom, but I'm going to put this into a struct + subTomoMeta.('mapBackGeometry').('tomoName').(tomoName).('tomoIdx') = tomoIdx; + + n_tomos_added = n_tomos_added + 1; + else + n_skipped = n_skipped + 1; + end + end + if (n_skipped) + fprintf('nTomosPossible = %d, nTomos = %d, n_skipped = %d\n',nTomosPossible,nTomos,n_skipped); + error('The number of model files in convmap/*.mod is not equal to the number in the recon/*.coords\n'); + end + subTomoMeta.('mapBackGeometry').(tiltName).('tomoList') = stm_tomoList; end % end of loop over stacks From 24823e912b7c9b187cfd4bc902f9feaae794c9df Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 1 Mar 2024 11:04:24 -0500 Subject: [PATCH 056/151] Remove even check in small slab movment as this was incorrect, revealed with 3 slabs rather than 7) --- ctf/BH_ctf_Correct3d.m | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 8c2abef3..25f8a635 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -889,19 +889,19 @@ % Check to ensure we don't have any tiny slabs leftover, if so, merge them into a neighboring slab biggest_slab = max(slab_list{iTomo}(:,5)); for iSlab = 1:n_slabs_to_reconstruct - if (slab_list{iTomo}(iSlab,1) && (~mod(slab_list{iTomo}(iSlab, 5),2) || slab_list{iTomo}(iSlab, 5) / biggest_slab < 0.2)) + if (slab_list{iTomo}(iSlab,1) && (slab_list{iTomo}(iSlab, 5) / biggest_slab < 0.2)) if (iSlab > 1 && slab_list{iTomo}(iSlab-1,1)) delta = slab_list{iTomo}(iSlab,5); slab_list{iTomo}(iSlab-1,5) = slab_list{iTomo}(iSlab-1,5) + delta; slab_list{iTomo}(iSlab,1) = 0; % we are adding slices from above the specimen in Z so the z shift is positive - slab_list{iTomo}(iSlab-1,6) = (slab_list{iTomo}(iSlab-1,6) + floor(delta/2)); + slab_list{iTomo}(iSlab-1,6) = (slab_list{iTomo}(iSlab-1,6) + ceil(delta/2)); elseif (iSlab < n_slabs_to_reconstruct && slab_list{iTomo}(iSlab+1,1)) delta = slab_list{iTomo}(iSlab,5); slab_list{iTomo}(iSlab+1,5) = slab_list{iTomo}(iSlab+1,5) + slab_list{iTomo}(iSlab,5); slab_list{iTomo}(iSlab,1) = 0; % we are adding slices from below the specimen in Z so the z shift is negative - slab_list{iTomo}(iSlab+1,6) = (slab_list{iTomo}(iSlab+1,6) - floor(delta/2)); + slab_list{iTomo}(iSlab+1,6) = (slab_list{iTomo}(iSlab+1,6) - ceil(delta/2)); end end end @@ -910,12 +910,12 @@ for iSlab = 1:n_slabs_to_reconstruct slab_idx = ((n_slabs_to_reconstruct-1)/-2+(iSlab-1)); - if (mod(slab_list{iTomo}(iSlab,5),2) == 0) + if (slab_list{iTomo}(iSlab,1) > 0 && mod(slab_list{iTomo}(iSlab,5),2) == 0) slab_list{iTomo}(iSlab,5) = slab_list{iTomo}(iSlab,5) + 1; if (slab_idx > 0) - slab_list{iTomo}(iSlab,3) = 1; + slab_list{iTomo}(iSlab,3) = slab_list{iTomo}(iSlab,3) + 1; else - slab_list{iTomo}(iSlab,4) = 1; + slab_list{iTomo}(iSlab,4) = slab_list{iTomo}(iSlab,4) + 1; end end end From a8bc3397b5bea1e94be471b96249898ca45b22f5 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 1 Mar 2024 11:21:54 -0500 Subject: [PATCH 057/151] Set super sample and expand lines to 3 and true by default in parameter file. --- ctf/BH_ctf_Correct3d.m | 44 ++++++++++++-------------------- metaData/BH_parseParameterFile.m | 10 +++++++- 2 files changed, 25 insertions(+), 29 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 25f8a635..69235074 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -32,35 +32,27 @@ % Test David's new super sampling in reconstruction. No check that this % version (currently 4.10.40) is properly sourced. -try - super_sample = emc.('super_sample'); - if (super_sample > 0) - [~,v] = system('cat $IMOD_DIR/VERSION'); - v = split(v,'.'); - if (EMC_str2double(v{1}) < 4 || (EMC_str2double(v{2}) <= 10 && EMC_str2double(v{3}) < 42)) - fprintf('Warning: imod version is too old for supersampling\n'); - super_sample = ''; - else - super_sample = sprintf(' -SuperSampleFactor %d',super_sample); - end - else + +super_sample = emc.('super_sample'); +if (super_sample > 0) + [~,v] = system('cat $IMOD_DIR/VERSION'); + v = split(v,'.'); + if (EMC_str2double(v{1}) < 4 || (EMC_str2double(v{2}) <= 10 && EMC_str2double(v{3}) < 42)) + fprintf('Warning: imod version is too old for supersampling\n'); super_sample = ''; + else + super_sample = sprintf(' -SuperSampleFactor %d',super_sample); end - -catch +else super_sample = ''; end + - -try - expand_lines = emc.('expand_lines'); - if isempty(super_sample) || expand_lines == false - expand_lines = ''; - else - expand_lines = ' -ExpandInputLines'; - end -catch +expand_lines = emc.('expand_lines'); +if isempty(super_sample) || expand_lines == false expand_lines = ''; +else + expand_lines = ' -ExpandInputLines'; end fprintf('\n Superampling in imod is [%s] with expandLines [%s]\n',super_sample ,expand_lines); @@ -912,11 +904,7 @@ if (slab_list{iTomo}(iSlab,1) > 0 && mod(slab_list{iTomo}(iSlab,5),2) == 0) slab_list{iTomo}(iSlab,5) = slab_list{iTomo}(iSlab,5) + 1; - if (slab_idx > 0) - slab_list{iTomo}(iSlab,3) = slab_list{iTomo}(iSlab,3) + 1; - else - slab_list{iTomo}(iSlab,4) = slab_list{iTomo}(iSlab,4) + 1; - end + slab_list{iTomo}(iSlab,4) = slab_list{iTomo}(iSlab,4) + 1; end end diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 4bd09c73..e9db8f6d 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -590,7 +590,15 @@ emc.max_ctf3dDepth = 100*10^-9; end +if isfield(emc, 'expand_lines') + EMC_assert_boolean(emc.expand_lines); +else + emc.expand_lines = true; +end - +if isfield(emc, 'super_sample') + EMC_assert_numeric(emc.super_sample, 3, [2, 5]); +else + emc.super_sample = 3; end From 44fea6575cf221d02897799b96e6eb46374a123c Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 4 Mar 2024 14:50:50 -0500 Subject: [PATCH 058/151] wip --- alignment/BH_alignRaw3d_v2.m | 2 +- ctf/BH_ctf_Correct3d.m | 44 +++++++++++++++-------- metaData/BH_parseParameterFile.m | 3 +- mexFiles/mexCompile.m | 4 +++ mexFiles/mexFFT.cu | 15 ++++++-- synthetic/BH_to_cisTEM_mapBack.m | 35 +++++++++--------- testScripts/emClarity.m | 11 +++--- testScripts/mCompile.sh | 2 +- transformations/emc_get_max_specimen_NZ.m | 3 ++ 9 files changed, 76 insertions(+), 43 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index ec035832..355b3890 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -663,7 +663,7 @@ end - sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) + % sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) tic; % Load the tomo into gpu diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 69235074..c3d4fe35 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -248,6 +248,8 @@ tiltRecGeom = 0; tomoList = {}; nTomos= 0; +tiltRecGeom = {}; +tiltTomoList = {}; if (recon_for_subTomo) [tiltList, nTilts] = BH_returnIncludedTilts(subTomoMeta.mapBackGeometry); else @@ -286,17 +288,22 @@ % If there is only one tilt, things break in a weird way nGPUs = min(nGPUs, nTilts); -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores); +% For now, limit the number of processes to avoid memory issues +% TODO: determine something more precise +n_cores_wanted = min(emc.nCpuCores, floor(emc.pixel_size_angstroms*0.7)*nGPUs); +[ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, n_cores_wanted); -% try -% EMC_parpool(nParProcesses) -% catch -% delete(gcp('nocreate')) -% EMC_parpool(nParProcesses) -% end +try + EMC_parpool(nParProcesses) +catch + delete(gcp('nocreate')) + EMC_parpool(nParProcesses) +end -% parfor iParProc = 1:nParProcesses -for iParProc = 1:nParProcesses %%revert + + +parfor iParProc = 1:nParProcesses +% for iParProc = 1:nParProcesses %%revert % iGPU = mod(iParProc,nGPUs); for iTilt = iterList{iParProc} nTomos = 0; @@ -362,8 +369,8 @@ end % All data is handled through disk i/o so everything unique created in the -% parfor iParProc = 1:nParProcesses - for iParProc = 1:nParProcesses %%revert +parfor iParProc = 1:nParProcesses + % for iParProc = 1:nParProcesses %%revert iGPU = mod(iParProc,nGPUs); % for iGPU = 1:nGPUs %%revert @@ -561,7 +568,12 @@ for iTomo = 1:nTomos if (slab_list{iTomo}(iSection,1)) - this_tomo_idx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + if (recon_for_templateMatching) + this_tomo_idx = iTomo; + else + this_tomo_idx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + end + reconName = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, this_tomo_idx, iSection); if (recon_for_tomoCPR) @@ -636,7 +648,7 @@ iGPU, ... floor(iCoords{iTomo}.NX ./ samplingRate),... % WIDTH = NX floor(round(slab_list{iTomo}(iSection,5))), ... % THICKNESS = NZ - iCoords{iTomo}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X + -iCoords{iTomo}.dX_specimen_to_tomo ./ samplingRate, ... % SHIFT X is negative dX which describes the vector from the specimen origin to the tomo origin slab_list{iTomo}(iSection,6)); @@ -743,7 +755,11 @@ % for iSection = 1:n_slabs_to_reconstruct for iSection = slab_order if (slab_list{iTomo}(iSection,1)) - this_tomo_idx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + if (recon_for_templateMatching) + this_tomo_idx = iTomo; + else + this_tomo_idx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + end this_slab = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, this_tomo_idx, iSection); cleanup3 = sprintf('%s %s',cleanup3,this_slab); fprintf(recombineCMD, '%s\n', this_slab); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index e9db8f6d..2dd6a76c 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -597,7 +597,8 @@ end if isfield(emc, 'super_sample') - EMC_assert_numeric(emc.super_sample, 3, [2, 5]); + emc.super_sample + EMC_assert_numeric(emc.super_sample, 1, [2, 5]); else emc.super_sample = 3; end diff --git a/mexFiles/mexCompile.m b/mexFiles/mexCompile.m index a231d070..293bd21f 100644 --- a/mexFiles/mexCompile.m +++ b/mexFiles/mexCompile.m @@ -25,6 +25,10 @@ % --warn-on-spills % -Wno-deprecated-gpu-targets % '-lcublas_static' ... % Link to cuBLAS + +% '-lcufft_static_nocallback' ... +% '-lculibos' ... +% '-lcudart_static' ... mexcuda_opts = { ... CUDA_LIB ... '-lmwlapack' ... % Link to LAPACK diff --git a/mexFiles/mexFFT.cu b/mexFiles/mexFFT.cu index f6f073bb..662ada7e 100644 --- a/mexFiles/mexFFT.cu +++ b/mexFiles/mexFFT.cu @@ -86,9 +86,18 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) if (numel_input == 1) { // mexPrintf("Destroying the plans\n"); - cufftDestroy(*plan); - cufftDestroy(*planInv); - mxGPUDestroyGPUArray(inputArray); + if ( mxGPUIsValidGPUData(prhs[2])) + cufftDestroy(*plan); + // else + // mexPrintf("The fwd plan is not valid in destructor\n"); + if ( mxGPUIsValidGPUData(prhs[3])) + cufftDestroy(*planInv); + // else + // mexPrintf("The inv plan is not valid in destructor\n"); + if (mxGPUIsValidGPUData(prhs[0]) ) + mxGPUDestroyGPUArray(inputArray); + // else + // mexPrintf("The inputArray is not valid in destructor\n"); return; } } diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index a94ac8dc..1456515b 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -389,8 +389,8 @@ tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); - tomo_origin_wrt_tilt_origin = [ reconGeometry.dX_specimen_to_tomo, ... + reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + tomo_origin_wrt_specimen_origin = [ reconGeometry.dX_specimen_to_tomo, ... reconGeometry.dY_specimen_to_tomo, ... reconGeometry.dZ_specimen_to_tomo]; tomo_origin_in_tomo_frame = emc_get_origin_index([reconGeometry.NX, ... @@ -408,21 +408,19 @@ continue; end - modelRot = BH_defineMatrix([0,90,0],'Bah','fwdVector'); - + modelRot = BH_defineMatrix([0,0,0],'Bah','fwdVector'); + for iSubTomo = 1:nSubTomos subtomo_rot_matrix = reshape(positionList(iSubTomo,17:25),3,3); subtomo_origin_in_tomo_frame = (positionList(iSubTomo,11:13)); - subtomo_origin_wrt_tilt_origin = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_tilt_origin; + subtomo_origin_wrt_specimen_origin = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_specimen_origin; % This extra shift came from experiments with real data but is both anny_starting and not understood. - subtomo_origin_wrt_tilt_origin = subtomo_origin_wrt_tilt_origin - emc.flgPreShift; + subtomo_origin_wrt_specimen_origin = subtomo_origin_wrt_specimen_origin - emc.flgPreShift; - % subTomo origin relative to reconLowerLeft - subtomo_origin_in_sample = originRec + subtomo_origin_wrt_tilt_origin; % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_tilt_origin' + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,3,2])', fidIDX); + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_specimen_origin' + [originRec(1),originRec(2),originRec(3)]'- emc.prjVectorShift([1,3,2])', fidIDX); nPrjsIncluded = 0; for iPrj = 1:nPrjs @@ -438,13 +436,13 @@ rTilt = BH_defineMatrix(TLT(iPrj_nat,4),'TILT','fwdVector') ; - prjCoords = rTilt*subtomo_origin_wrt_tilt_origin'; + prjCoords = rTilt*subtomo_origin_wrt_specimen_origin'; % I think this is for comparison with the values obtained from projecting using IMOD: FIXME fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, abs(TLT(iPrj_nat,15)) - prjCoords(3).*pixel_size.*10^-10); % Defocus value adjusted for Z coordinate in the tomogram. nm - d1 = (abs(TLT(iPrj_nat,15)) - subtomo_origin_wrt_tilt_origin(3).*pixel_size.*10^-10) * 10^9; + d1 = (abs(TLT(iPrj_nat,15)) - subtomo_origin_wrt_specimen_origin(3).*pixel_size.*10^-10) * 10^9; d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n', ... @@ -467,8 +465,10 @@ fclose(coordOUT); fclose(coordSTART); - - p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 -ImageForCoordinates %s ',... + % p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -scat -values 1 ',... + % '%s/%s.coord %s/%s.3dfid'], ... + % mbOUT{1:2},mbOUT{1:2}); + p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -scat -values 1 -ImageForCoordinates %s ',... '%s/%s.coord %s/%s.3dfid'], ... tilt_filepath, mbOUT{1:2},mbOUT{1:2}); system(p2m); @@ -506,12 +506,12 @@ iSave = 1; reModFileName = sprintf('%s/%s_%d_reMod.sh',mbOUT{1:2},iSave); reModFile = fopen(reModFileName,'w'); - invertTiltAngles = 0; fprintf(reModFile,['#!/bin/bash\n\n',... 'tilt -StandardInput << EOF\n',... 'input %s\n', ... 'output %s/%s.fid\n', ... 'COSINTERP 0\n', ... + 'RotateBy90 \n',... 'THICKNESS %d\n', ... 'TILTFILE %s/%s_align.rawtlt\n', ... 'DefocusFile %s/%s_align.defocus\n', ... @@ -530,7 +530,8 @@ mbOUT{1:2},... mbOUT{1:2},... mbOUT{1:2},... - lastLine1,lastLine2,... + lastLine1,... + lastLine2,... lastLine3); fclose(reModFile); @@ -714,7 +715,7 @@ % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','fwdVector'); % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); - rTilt = BH_defineMatrix([-wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','fwdVector'); + rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','fwdVector'); rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); end @@ -856,7 +857,7 @@ system(sprintf('chmod a=wrx %s_rec.sh',output_prefix)); system(sprintf('./%s_rec.sh',output_prefix)); - +error('NO REFINE') %%%%%%%%%%%%%%%%%%%%%%%%% % Refine %%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 2d024476..acbd6764 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -756,14 +756,13 @@ bh_global_ram_disk = '/dev/shm'; end - testFileName = sprintf('%s/thisEmCDiskCheck123456.txt',bh_global_ram_disk); - [writeError] = system(sprintf('echo a > %s',testFileName)); - if (writeError) + [status , fileAttributes] = fileattrib(bh_global_ram_disk); + if (status && fileAttributes.UserWrite) + fprintf('Found and using your ramDisk\n'); + else fprintf('\nRan into an error trying to write to the fastScatchDisk %s\n',bh_global_ram_disk); + fprintf('Please check that it exists and is writable status (%d) UserWrite (%d)\n', status, fileAttributes.UserWrite); bh_global_ram_disk = ''; - else - fprintf('Found and using your ramDisk\n'); - system(sprintf('rm %s',testFileName)); end diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 5ce9387e..fac4e5d6 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -78,7 +78,7 @@ else fi # NOTE: warnings are disabled to ensure that failed builds are caught. Ideally they would be addressed and removed. -${MATLAB_FOR_COMPILING} -nosplash -nodisplay -nojvm -r " ${mexCompile} mcc -w disable -w off -m ${mFile} -a fitInMap.py -a ../alignment/emC_autoAlign -a ../alignment/emC_findBeads -a ../metaData/BH_checkInstall -R -nodisplay -o "$(basename ${mFile} .m)_${binaryOutName}" ; exit" & +${MATLAB_FOR_COMPILING} -nosplash -nodisplay -nojvm -r " ${mexCompile} mcc -w disable -m ${mFile} -a fitInMap.py -a ../alignment/emC_autoAlign -a ../alignment/emC_findBeads -a ../metaData/BH_checkInstall -R -nodisplay -o "$(basename ${mFile} .m)_${binaryOutName}" ; exit" & wait diff --git a/transformations/emc_get_max_specimen_NZ.m b/transformations/emc_get_max_specimen_NZ.m index 6942f092..078e5c36 100644 --- a/transformations/emc_get_max_specimen_NZ.m +++ b/transformations/emc_get_max_specimen_NZ.m @@ -10,6 +10,9 @@ if isa(subTomoMeta_coords,'cell') nZ = subTomoMeta_coords{iTomo}.NZ; dZ = subTomoMeta_coords{iTomo}.dZ_specimen_to_tomo; + elseif isa(subTomoMeta_coords,'struct') + nZ = subTomoMeta_coords.(tomo_name_list{iTomo}).NZ; + dZ = subTomoMeta_coords.(tomo_name_list{iTomo}).dZ_specimen_to_tomo; else error('The field NZ or the field %s is not present in the subTomoMeta_coords', tomo_name_list{iTomo}); end From 2821a37a7bf48fb31b99bce5bdfea966f792f62a Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 5 Mar 2024 07:08:44 -0500 Subject: [PATCH 059/151] It seems there was no major flaw with correcting the x inversion in ctf 3d. Setting the default mtf to 1 (from 2) since the later maps seem to be oversharpened. This may be a contributor to overfitting. --- ctf/BH_ctf_Correct3d.m | 6 +++--- metaData/BH_parseParameterFile.m | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index c3d4fe35..6f437458 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -1064,7 +1064,7 @@ iProjection = BH_padZeros3d(maskedStack(:,:,TLT(iPrj,1)),padVal(1,:),padVal(2,:),'GPU','singleTaper'); iProjectionFT = fftn(iProjection).*iExposureFilter; clear iExposureFilter - correctedPrj = zeros([d1,d2],'single','gpuArray'); + correctedPrj = zeros([d1,d2],'single','gpuArray'); % Gridvectors for the specimen plane [rX,rY,~] = BH_multi_gridCoordinates([d1,d2],'Cartesian','GPU',{'none'},0,1,0); @@ -1139,8 +1139,8 @@ else tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz)),trimVal(1,:),trimVal(2,:),'GPU','single'); end - - tmpMask = (tZ > iDefocus - ctf3dDepth/2 & tZ <= iDefocus + ctf3dDepth/2); + + tmpMask = (tZ > iDefocus - ctf3dDepth/2 & tZ <= iDefocus + ctf3dDepth/2); linearIDX = unique(sub2ind([d1,d2],tX(tmpMask),tY(tmpMask))); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 2dd6a76c..528e03c7 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -574,7 +574,7 @@ if isfield(emc, 'mtf_value') EMC_assert_numeric(emc.mtf_value, 1); else - emc.mtf_value = 2.0; + emc.mtf_value = 1; end % Number of tilt processes to run in parallel in ctf 3d. From b0a7c19584923f2ea27fc8d6391cc2a4d091df2b Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 6 Mar 2024 08:04:03 -0500 Subject: [PATCH 060/151] center model in rotated frame --- ctf/BH_ctf_Correct3d.m | 6 ++-- masking/BH_mask3d.m | 9 ++++-- masking/EMC_maskReference.m | 11 +++---- metaData/BH_parseParameterFile.m | 11 +++++-- synthetic/BH_synthetic_mapBack.m | 50 +++++++++++++++----------------- synthetic/BH_to_cisTEM_mapBack.m | 7 +++-- testScripts/emClarity.m | 5 ++-- transformations/BH_average3d.m | 6 ++-- 8 files changed, 58 insertions(+), 47 deletions(-) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 6f437458..a175d4c8 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -638,7 +638,7 @@ n_slices_in_Y = tiltChunks(end) - tiltChunks(1) + 1; end - rCMD = sprintf(['tilt %s %s -MODE 12 -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... + rCMD = sprintf(['tilt %s %s -input %s -output %s.TMPPAD -TILTFILE %s -UseGPU %d ', ... '-WIDTH %d -COSINTERP 0 -THICKNESS %d -SHIFT %f,%f '],... super_sample, ... expand_lines, ... @@ -655,9 +655,9 @@ reconScaling = 1; % Explicitly set Radial to Nyquist if (flgLocal) - rCMD = [rCMD sprintf('-LOCALFILE %s -RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d', LOCAL, reconScaling)]; + rCMD = [rCMD sprintf('-LOCALFILE %s -RADIAL 0.5,.05 -MODE 12 -SCALE 0,%d', LOCAL, reconScaling)]; else - rCMD = [rCMD sprintf('-RADIAL 0.5,.05 -MODE 2 -SCALE 0,%d', reconScaling)]; + rCMD = [rCMD sprintf('-RADIAL 0.5,.05 -MODE 12 -SCALE 0,%d', reconScaling)]; end if isfile(sprintf('%s.sh',reconName)) diff --git a/masking/BH_mask3d.m b/masking/BH_mask3d.m index e9981d27..651ad775 100755 --- a/masking/BH_mask3d.m +++ b/masking/BH_mask3d.m @@ -34,6 +34,9 @@ flgCOM = 0; flg3d = 1; +global emc_debug_print; + + global bh_global_binary_mask_low_pass global bh_global_binary_mask_threshold global bh_global_vol_est_scaling @@ -428,8 +431,10 @@ maskVolume = sum(currentMask(:)>0); particleVolEstimate = particleVolEstimate ./ localParticleScaling; particleFraction = particleVolEstimate ./ maskVolume .* powerReduction; - fprintf('Estimated partVol, %d voxels\nmaskVol %d voxels\npwrReduction %2.3f\npartFract %2.3f\n',... - particleVolEstimate, maskVolume, powerReduction,particleFraction); + if (emc_debug_print) + fprintf('Estimated partVol, %d voxels\nmaskVol %d voxels\npwrReduction %2.3f\npartFract %2.3f\n',... + particleVolEstimate, maskVolume, powerReduction,particleFraction); + end % Should probably use varargout, but for now, returning the center of % mass is not done at the same stage as a particle volume estimate. diff --git a/masking/EMC_maskReference.m b/masking/EMC_maskReference.m index fe505557..07a7c5e5 100644 --- a/masking/EMC_maskReference.m +++ b/masking/EMC_maskReference.m @@ -83,10 +83,11 @@ % % Global verbosity. -global EMC_gp_verbose -if isempty(EMC_gp_verbose) - EMC_gp_verbose = true; -end +global emc_debug_print + +% if isempty(emc_debug_print) +% emc_debug_print = true; +% end %% MAIN [SIZE, OPTION, flg, tofloat] = checkIN(IMAGE, PIXEL, OPTION); @@ -199,7 +200,7 @@ % to take into account the power reduction due to the taper). FRACTION = maskVolume ./ (particleVolEstimate .* powerReduction); - if EMC_gp_verbose + if emc_debug_print fprintf(['FSC mask: Estimated particule volume : %d voxels\n', ... ' Estimated mask volume : %d voxels\n', ... ' Power reduction : %2.3f\n', ... diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 528e03c7..51caf7de 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -399,19 +399,19 @@ if isfield(emc, 'flgPostShift') EMC_assert_numeric(emc.flgPostShift, 2); else - emc.flgPostShift = 0.*[-1.5,1.0]; + emc.flgPostShift = [1.0,-1.0]; end if isfield(emc, 'prjVectorShift') EMC_assert_numeric(emc.prjVectorShift, 3); else - emc.prjVectorShift = 0.*[0.5,0.5,0.5]; + emc.prjVectorShift = [0.5,0.5,1.5]; end if isfield(emc,'pixelShift') EMC_assert_numeric(emc.pixelShift, 1); else - emc.pixelShift = 0.*-1; + emc.pixelShift = 0; end if isfield(emc, 'pixelMultiplier') @@ -603,3 +603,8 @@ emc.super_sample = 3; end +if isfield(emc, 'debug_print') + EMC_assert_boolean(emc.debug_print); +else + emc.debug_print = false; +end diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index acf2295d..5e4880ce 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -520,7 +520,6 @@ fclose(iXF); positionList = geometry.(tomoList{iTomo}); - tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; positionList = positionList(positionList(:,26) ~= -9999,:); nFidsTotal = nFidsTotal + size(positionList,1); @@ -586,7 +585,7 @@ % We need to rotate the model 90 degrees around X to match the "natural" reconstruction reference frame of imod % that is [x,z,-y] - modelRot = BH_defineMatrix([0,90,0],'Bah','fwdVector'); + modelRot = BH_defineMatrix([0,0,0],'Bah','fwdVector'); for iSubTomo = 1:nSubTomos @@ -685,8 +684,8 @@ % Reproject using tilt, so just save the 3d coords. fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', (modelRot * subtomo_origin_wrt_tilt_origin')./samplingRate + ... - [binned_specimen_origin_in_specimen_frame(1),binned_specimen_origin_in_specimen_frame(3),binned_specimen_origin_in_specimen_frame(2)]' - ... - emc.prjVectorShift([1,3,2])', ... + [binned_specimen_origin_in_specimen_frame(1),binned_specimen_origin_in_specimen_frame(2),binned_specimen_origin_in_specimen_frame(3)]' - ... + emc.prjVectorShift([1,2,3])', ... fidIDX); % Save a non-rotated model with each class on its own object for visualization @@ -759,13 +758,13 @@ system(p2m); for iSave = 1 - SAVE_IMG(MRCImage(gather(avgTomo{iSave})),sprintf('%smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave),pixel_size); + SAVE_IMG(avgTomo{iSave},{sprintf('%smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave), 'half'},pixel_size); avgTomo{iSave} = []; end clear avgTomo if (emc.save_mapback_classes || flgClassAvg) - SAVE_IMG(MRCImage(gather(avgColor)),sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),pixel_size); + SAVE_IMG(avgColor,{sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),'half'},pixel_size); clear avgColor end @@ -787,11 +786,12 @@ system(rotCMD); - system(sprintf('rm %smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave)); + % system(sprintf('rm %smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave)); end clear avgTomo{1} wgt end + % % % % It may be faster to work with a rotated vol since the reading in may cause % % % % problems, but the projection is so slow, that this isn't worth dealing with % % % % now. @@ -823,10 +823,8 @@ if (localFile) lastLine1 = sprintf('LOCALFILE %s', localFile) % Used if GPU fails - cpuLastLine = lastLine1; else lastLine1 = ''; - cpuLastLine = ''; end if strcmpi('GPU', 'GPU') @@ -905,8 +903,7 @@ end end - % Special case, initialize the full sized volume and the - % header but don't actually reproject anything. + fprintf('Reprojecting volume %d/%d with size %d\n',... iChunk,nChunks,mapBackRePrjSize); rePrjFileName = sprintf('%smapBack%d/%s_%d_rePrj.sh',mbOUT{1:3},iSave); @@ -921,16 +918,16 @@ 'THICKNESS %d\n', ... 'TILTFILE %smapBack%d/%s_align.rawtlt\n', ... 'REPROJECT %s\n', ... - 'RecFileToReproject %smapBack%d/%s.tmpRot%d\n',... - 'TOTALSLICES %d,%d\n',... + 'RecFileToReproject %smapBack%d/%s.tmpRot%d\n',... + 'TOTALSLICES %d,%d\n',... 'ZMinAndMaxReproj %d,%d\n',... '%s\n', ... '%s\n', ... '%s\n',... 'EOF'],tilt_binned_filename ,outputStackName, maxZ, ... mbOUT{1:3},... - taStr, mbOUT{1:3},iSave,... - 0,sTY-1,... + taStr, mbOUT{1:3},iSave,... + 0,sTY-1,... inc(iChunk),inc(iChunk+1),... lastLine1,lastLine2,... lastLine3); @@ -942,7 +939,7 @@ [failedToRun,~] = system(sprintf('%s',rePrjFileName)); - + if (failedToRun) % Reduce size switch mapBackRePrjSize @@ -979,10 +976,10 @@ system(sprintf('mv %s %s.tmp',rePrjFileName,rePrjFileName)); system(sprintf('cp %s.tmp %s',rePrjFileName,rePrjFileName)); system(sprintf('rm %s.tmp %s',rePrjFileName,rePrjFileName)); - % system(sprintf('%s',rePrjFileName)); + system(sprintf('%s',rePrjFileName)); - % Break out to next iter of while loop, recalculating the - % chunk size at reduced depth. + Break out to next iter of while loop, recalculating the + chunk size at reduced depth. break @@ -1005,6 +1002,7 @@ 'input %s\n', ... 'output %smapBack%d/%s.fid\n', ... 'COSINTERP 0\n', ... + 'RotateBy90\n',... 'THICKNESS %d\n', ... 'TILTFILE %smapBack%d/%s_align.rawtlt \n', ... 'DefocusFile %smapBack%d/%s_align.defocus \n', ... @@ -1047,10 +1045,10 @@ end - for iSave = 1 - % Remove the full size tomo - system(sprintf('rm %smapBack%d/%s.tmpRot%d',mbOUT{1:3},iSave)); - end + % for iSave = 1 FIXME revert + % % Remove the full size tomo + % system(sprintf('rm %smapBack%d/%s.tmpRot%d',mbOUT{1:3},iSave)); + % end fidList = load(sprintf('%smapBack%d/%s.coordPrj',mbOUT{1:3})); parList = load(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3})); @@ -1068,7 +1066,7 @@ % Give every instance of each fiducial a unique identifier. fidList = [1:size(fidList,1);fidList']'; - + error('check') % for center of mass COM = 3; [bx,by] = ndgrid(-COM:COM,-COM:COM); @@ -1174,8 +1172,8 @@ end % FIXME revert - for iPrj = 1:nPrjs - % parfor iPrj = 1:nPrjs + % for iPrj = 1:nPrjs + parfor iPrj = 1:nPrjs % % For some reason if these mrc objects are created before the parfor % loop begins, they fail to load. It is fine as a regular for loop diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 1456515b..c31ff0ee 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -148,7 +148,8 @@ output_cell = {}; newstack_file = sprintf('%s/temp_particle_stack.newstack',mbOUT{1}); newstack_file_handle = fopen(newstack_file,'w'); - +skip_to_end = false; +if ~(skip_to_end) for iTiltSeries = tiltStart:nTiltSeries n_particles_added_to_stack = 0; @@ -567,7 +568,7 @@ mbOUT{1:2}, mbOUT{1:2})) end - + error('test') try fidList = load(sprintf('%s/%s.coordPrj',mbOUT{1:2})); catch @@ -802,6 +803,7 @@ system(sprintf('cat %s >> %s', newstack_file, newstack_file_with_n_stacks)); system(sprintf('newstack -FileOfInputs %s %s.mrc > /dev/null', newstack_file_with_n_stacks, output_prefix)); +end % skip to here % SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',output_prefix),pixelSize); maxThreads = emc.('nCpuCores'); @@ -857,7 +859,6 @@ system(sprintf('chmod a=wrx %s_rec.sh',output_prefix)); system(sprintf('./%s_rec.sh',output_prefix)); -error('NO REFINE') %%%%%%%%%%%%%%%%%%%%%%%%% % Refine %%%%%%%%%%%%%%%%%%%%%%%%%%%% diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index acbd6764..e2402d8f 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -748,8 +748,9 @@ bh_global_fast_scratch_disk=''; end - - + global emc_debug_print + emc_debug_print = emc.('debug_print'); + try bh_global_ram_disk = emc.('ramDisk'); catch diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index ca38aaaf..2b5c5c89 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -919,8 +919,8 @@ else iParticle = gpuArray(OPEN_IMG('single', volumeData,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); end @@ -937,7 +937,7 @@ particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - positionList(iSubTomo,[11:13]+26*(iPeak-1)) = shiftVAL+CUTPADDING+ceil((sizeWindow+1)./2); + positionList(iSubTomo,[11:13]+26*(iPeak-1)) = shiftVAL+CUTPADDING+emc_get_origin_index(sizeWindow); if (emc.projectVolumes) SAVE_IMG(sum(iParticle,3),particleOUT_name,emc.pixel_size_angstroms); else From 50907670e482ff07d3089e236071a86952094ea8 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 6 Mar 2024 08:17:14 -0500 Subject: [PATCH 061/151] also centered in rotated view --- synthetic/BH_synthetic_mapBack.m | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 5e4880ce..c088b800 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -585,7 +585,7 @@ % We need to rotate the model 90 degrees around X to match the "natural" reconstruction reference frame of imod % that is [x,z,-y] - modelRot = BH_defineMatrix([0,0,0],'Bah','fwdVector'); + modelRot = BH_defineMatrix([0,90,0],'Bah','fwdVector'); for iSubTomo = 1:nSubTomos @@ -684,7 +684,7 @@ % Reproject using tilt, so just save the 3d coords. fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', (modelRot * subtomo_origin_wrt_tilt_origin')./samplingRate + ... - [binned_specimen_origin_in_specimen_frame(1),binned_specimen_origin_in_specimen_frame(2),binned_specimen_origin_in_specimen_frame(3)]' - ... + [binned_specimen_origin_in_specimen_frame(1),binned_specimen_origin_in_specimen_frame(3),binned_specimen_origin_in_specimen_frame(2)]' - ... emc.prjVectorShift([1,2,3])', ... fidIDX); From f53c42c8e9134d4334f1958dff56a658bb9a6936 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 6 Mar 2024 08:36:57 -0500 Subject: [PATCH 062/151] prj now matches (test case) but there was some opaque failure to reproject some slices. Reducing the slab to 256 seemed to work. This would be a good target to parallelize as in ctf3d. --- synthetic/BH_synthetic_mapBack.m | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index c088b800..166e16f7 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -241,7 +241,7 @@ end - mapBackRePrjSize = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize'); + mapBackRePrjSize = min(256,subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize')); % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos if nTomograms == 0 @@ -901,7 +901,7 @@ if (failedToRun) error('failed to initialize reporojection %s\n',outputStackName); end - end + end % if iChunk == 1, initialize the projection fprintf('Reprojecting volume %d/%d with size %d\n',... @@ -928,10 +928,11 @@ mbOUT{1:3},... taStr, mbOUT{1:3},iSave,... 0,sTY-1,... - inc(iChunk),inc(iChunk+1),... + inc(iChunk),inc(iChunk+1)-1,... lastLine1,lastLine2,... lastLine3); - + inc(iChunk) + inc(iChunk+1)-1 fclose(reProjFile); system(sprintf('chmod a=wrx %s',rePrjFileName)); @@ -939,7 +940,6 @@ [failedToRun,~] = system(sprintf('%s',rePrjFileName)); - if (failedToRun) % Reduce size switch mapBackRePrjSize @@ -1002,7 +1002,6 @@ 'input %s\n', ... 'output %smapBack%d/%s.fid\n', ... 'COSINTERP 0\n', ... - 'RotateBy90\n',... 'THICKNESS %d\n', ... 'TILTFILE %smapBack%d/%s_align.rawtlt \n', ... 'DefocusFile %smapBack%d/%s_align.defocus \n', ... @@ -1065,8 +1064,7 @@ % Give every instance of each fiducial a unique identifier. fidList = [1:size(fidList,1);fidList']'; - - error('check') + % for center of mass COM = 3; [bx,by] = ndgrid(-COM:COM,-COM:COM); @@ -1785,7 +1783,7 @@ %%%system(sprintf('grep -A %d " At minimum tilt" ./mapBack/%s_ta.log > tmp.log',nPrjs+2,TN)); %%%system(sprintf('awk ''{if(NR >3) print $5}'' tmp.log > mapBack/%s.mag',TN)); %%%end %uf cibdutuib - + error('early exit') end % loop over tilts From 3d766a188039d6297c7de9d8375bf36ba7d7770b Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 6 Mar 2024 10:11:08 -0500 Subject: [PATCH 063/151] Scaling looks right, but the shifts are a bit massive --- synthetic/BH_synthetic_mapBack.m | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 166e16f7..974f81fe 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -530,6 +530,7 @@ tiltHeader = getHeader(MRCImage(tiltName,0)); + % FIXME, this isn't necessarily going to be the correct size fullTiltSizeXandY = [tiltHeader.nX,tiltHeader.nY].*samplingRate; @@ -858,7 +859,11 @@ nChunks = length(inc)-1; for iChunk = 1:nChunks - + if iChunk == nChunks + extend_by = 1; + else + extend_by = 0; + end if iChunk == 1 if exist(outputStackName,'file') fprintf('removing %s\n',outputStackName); @@ -928,11 +933,10 @@ mbOUT{1:3},... taStr, mbOUT{1:3},iSave,... 0,sTY-1,... - inc(iChunk),inc(iChunk+1)-1,... + inc(iChunk),inc(iChunk+1)-1+extend_by,... lastLine1,lastLine2,... lastLine3); - inc(iChunk) - inc(iChunk+1)-1 + fclose(reProjFile); system(sprintf('chmod a=wrx %s',rePrjFileName)); @@ -1623,7 +1627,9 @@ % The model ends up seeing the pixel size as 1, so even though it loads % properly on the full aligned stack, these coords need to be scaled by % the pixel size since this is the input to tiltalign. - fFull(:,2:3) = fFull(:,2:3).*pixel_size; + % fFull(:,2:3) = fFull(:,2:3).*pixel_size; + + fFull(:,2:3) = (samplingRate.*(fFull(:,2:3) - emc_get_origin_index(fullTiltSizeXandY./samplingRate)) + emc_get_origin_index(fullTiltSizeXandY)); fprintf(fidCombine,'%d %4.4f %4.4f %d\n',fCombine'); fclose(fidCombine); @@ -1647,6 +1653,8 @@ final_line1 = 'ShiftZFromOriginal'; final_line2 = 'AxisZShift 0.0'; final_line3 = 'LocalOutputOptions 1,1,1'; + + else final_line3 = ''; final_line2 = ''; @@ -1661,6 +1669,12 @@ tilt_script_name = sprintf('cache/mapBack%d/%s%s',mbOUT{2},tn2,tn3); end + % TODO: It looks like the output model file (3dmod) is the solved positions, + % but is saved at a pixel size of 1. Scaling by the sampling rate in all dimensions + % and then adding the origin (only for Z) places the coordinates back into the bin6 model + % I think we could get shifts from this + % TODO: could use ImageOriginXandY to accound for a diffence in origin due to binning + % as ImageSizeXandY given as binned size*sampling rate, which may not equal full size fprintf(aliCom,['#!/bin/bash\n\n',... '#iTiltSeries %d\n',... 'tiltalign -StandardInput << EOF\n',... @@ -1783,7 +1797,6 @@ %%%system(sprintf('grep -A %d " At minimum tilt" ./mapBack/%s_ta.log > tmp.log',nPrjs+2,TN)); %%%system(sprintf('awk ''{if(NR >3) print $5}'' tmp.log > mapBack/%s.mag',TN)); %%%end %uf cibdutuib - error('early exit') end % loop over tilts From 23c956b860f9500883e04a379bf76cb7fbc89d52 Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 7 Mar 2024 08:18:42 -0500 Subject: [PATCH 064/151] Stepping through XCF calc in tomoCPR and there is a clear bug I think I've noticed before with the fourier transformer class. As saved, I get a normal ccf at line 1485, but if i take the conjh in line 1483, I get bullshit. --- metaData/BH_parseParameterFile.m | 2 +- synthetic/BH_synthetic_mapBack.m | 132 +++++++++++++++++++------------ 2 files changed, 84 insertions(+), 50 deletions(-) diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 51caf7de..c342c3a7 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -399,7 +399,7 @@ if isfield(emc, 'flgPostShift') EMC_assert_numeric(emc.flgPostShift, 2); else - emc.flgPostShift = [1.0,-1.0]; + emc.flgPostShift = 0.*[1.0,-1.0]; end if isfield(emc, 'prjVectorShift') diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 974f81fe..fb380049 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -102,14 +102,14 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% try - use_PCF = emc.('use_PCF'); + use_PCF = emc.('use_PCF') catch - use_PCF = 0; + use_PCF = 0 end -if (use_PCF) - error('The PCF scaling is not working correctly, please set use_PCF=0'); -end +% if (use_PCF) +% error('The PCF scaling is not working correctly, please set use_PCF=0'); +% end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -1083,13 +1083,13 @@ CTFSIZE = BH_multi_iterator([2.*tileSize,1], 'fourier'); CTFSIZE = CTFSIZE(1:2); - ctfOrigin = floor(CTFSIZE./2) + 1; + ctfOrigin = emc_get_origin_index(CTFSIZE); padCTF = BH_multi_padVal(tileSize,CTFSIZE); ctfMask = BH_mask3d('sphere',CTFSIZE,ctfOrigin-7,[0,0],'2d'); if (eraseMask) - peakMask = BH_mask3d(eraseMaskType,CTFSIZE,eraseMaskRadius,[0,0],'2d'); + peakMask = BH_mask3d(eraseMaskType,CTFSIZE,3.*eraseMaskRadius,[0,0],'2d'); else peakMask = BH_mask3d('sphere',CTFSIZE,peak_search_radius,[0,0],'2d'); end @@ -1109,7 +1109,9 @@ % Any large shifts should be obvious in the original alignment, so only % look around +/- this value - globalPeak = max(2,ceil(10/pixel_size)); + % globalPeak = max(2,ceil(10/pixel_size)); + % globalPeak = globalPeak + mod(globalPeak,2); + globalPeak = floor(max(sTX,sTY).*0.25); globalPeak = globalPeak + mod(globalPeak,2); globalPeakMask = zeros([sTX,sTY,1],'single'); @@ -1174,8 +1176,8 @@ end % FIXME revert - % for iPrj = 1:nPrjs - parfor iPrj = 1:nPrjs + for iPrj = 1:nPrjs + % parfor iPrj = 1:nPrjs % % For some reason if these mrc objects are created before the parfor % loop begins, they fail to load. It is fine as a regular for loop @@ -1317,11 +1319,14 @@ - cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).* abs(HqzUnMod).*... - conj(fftn(refPrj).*Hqz)))); + cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).*conj(fftn(refPrj).*Hqz)))); % cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).*... - % conj(fftn(refPrj).*Hqz)))); - + % + % conj(fftn(refPrj).*Hqz)))); + if (use_PCF) + cccPrj = cccPrj .* cccPrj ./ (abs(cccPrj) + 0.1); + end + cccPrj = (cccPrj-min(cccPrj(:))) .* globalPeakMask; [~,maxPRJ] = max(cccPrj(:)); [mRx, mRy] = ind2sub(size(cccPrj), maxPRJ); @@ -1339,16 +1344,22 @@ comPRJX = sum(sum(bx.*cccPRJ))./sum(cccPRJ(:)); comPRJY = sum(sum(by.*cccPRJ))./sum(cccPRJ(:)); + - - - estPeak = [mRx, mRy] - binned_specimen_origin_in_specimen_frame(1:2) + [comPRJX, comPRJY]; + % max index - origin + fraxtional COM + estimated_global_offset = [mRx, mRy] - binned_specimen_origin_in_specimen_frame(1:2) + [comPRJX, comPRJY]; glbList = fopen(sprintf('%smapBack%d/%s_%03d.global',mbOUT{1:3},iPrj),'w'); % Add unique indicies to prevent ambiquity when comparing with paral - fprintf(glbList,'%f degree tilt at %f %f\n', TLT(iPrj,4),estPeak); - dataPrj = BH_resample2d(dataPrj,[0,0,0],[estPeak,0],'Bah','GPU','inv',1,size(dataPrj)); + fprintf(glbList,'%f degree tilt at %f %f\n', TLT(iPrj,4),estimated_global_offset); + dataPrj = BH_resample2d(dataPrj,[0,0,0],[estimated_global_offset,0],'Bah','GPU','inv',1,size(dataPrj)); %mapBack%d, imshow3D(gather(fftshift(real(ifftn(fftn(dataPrj).*conj(fftn(refPrj))))))); + cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).* abs(HqzUnMod).*... + conj(fftn(refPrj).*Hqz)))); + % cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).*... + % + % conj(fftn(refPrj).*Hqz)))); + % figure, imshow3D(gather(cccPrj)); @@ -1371,7 +1382,7 @@ coordOUT = fopen(sprintf('%smapBack%d/%s_%03d.coordFIT',mbOUT{1:3},iPrj),'w'); - + tmpOut = zeros([CTFSIZE,size(wrkFid,1)],'single','gpuArray'); for iFid = 1:size(wrkFid,1) @@ -1380,14 +1391,19 @@ continue end - pixelX = wrkFid(iFid,3) - emc.pixelShift + emc.flgPostShift(1); - pixelY = wrkFid(iFid,4) - emc.pixelShift + emc.flgPostShift(2); + % pixelX = wrkFid(iFid,3) - emc.pixelShift + emc.flgPostShift(1); + % pixelY = wrkFid(iFid,4) - emc.pixelShift + emc.flgPostShift(2); + pixelX = wrkFid(iFid,3); + pixelY = wrkFid(iFid,4); - ox = floor(pixelX) - tileRadius; - oy = floor(pixelY) - tileRadius; + ox = floor(pixelX+0.5) - tileRadius; + oy = floor(pixelY+0.5) - tileRadius; + + sx = (pixelX - ox); + sy = (pixelY - oy); - sx = emc.pixelMultiplier*(pixelX - floor(pixelX)); - sy = emc.pixelMultiplier*(pixelY - floor(pixelY)); + % sx = emc.pixelMultiplier*(pixelX - floor(pixelX)); + % sy = emc.pixelMultiplier*(pixelY - floor(pixelY)); % ox = floor(wrkFid(iFid,3)) - tileRadius; % oy = floor(wrkFid(iFid,4)) - tileRadius; @@ -1426,6 +1442,8 @@ dataTile = dataTile./rms(dataTile(:)); refTile = refTile ./ rms(refTile(:)); + + dataTile = ctfMask.*BH_padZeros3d(dataTile,'fwd',padCTF, ... 'GPU','singleTaper'); @@ -1451,22 +1469,30 @@ bestScore = -inf; bestCTF = 1; for deltaCTF = 1:nDefTotal - + SAVE_IMG(real(bhF.invFFT(conj(refFT))), 'refFT.mrc'); + SAVE_IMG(real(mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... + single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... + single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18)))), 'iCTF.mrc'); iRefCTF = refFT .* ... mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18))); - + SAVE_IMG(real(bhF.invFFT(conj(iRefCTF))), 'iRefCTF.mrc'); % try iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')); - + SAVE_IMG(real(bhF.invFFT(iRefCTF)), 'iRefCTF2.mrc'); cccMap = dataFT .* iRefCTF; + SAVE_IMG(real(bhF.invFFT(cccMap)), 'cccMap.mrc') if (use_PCF) cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); end + SAVE_IMG(real(bhF.invFFT(cccMap)), 'cccMap2.mrc'); + error('sdf') cccMap = peakMask.*real(bhF.invFFT(cccMap)); - + tmpOut(:,:,iFid) = cccMap; + dXY = [1,1]; + continue; [maxVal,maxMap] = max(cccMap(:)); defocusCCC{iPrj}(deltaCTF,iFid) = maxVal; @@ -1494,11 +1520,11 @@ % peak in Map is where query is relative to ref, dXY then is the shift % needed to move the predicted position to the measured. - % Data moved from a position of estPeak, so add this to dXY + % Data moved from a position of estimated_global_offset, so add this to dXY - dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estPeak - [sx,sy]; + dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estimated_global_offset + [sx,sy]; catch - dXY = estPeak - [sx,sy]; % TODO double check me + dXY = estimated_global_offset + [sx,sy]; % TODO double check me end end @@ -1544,13 +1570,14 @@ % peak in Map is where query is relative to ref, dXY then is the shift % needed to move the predicted position to the measured. - % Data moved from a position of estPeak, so add this to dXY + % Data moved from a position of estimated_global_offset, so add this to dXY - dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estPeak - [sx,sy]; + dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estimated_global_offset + [sx,sy]; end fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), dXY, wrkFid(iFid,5)); end % end of loop over fiducials - + SAVE_IMG(tmpOut,'tmpOut.mrc',pixel_size); + error('tmpOut'); if (calcCTF) [~,imDefC] = max(defocusCCC{iPrj},[],1); expectedDefocus = mean(defShiftVect(imDefC)); @@ -1560,7 +1587,7 @@ evalMaskCell{iPrj} = uint8(evalMask); evalMask = []; fclose(coordOUT); end % end of the parfor loop - + if ( calcCTF ) save(sprintf('%smapBack%d/%s%s.defShiftsMat',mbOUT{1:3},outCTF),'defocusShifts'); @@ -1629,7 +1656,7 @@ % the pixel size since this is the input to tiltalign. % fFull(:,2:3) = fFull(:,2:3).*pixel_size; - fFull(:,2:3) = (samplingRate.*(fFull(:,2:3) - emc_get_origin_index(fullTiltSizeXandY./samplingRate)) + emc_get_origin_index(fullTiltSizeXandY)); + % fFull(:,2:3) = (samplingRate.*(fFull(:,2:3) - emc_get_origin_index(fullTiltSizeXandY./samplingRate)) + emc_get_origin_index(fullTiltSizeXandY)); fprintf(fidCombine,'%d %4.4f %4.4f %d\n',fCombine'); fclose(fidCombine); @@ -1638,11 +1665,11 @@ fclose(fidFull); % convert to model system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... - '%smapBack%d/%s.coordCombine %smapBack%d/%s_fit-comb.fid'],mbOUT{1:3},mbOUT{1:3})); - system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... - '%smapBack%d/%s.coordFull %smapBack%d/%s_fit-full.fid'],mbOUT{1:3},mbOUT{1:3})); - system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... - '%smapBack%d/%s.coordBin%d %smapBack%d/%s_fit-bin%d.fid'],mbOUT{1:3},samplingRate,mbOUT{1:3},samplingRate)); + '%smapBack%d/%s.coordCombine %smapBack%d/%s_fit-comb.fid -ImageForCoordinates %smapBack%d/%s_1_mapBack.st '],mbOUT{1:3},mbOUT{1:3},mbOUT{1:3})); + system(sprintf(['point2model -zero -circle 3 -color 0,0,255 -ImageForCoordinates %smapBack%d/%s_1_mapBack.st ',... + '%smapBack%d/%s.coordFull %smapBack%d/%s_fit-full.fid'],mbOUT{1:3},mbOUT{1:3},mbOUT{1:3})); + system(sprintf(['point2model -zero -circle 3 -color 0,0,255 -ImageForCoordinates %smapBack%d/%s_1_mapBack.st ',... + '%smapBack%d/%s.coordBin%d %smapBack%d/%s_fit-bin%d.fid'],mbOUT{1:3},samplingRate,mbOUT{1:3},samplingRate,mbOUT{1:3})); % write the com script for running tiltalign TN =tiltBaseName RotDef = 5; @@ -1675,13 +1702,15 @@ % I think we could get shifts from this % TODO: could use ImageOriginXandY to accound for a diffence in origin due to binning % as ImageSizeXandY given as binned size*sampling rate, which may not equal full size + % fullTiltSizeXandY,... + % unsampled_pixel_size,unsampled_pixel_size,... + % 'ImageSizeXandY %d,%d\n',... + % 'ImagePixelSizeXandY %f,%f\n',... fprintf(aliCom,['#!/bin/bash\n\n',... '#iTiltSeries %d\n',... 'tiltalign -StandardInput << EOF\n',... 'ModelFile %smapBack%d/%s_fit-full.fid\n',... - 'ImageSizeXandY %d,%d\n',... - 'ImagePixelSizeXandY %f,%f\n',... - 'ImagesAreBinned 1\n',... + 'ImagesAreBinned %d\n',... 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... 'OutputResidualFile %smapBack%d/%s%s.resid\n',... 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... @@ -1718,9 +1747,13 @@ '%s\n',... '%s\n',... '%s\n',... - 'EOF'],iTiltSeries,mbOutAlt{1:3},fullTiltSizeXandY,... - unsampled_pixel_size,unsampled_pixel_size,... - mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... + 'EOF'],... + iTiltSeries,... + mbOutAlt{1:3},... % for ModelFile + samplingRate, ... + mbOutAlt{1:3},... + outCTF,... + mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... tilt_script_name,n_surfaces, ... emc.rot_option_global, ... @@ -1797,6 +1830,7 @@ %%%system(sprintf('grep -A %d " At minimum tilt" ./mapBack/%s_ta.log > tmp.log',nPrjs+2,TN)); %%%system(sprintf('awk ''{if(NR >3) print $5}'' tmp.log > mapBack/%s.mag',TN)); %%%end %uf cibdutuib + error('early exit') end % loop over tilts From 431aede363b4eea85ef6a4ed358e9a664e16e5cf Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 7 Mar 2024 12:44:44 -0500 Subject: [PATCH 065/151] wip --- mexFiles/mexFFT.cu | 25 ++++----- synthetic/BH_synthetic_mapBack.m | 93 +++++++++++++++----------------- testScripts/fourierTransformer.m | 17 ++++-- 3 files changed, 68 insertions(+), 67 deletions(-) diff --git a/mexFiles/mexFFT.cu b/mexFiles/mexFFT.cu index 662ada7e..11c25391 100644 --- a/mexFiles/mexFFT.cu +++ b/mexFiles/mexFFT.cu @@ -85,12 +85,13 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) if (numel_input == 1) { + size_t dummy; // mexPrintf("Destroying the plans\n"); - if ( mxGPUIsValidGPUData(prhs[2])) + if (cufftGetSize(*plan, &dummy) != CUFFT_INVALID_PLAN) cufftDestroy(*plan); // else // mexPrintf("The fwd plan is not valid in destructor\n"); - if ( mxGPUIsValidGPUData(prhs[3])) + if (cufftGetSize(*planInv, &dummy) != CUFFT_INVALID_PLAN) cufftDestroy(*planInv); // else // mexPrintf("The inv plan is not valid in destructor\n"); @@ -146,9 +147,9 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) - int xFormRank; + // int xFormRank; int fft_dims[input_dims]; - int batchSize; + // int batchSize; if (input_dims > 2) { fft_dims[2] = (int) input_size[0]; @@ -161,14 +162,14 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) fft_dims[0] = (int) input_size[1]; } - if (input_dims > 2) - { - if (fft_dims[2] > 1) { xFormRank = 3; batchSize = fft_dims[2]; } else { xFormRank = 2; batchSize = fft_dims[1]; } - } - else - { - if (fft_dims[1] > 1) { xFormRank = 2; batchSize = fft_dims[1]; } else { xFormRank = 1; batchSize = 1;} - } + // if (input_dims > 2) + // { + // if (fft_dims[2] > 1) { xFormRank = 3; batchSize = fft_dims[2]; } else { xFormRank = 2; batchSize = fft_dims[1]; } + // } + // else + // { + // if (fft_dims[1] > 1) { xFormRank = 2; batchSize = fft_dims[1]; } else { xFormRank = 1; batchSize = 1;} + // } outputArray = mxGPUCreateGPUArray(input_dims, output_size, diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index fb380049..331d3dab 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -91,7 +91,7 @@ nFiducialsPerPatch = ceil(100./sqrt(molMass)); end - +nFiducialsPerPatch = min(nFiducialsPerPatch, 32); % Used to calc defocus values using tilt instead of manually. Convention % diff. flgInvertTiltAngles = 0; @@ -102,14 +102,14 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% try - use_PCF = emc.('use_PCF') + use_PCF = emc.('use_PCF'); catch - use_PCF = 0 + use_PCF = 0; end -% if (use_PCF) -% error('The PCF scaling is not working correctly, please set use_PCF=0'); -% end +if (use_PCF) + error('The PCF scaling is not working correctly, please set use_PCF=0'); +end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -336,7 +336,7 @@ end % TODO: these defaults should be re-examined - targetPatchSize = max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch))); + targetPatchSize = ceil(max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch)))); % The binned stacks should already exist, if not, this will re-create it in the cache dir. % Note that this will also be checked when reconstructing the full 3d background tomo. @@ -1048,10 +1048,10 @@ end - % for iSave = 1 FIXME revert - % % Remove the full size tomo - % system(sprintf('rm %smapBack%d/%s.tmpRot%d',mbOUT{1:3},iSave)); - % end + for iSave = 1 + % Remove the full size tomo + system(sprintf('rm %smapBack%d/%s.tmpRot%d',mbOUT{1:3},iSave)); + end fidList = load(sprintf('%smapBack%d/%s.coordPrj',mbOUT{1:3})); parList = load(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3})); @@ -1076,7 +1076,7 @@ % add optional half radius for edge case and make the padding more % logical, twice the particle radius, and then CTF size using mulit_iter % with an optimization step - particlePad = 1.5; + particlePad = 2.5; tileRadius = floor(particlePad.*particle_radius); tileSize = (2.*tileRadius + 1).*[1,1]; @@ -1088,13 +1088,16 @@ padCTF = BH_multi_padVal(tileSize,CTFSIZE); ctfMask = BH_mask3d('sphere',CTFSIZE,ctfOrigin-7,[0,0],'2d'); - if (eraseMask) - peakMask = BH_mask3d(eraseMaskType,CTFSIZE,3.*eraseMaskRadius,[0,0],'2d'); - else - peakMask = BH_mask3d('sphere',CTFSIZE,peak_search_radius,[0,0],'2d'); - end + % if (eraseMask) + % peakMask = BH_mask3d(eraseMaskType,CTFSIZE,3.*eraseMaskRadius,[0,0],'2d'); + % else + % peakMask = BH_mask3d('sphere',CTFSIZE,peak_search_radius,[0,0],'2d'); + % end - peakMask(peakMask < 0.99) = 0; + % peakMask(peakMask < 0.99) = 0; + + peakMask = EMC_gaussianKernel(CTFSIZE,tileRadius./2,'gpu',{}); + peakMask = peakMask ./ max(peakMask(:)); @@ -1175,9 +1178,9 @@ emc.k_factor_scaling = 10 / sqrt(nFidsTotal); end - % FIXME revert - for iPrj = 1:nPrjs - % parfor iPrj = 1:nPrjs + + % for iPrj = 1:nPrjs + parfor iPrj = 1:nPrjs % % For some reason if these mrc objects are created before the parfor % loop begins, they fail to load. It is fine as a regular for loop @@ -1399,8 +1402,8 @@ ox = floor(pixelX+0.5) - tileRadius; oy = floor(pixelY+0.5) - tileRadius; - sx = (pixelX - ox); - sy = (pixelY - oy); + sx = (pixelX - floor(pixelX+0.5)); + sy = (pixelY - floor(pixelY+0.5)); % sx = emc.pixelMultiplier*(pixelX - floor(pixelX)); % sy = emc.pixelMultiplier*(pixelY - floor(pixelY)); @@ -1469,30 +1472,23 @@ bestScore = -inf; bestCTF = 1; for deltaCTF = 1:nDefTotal - SAVE_IMG(real(bhF.invFFT(conj(refFT))), 'refFT.mrc'); - SAVE_IMG(real(mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... - single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... - single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18)))), 'iCTF.mrc'); iRefCTF = refFT .* ... mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18))); - SAVE_IMG(real(bhF.invFFT(conj(iRefCTF))), 'iRefCTF.mrc'); % try iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')); - SAVE_IMG(real(bhF.invFFT(iRefCTF)), 'iRefCTF2.mrc'); cccMap = dataFT .* iRefCTF; - SAVE_IMG(real(bhF.invFFT(cccMap)), 'cccMap.mrc') + + if (use_PCF) - cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); + cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.001); end - SAVE_IMG(real(bhF.invFFT(cccMap)), 'cccMap2.mrc'); - error('sdf') + cccMap = peakMask.*real(bhF.invFFT(cccMap)); - + tmpOut(:,:,iFid) = cccMap; - dXY = [1,1]; - continue; + [maxVal,maxMap] = max(cccMap(:)); defocusCCC{iPrj}(deltaCTF,iFid) = maxVal; @@ -1517,6 +1513,7 @@ comMapX = sum(sum(bx.*cccMap))./sum(cccMap(:)); comMapY = sum(sum(by.*cccMap))./sum(cccMap(:)); + % peak in Map is where query is relative to ref, dXY then is the shift % needed to move the predicted position to the measured. @@ -1567,7 +1564,7 @@ comMapX = sum(sum(bx.*cccMap))./sum(cccMap(:)); comMapY = sum(sum(by.*cccMap))./sum(cccMap(:)); - + % peak in Map is where query is relative to ref, dXY then is the shift % needed to move the predicted position to the measured. % Data moved from a position of estimated_global_offset, so add this to dXY @@ -1576,8 +1573,7 @@ end fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), dXY, wrkFid(iFid,5)); end % end of loop over fiducials - SAVE_IMG(tmpOut,'tmpOut.mrc',pixel_size); - error('tmpOut'); + if (calcCTF) [~,imDefC] = max(defocusCCC{iPrj},[],1); expectedDefocus = mean(defShiftVect(imDefC)); @@ -1656,7 +1652,7 @@ % the pixel size since this is the input to tiltalign. % fFull(:,2:3) = fFull(:,2:3).*pixel_size; - % fFull(:,2:3) = (samplingRate.*(fFull(:,2:3) - emc_get_origin_index(fullTiltSizeXandY./samplingRate)) + emc_get_origin_index(fullTiltSizeXandY)); + fFull(:,2:3) = (samplingRate.*(fFull(:,2:3) - emc_get_origin_index(fullTiltSizeXandY./samplingRate)) + emc_get_origin_index(fullTiltSizeXandY)); fprintf(fidCombine,'%d %4.4f %4.4f %d\n',fCombine'); fclose(fidCombine); @@ -1664,14 +1660,10 @@ fprintf(fidFull,'%d %4.4f %4.4f %d\n',fFull'); fclose(fidFull); % convert to model - system(sprintf(['point2model -zero -circle 3 -color 0,0,255 ',... - '%smapBack%d/%s.coordCombine %smapBack%d/%s_fit-comb.fid -ImageForCoordinates %smapBack%d/%s_1_mapBack.st '],mbOUT{1:3},mbOUT{1:3},mbOUT{1:3})); - system(sprintf(['point2model -zero -circle 3 -color 0,0,255 -ImageForCoordinates %smapBack%d/%s_1_mapBack.st ',... - '%smapBack%d/%s.coordFull %smapBack%d/%s_fit-full.fid'],mbOUT{1:3},mbOUT{1:3},mbOUT{1:3})); - system(sprintf(['point2model -zero -circle 3 -color 0,0,255 -ImageForCoordinates %smapBack%d/%s_1_mapBack.st ',... - '%smapBack%d/%s.coordBin%d %smapBack%d/%s_fit-bin%d.fid'],mbOUT{1:3},samplingRate,mbOUT{1:3},samplingRate,mbOUT{1:3})); + system(sprintf('point2model -zero -circle 3 -color 0,0,255 -ImageForCoordinates %smapBack%d/%s_1_mapBack.st %smapBack%d/%s.coordCombine %smapBack%d/%s_fit-comb.fid',mbOUT{1:3},mbOUT{1:3},mbOUT{1:3})); + system(sprintf('point2model -zero -circle 3 -color 0,0,255 -ImageForCoordinates %s %smapBack%d/%s.coordFull %smapBack%d/%s_fit-full.fid',tilt_filename,mbOUT{1:3},mbOUT{1:3})); + system(sprintf('point2model -zero -circle 3 -color 0,0,255 -ImageForCoordinates %smapBack%d/%s_1_mapBack.st %smapBack%d/%s.coordBin%d %smapBack%d/%s_fit-bin%d.fid',mbOUT{1:3},mbOUT{1:3},samplingRate,mbOUT{1:3},samplingRate)); % write the com script for running tiltalign - TN =tiltBaseName RotDef = 5; TltDef = 4; aliCom = fopen(sprintf('%smapBack%d/%s.align',mbOUT{1:3}),'w'); @@ -1710,7 +1702,7 @@ '#iTiltSeries %d\n',... 'tiltalign -StandardInput << EOF\n',... 'ModelFile %smapBack%d/%s_fit-full.fid\n',... - 'ImagesAreBinned %d\n',... + 'ImagesAreBinned 1\n',... 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... 'OutputResidualFile %smapBack%d/%s%s.resid\n',... 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... @@ -1750,12 +1742,12 @@ 'EOF'],... iTiltSeries,... mbOutAlt{1:3},... % for ModelFile - samplingRate, ... mbOutAlt{1:3},... outCTF,... mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... - tilt_script_name,n_surfaces, ... + tilt_script_name,... + n_surfaces, ... emc.rot_option_global, ... emc.tilt_option_global, ... emc.tilt_default_grouping_global, ... @@ -1830,7 +1822,6 @@ %%%system(sprintf('grep -A %d " At minimum tilt" ./mapBack/%s_ta.log > tmp.log',nPrjs+2,TN)); %%%system(sprintf('awk ''{if(NR >3) print $5}'' tmp.log > mapBack/%s.mag',TN)); %%%end %uf cibdutuib - error('early exit') end % loop over tilts diff --git a/testScripts/fourierTransformer.m b/testScripts/fourierTransformer.m index 05cc8997..4b6be75c 100644 --- a/testScripts/fourierTransformer.m +++ b/testScripts/fourierTransformer.m @@ -137,11 +137,15 @@ doNorm = 0; end - + % FIXME: inputVol ends up being modified, even though the pointers ar to read only memory% + % I checked the addresses of input/output and the are far enough apart that it doesn't seem like overlap. + % Force a copy of the inputVol. + ft = inputVol; + ft(1) = 0.9999999999999999999.*ft(1); if isempty([obj.plan_FWD,obj.plan_INV]) - [ ft, obj.plan_FWD, obj.plan_INV ] = mexFFT(inputVol,obj.invTrim); + [ ft, obj.plan_FWD, obj.plan_INV ] = mexFFT(ft,obj.invTrim); else - [ ft ] = mexFFT(inputVol,obj.invTrim,obj.plan_FWD, obj.plan_INV); + [ ft ] = mexFFT(ft,obj.invTrim,obj.plan_FWD, obj.plan_INV); end @@ -170,7 +174,12 @@ doNorm = 0; end - [ ft ] = mexFFT(inputVol,obj.invTrim,obj.plan_FWD, obj.plan_INV); + % FIXME: inputVol ends up being modified, even though the pointers ar to read only memory% + % I checked the addresses of input/output and the are far enough apart that it doesn't seem like overlap. + % Force a copy of the inputVol. + ft = inputVol; + ft(1) = 0.9999999999999999999.*ft(1); + [ ft ] = mexFFT(ft,obj.invTrim,obj.plan_FWD, obj.plan_INV); % For some reason calling this is OUTRAGEOUSLY slow, add option to jsut pass it in to the mexFFT if (doNorm) From b75fa8b062c6f591bfaaf891947cfcb9e2fb058f Mon Sep 17 00:00:00 2001 From: himesb Date: Thu, 7 Mar 2024 16:06:37 -0500 Subject: [PATCH 066/151] tomoCPR *feels* like it is going to be sorted. Moved imod transformation updates to be handled with xfproduct and resampling with newstack. --- ctf/BH_ctf_Estimate.m | 2 +- ctf/BH_ctf_Updatefft.m | 201 ++++++++++--------------------- metaData/SAVE_IMG.m | 2 + synthetic/BH_synthetic_mapBack.m | 2 +- 4 files changed, 69 insertions(+), 138 deletions(-) diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index b8d38ea4..02705e6f 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -204,7 +204,7 @@ TLT(:,1) = tltOrder(:,1); TLT(:,12) = (tltOrder(:,4)-tltOrder(:,5))./2 .* 10^-10; TLT(:,13) = tltOrder(:,6) .* (pi / 180); - TLT(:,15) = -1.*(tltOrder(:,4)+tltOrder(:,5))./2 .* 10^-10; + TLT(:,15) = 1.*(tltOrder(:,4)+tltOrder(:,5))./2 .* 10^-10; TLT(:,14) = tltOrder(:,3); sorted_dose = sortrows(tltOrder,2); cummul_dose = cumsum(sorted_dose(:,3)); diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 1945defd..90225167 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -1,7 +1,6 @@ function [ ] = BH_ctf_Updatefft( PARAMETER_FILE, STACK_PRFX, applyFullorUpdate) -global bh_global_do_2d_fourier_interp; emc = BH_parseParameterFile(PARAMETER_FILE); @@ -74,6 +73,8 @@ % For some reason matlab was geeking out about calling this in the parfor % loop, getting confused about whether it is a variable or a function. parfor iGPU = 1:nGPUs +% for iGPU = 1:nGPUs + % for iTilt = 1:length(ITER_LIST{iGPU}) if ( flgParallel ) @@ -216,9 +217,9 @@ gradientAliasMask = BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'); TLT = INPUT_CELL{iStack,1}; - pathName = INPUT_CELL{iStack,3} - fileName = INPUT_CELL{iStack,4} - extension = INPUT_CELL{iStack,5} + pathName = INPUT_CELL{iStack,3}; + fileName = INPUT_CELL{iStack,4}; + extension = INPUT_CELL{iStack,5}; % Optionally address magnification changes. @@ -252,7 +253,7 @@ if ( emc.eucentric_fit && mapBackIter ) toFit = abs(mbTLT) > emc.eucentric_maxTilt; - + error('eucentric fit not implemented') % For now take the mean, but it would probably be better to fit a line, % use the Y intercept, and use the deviation from 0 of the slope as a % measure of quality. @@ -302,31 +303,70 @@ sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); sizeCropped(3) = d3; - STACK = zeros(sizeCropped,'single'); - samplingMaskStack = zeros(sizeCropped,'single'); + if (mapBackIter) + tmp_xf = tempname; + tmp_xf_fd = fopen(tmp_xf,"w"); + for i = 1:d3 + fprintf(tmp_xf_fd,"%f %f %f %f %f %f\n",tlt_tmp{i}([7,8,9,10,2,3])); + end + fclose(tmp_xf_fd); + tmp_combined_xf = tempname; + cmd_base = sprintf('xfproduct %s %s.tltxf %s', tmp_xf, mapBackPrfx, tmp_combined_xf); + [ xfprod_err ] = system(sprintf('%s > /dev/null',cmd_base)); + if (xfprod_err) + system(cmd_base); + error('xfprod failed'); + else + mbEST = load(tmp_combined_xf, '-ascii'); + end + else + error('Why would we get here?') + end + + + base_cmd = sprintf('newstack -mode 12 -meansd 0,1 -xf %s %s %s',tmp_combined_xf,INPUT_CELL{iStack,2},outputStackName); + [ newstack_err ] = system(sprintf('%s > /dev/null',base_cmd)); + if (newstack_err) + system(base_cmd); + error('newstack failed'); + end + samplingMaskStack = ones(sizeCropped,'single'); + SAVE_IMG(samplingMaskStack,{sprintf('%s.samplingMask_pre',outputStackName), 'half'}, iPixelHeader,iOriginHeader); + + base_cmd = sprintf('newstack -mode 12 -fill 0 -xf %s %s.samplingMask_pre %s.samplingMask',tmp_combined_xf,outputStackName,outputStackName); + [ newstack_err ] = system(sprintf('%s > /dev/null',base_cmd)); + if (newstack_err) + system(base_cmd); + error('newstack failed'); + end + + system(sprintf('rm %s.samplingMask_pre',outputStackName)); + system(sprintf('rm %s %s',tmp_xf,tmp_combined_xf)); + for i = 1:d3 updateScale = 1; if (mapBackIter) - % Stored in row order as output by imod, st transpose is needed. Inversion - % of the xform is handled in resample2d. - origXF = reshape(tlt_tmp{i}(7:10),2,2)'; - newXF = reshape(mbEST(i,1:4),2,2)'; + % % Stored in row order as output by imod, st transpose is needed. Inversion + % % of the xform is handled in resample2d. + % origXF = reshape(tlt_tmp{i}(7:10),2,2)'; + % newXF = reshape(mbEST(i,1:4),2,2)'; - - dXYZ = [(newXF*tlt_tmp{i}(2:3)')' + mbEST(i,5:6).*updateScale , 0]; - if ~isvector(dXYZ) - % In case some implicit expansion were to happen for whatever reason. - error('dXYZ is a matrix and should be a vector'); - end + % dXYZ = [(newXF*tlt_tmp{i}(2:3)')' + mbEST(i,5:6).*updateScale , 0]; + % if ~isvector(dXYZ) + % % In case some implicit expansion were to happen for whatever reason. + % error('dXYZ is a matrix and should be a vector'); + % end + dXYZ(1:2) = tlt_tmp{i}(2:3); + combinedXF = tlt_tmp{i}(7:10); tlt_tmp{i}(2:3) = dXYZ(1:2); - combinedXF = reshape((newXF*origXF)',1,4); + % combinedXF = reshape((newXF*origXF)',1,4); tlt_tmp{i}(7:10) = combinedXF; else combinedXF = tlt_tmp{i}(7:10); @@ -338,123 +378,12 @@ % alignment.flgSkipUpdate dXYZ = dXYZ ./ updateScale; - - % Pad the projection prior to xforming in Fourier space. - sizeODD = [d1,d2]-[osX,osY]; - - % If it is even sized, shift up one pixel so that the origin is in the middle - % of the odd output here we can just read it in this way, unlike super res. - - iProjection = ... - OPEN_IMG('single', iMrcObj,[1+osX,d1],[1+osY,d2],tlt_tmp{i}(23),'keep'); - - iProjection = real(ifftn(fftn(iProjection).*gradientAliasMask)); - - largeOutliersMean= mean(iProjection(:)); - - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); - - % Because the rotation/scaling and translation are done separately, - % we must use a square transform; otherwise, a rotation angle dependent - % anisotropic distortion (like mag distortion) is introduced. - sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp.*0.25).*max(sizeODD)); - - - padVal = BH_multi_padVal(sizeODD,sizeSQ); - trimVal = BH_multi_padVal(sizeSQ,sizeCropped(1:2)); - - iProjection = iProjection - mean(iProjection(:)); - iProjection = iProjection ./ std(iProjection(:)); - - - iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - 'GPU','singleTaper'); - - if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); - end - - - if (flgApplyFullXform) - % Do the phase shift after rotating - need to invert the scaling since - % we are in reciprocal space - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(combinedXF); - % Assuming stretch and skew are not fit, leave defined for possible - % later consideration. - - - if (bh_global_do_2d_fourier_interp) - if (i == 1) - fprintf('resampling at 2x padding with fourier interp\n'); - end - % combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - combinedInverted = combinedInverted([1,2,4,5]); - - - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); - else - if (i == 1) - fprintf('resampling at 1x padding with linear interp\n'); - end - % Real space, do not invert mag - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); - end - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - - else - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(mbEST(i,1:4)); - % Assuming stretch and skew are not fit, leave defined for possible - % later consideration. - - % NOTE mag is ignored when the rotation matrix has 4 elements (IMOD) - - if (bh_global_do_2d_fourier_interp) - if (i == 1) - fprintf('resampling at 2x padding with fourier interp\n'); - end - % mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - mbEstInverted = mbEstInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',imodMAG,size(iProjection),bhF); - else - if (i == 1) - fprintf('resampling at 1x padding with linear interp\n'); - end - mbEstInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - mbEstInverted = mbEstInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,mbEstInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); - end - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),mbEST(i,1:4),dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - end - - - - STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... - trimVal(1,:),trimVal(2,:),... - 'GPU','single'))); - - iSamplingMask(isnan(iSamplingMask(:))) = 0; - samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); - iSamplingMask = []; - % - % end - - end - - for i= 1:d3 + TLT(i,:) = tlt_tmp{i}; % STACK(:,:,TLT(i,1)) = out_tmp{i}; end + out_tmp = []; if (mapBackIter) @@ -481,24 +410,23 @@ '%07.7f\t%07.7f\t%5e\t%5e\t%5e\t%7e\t%5e\t%5e\t%5e\t%5e\t%5e\t',... '%d\t%d\t%d\t%3.2f\n'], TLT'); - + STACK = gpuArray(OPEN_IMG('single',outputStackName)); if ( flgEraseBeads ) STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); end - fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',... - THICKNESS, STACK_PRFX); + fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',THICKNESS, STACK_PRFX); [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,emc.pixel_size_angstroms,gpuArray(samplingMaskStack)); - SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); - SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName)); + SAVE_IMG(STACK,{outputStackName,'half'},iPixelHeader,iOriginHeader); xShift= []; yShift = []; scale = []; angleShift = []; dZ = []; recZ= []; rotMat = []; angX = []; angY = []; else + error('This branch is deprecated and should not be reached.'); if ( flgEraseBeads ) STACK = BH_eraseBeads(STACK,eraseRadius, fileName, updateScale,mapBackIter,sortrows(TLT,1)); end @@ -527,6 +455,7 @@ % if (emc.eucentric_fit && mapBackIter) + error('eucentric fit not implemented') % Update the sub tomo z coords with an estimate of the shift cycle_to_update = subTomoMeta.('tomoCPR_run_in_cycle')(find(subTomoMeta.('tomoCPR_run_in_cycle')(:,1) == subTomoMeta.currentTomoCPR),2); for iGPU = 1:nGPUs diff --git a/metaData/SAVE_IMG.m b/metaData/SAVE_IMG.m index 66821205..cb086e5b 100644 --- a/metaData/SAVE_IMG.m +++ b/metaData/SAVE_IMG.m @@ -13,6 +13,8 @@ imgMean = gather(mean(vol(:))); imgRMS = gather(rms(vol(:))); +vol = gather(vol); + mRCImage = MRCImage(gather(vol)); % These are all expected to be singles diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 331d3dab..6394b648 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -1386,7 +1386,7 @@ coordOUT = fopen(sprintf('%smapBack%d/%s_%03d.coordFIT',mbOUT{1:3},iPrj),'w'); tmpOut = zeros([CTFSIZE,size(wrkFid,1)],'single','gpuArray'); - + dXY = [0,0]; for iFid = 1:size(wrkFid,1) if wrkFid(iFid,2) == -9999 From e23aab1c74f6a9f25ddadefe1ccba948dcad31cf Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 19 Mar 2024 11:20:26 -0400 Subject: [PATCH 067/151] wipo --- metaData/BH_parseParameterFile.m | 8 +- synthetic/BH_to_cisTEM_mapBack.m | 616 +++++++++++++++++++++++++++---- 2 files changed, 543 insertions(+), 81 deletions(-) diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index c342c3a7..f090c63b 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -399,7 +399,7 @@ if isfield(emc, 'flgPostShift') EMC_assert_numeric(emc.flgPostShift, 2); else - emc.flgPostShift = 0.*[1.0,-1.0]; + emc.flgPostShift = 1.*[1.0,-1.0]; end if isfield(emc, 'prjVectorShift') @@ -608,3 +608,9 @@ else emc.debug_print = false; end + +if isfield(emc, 'tmp_scan') + EMC_assert_numeric(emc.tmp_scan, 3, [-1, 1]); +else + emc.tmp_scan = [1,1,0]; +end \ No newline at end of file diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index c31ff0ee..bdbe0e65 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -14,13 +14,15 @@ % directlyCT % by the users (private methods-ish) - +skip_to_end = false; emc = BH_parseParameterFile(PARAMETER_FILE); MAX_EXPOSURE = EMC_str2double(MAX_EXPOSURE) if isnan(MAX_EXPOSURE) error('MAX_EXPOSURE is nan - if running from an interactive matlab session, did you enter as a string?'); end +% For trouble shooting on tilted images. +MIN_EXPOSURE = 0; % Ideally, we would transform fully and go back to the non-rotated stack. I think with the apoferritin test set, % The resolution will be high-enough to sort this out. useFixedNotAliStack = false; @@ -148,7 +150,7 @@ output_cell = {}; newstack_file = sprintf('%s/temp_particle_stack.newstack',mbOUT{1}); newstack_file_handle = fopen(newstack_file,'w'); -skip_to_end = false; + if ~(skip_to_end) for iTiltSeries = tiltStart:nTiltSeries @@ -391,7 +393,7 @@ tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; reconGeometry = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); - tomo_origin_wrt_specimen_origin = [ reconGeometry.dX_specimen_to_tomo, ... + tomo_origin_wrt_specimen_origin = [reconGeometry.dX_specimen_to_tomo, ... reconGeometry.dY_specimen_to_tomo, ... reconGeometry.dZ_specimen_to_tomo]; tomo_origin_in_tomo_frame = emc_get_origin_index([reconGeometry.NX, ... @@ -409,7 +411,7 @@ continue; end - modelRot = BH_defineMatrix([0,0,0],'Bah','fwdVector'); + modelRot = BH_defineMatrix([0,90,0],'Bah','fwdVector'); for iSubTomo = 1:nSubTomos @@ -421,13 +423,14 @@ subtomo_origin_wrt_specimen_origin = subtomo_origin_wrt_specimen_origin - emc.flgPreShift; % Reproject using tilt, so just save the 3d coords. - fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_specimen_origin' + [originRec(1),originRec(2),originRec(3)]'- emc.prjVectorShift([1,3,2])', fidIDX); + fprintf(coordOUT,'%0.4f %0.4f %0.4f %d\n', modelRot * subtomo_origin_wrt_specimen_origin' + ... + [originRec(1),originRec(3),originRec(2)]'- emc.prjVectorShift([1,2,3])', fidIDX); nPrjsIncluded = 0; for iPrj = 1:nPrjs iPrj_nat = find(TLT(:,1) == iPrj); - if (abs(TLT(iPrj_nat,11)) <= MAX_EXPOSURE) + if (MIN_EXPOSURE < abs(TLT(iPrj_nat,11)) && abs(TLT(iPrj_nat,11)) <= MAX_EXPOSURE) nPrjsIncluded = nPrjsIncluded + 1; % imod is indexing from zero @@ -469,9 +472,8 @@ % p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -scat -values 1 ',... % '%s/%s.coord %s/%s.3dfid'], ... % mbOUT{1:2},mbOUT{1:2}); - p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -scat -values 1 -ImageForCoordinates %s ',... - '%s/%s.coord %s/%s.3dfid'], ... - tilt_filepath, mbOUT{1:2},mbOUT{1:2}); + p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -scat -values -1 ', '%s/%s.coord %s/%s.3dfid'], ... + mbOUT{1:2},mbOUT{1:2}); system(p2m); taStr = [sprintf('%f',rawTLT(1,2))]; @@ -512,7 +514,6 @@ 'input %s\n', ... 'output %s/%s.fid\n', ... 'COSINTERP 0\n', ... - 'RotateBy90 \n',... 'THICKNESS %d\n', ... 'TILTFILE %s/%s_align.rawtlt\n', ... 'DefocusFile %s/%s_align.defocus\n', ... @@ -568,7 +569,6 @@ mbOUT{1:2}, mbOUT{1:2})) end - error('test') try fidList = load(sprintf('%s/%s.coordPrj',mbOUT{1:2})); catch @@ -688,20 +688,18 @@ sx = pixelX - floor(pixelX); sy = pixelY - floor(pixelY); - % With a pixel running from -0.5 to 0.5, the center of the pixel is at 0.0, to get the index we need to do this - % pixelX = floor(x_coord + 0.5); - % pixelY = floor(y_coord + 0.5); - - % x_start = pixelX - tileOrigin(1) + 1; - % y_start = pixelY - tileOrigin(2) + 1; - - % % The remainder is the result of the windowing operation - % sx = x_coord - pixelX; - % sy = y_coord - pixelY; + % pixelX = wrkFid(iFid,3); + % pixelY = wrkFid(iFid,4); + + % x_start = floor(pixelX+0.5) - tileOrigin(1) + 1; + % y_start = floor(pixelY+0.5) - tileOrigin(2) + 1; + + % sx = (pixelX - floor(pixelX+0.5)); + % sy = (pixelY - floor(pixelY+0.5)); particle_was_skipped = false; if ( x_start > 0 && y_start > 0 && x_start + tileSize(1) - 1 < sTX && y_start + tileSize(2) - 1 < sTY ) - output_particle_stack(:,:,iGpuDataCounter) = STACK(x_start:x_start+tileSize(1)-1,y_start:y_start+tileSize(2)-1,TLT(iPrj,1)); + output_particle_stack(:,:,iGpuDataCounter) = STACK(x_start:x_start+tileSize(1) - 1,y_start:y_start+tileSize(2) - 1,TLT(iPrj,1)); % The trasformation of the particle is e1,e2,e3,esym into it's postion in the tomogram frame, then % the tomogram is tilted about the original Y axis and then the original Z @@ -716,7 +714,7 @@ % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','fwdVector'); % rTilt = BH_defineMatrix([0,wrkDefAngTilt(iFid,3),wrkDefAngTilt(iFid,2)],'SPIDER','forwardVector'); - rTilt = BH_defineMatrix([wrkDefAngTilt(iFid,2),wrkDefAngTilt(iFid,3),0],'SPIDER','fwdVector'); + rTilt = BH_defineMatrix([emc.tmp_scan(1)*wrkDefAngTilt(iFid,2),emc.tmp_scan(2)*wrkDefAngTilt(iFid,3),emc.tmp_scan(3)*wrkDefAngTilt(iFid,2)],'SPIDER','fwdVector'); rotFull = rTilt*reshape(wrkPar(iFid,7:15),3,3); end @@ -806,13 +804,43 @@ end % skip to here % SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',output_prefix),pixelSize); +outputHeader = getHeader(MRCImage(sprintf('%s.mrc',output_prefix),0)); +outputSizeXandY = [outputHeader.nX,outputHeader.nY]; +outputNumberOfSlices = outputHeader.nZ; +fprintf('The output stack has %d slices and is %d x %d\n',outputNumberOfSlices,outputSizeXandY(1),outputSizeXandY(2)); + maxThreads = emc.('nCpuCores'); -%%%%%%%%%%%%%%%%%%%%%%%%%% +split_into_n_procs = 4; +% Make sure we have at least enough threads for 1 / proc +n_recon_procs = min(maxThreads, split_into_n_procs); +n_threads_per_proc = floor(maxThreads/n_recon_procs) .* ones(1,n_recon_procs); +% Use all available threads, even if some processors have more than others +leftover_threads = maxThreads - n_threads_per_proc(1)*n_recon_procs; +thread_counter = 1; +while (leftover_threads > 0) + this_proc = mod(thread_counder,4)+1; + n_threads_per_proc(this_proc) = n_threads_per_proc(this_proc) + 1; + leftover_threads = leftover_threads - 1; +end + +% Divide the stack into n_recon_procs parts +stack_boundaries = 1:floor((outputNumberOfSlices+n_recon_procs)/n_recon_procs):outputNumberOfSlices; + +if length(stack_boundaries) == n_recon_procs + stack_boundaries(n_recon_procs+1) = outputNumberOfSlices+1 +else + stack_boundaries(end) = outputNumberOfSlices+1; +end + + + +% %%%%%%%%%%%%%%%%%%%%%%%%% % Initial reconstruction -%%%%%%%%%%%%%%%%%%%%%%%%%%% -system(sprintf('rm -f %s_rec.sh',output_prefix)); -recScript = fopen(sprintf('%s_rec.sh',output_prefix), 'w'); +% %%%%%%%%%%%%%%%%%%%%%%%%%%% +for iProc = 1:n_recon_procs +system(sprintf('rm -f %s_rec_%d.sh',output_prefix, iProc)); +recScript = fopen(sprintf('%s_rec_%d.sh',output_prefix, iProc), 'w'); fprintf(recScript,[ ... '#!/bin/bash\n\n', ... '%s << eof\n', ... @@ -824,8 +852,8 @@ '%s_recFilt.mrc\n',... '%s_stats.txt\n',... '%s\n', ... - '1\n', ... - '0\n', ... + '%d\n', ... + '%d\n', ... '%3.3f\n', ... pixel size '%4.4f\n', ... molecularMass' '%3.3f\n', ... inermask ang @@ -845,25 +873,71 @@ 'No\n', ...Center mass [No] : 'No\n', ...Apply likelihood blurring [No] : 'No\n', ...Threshold input reconstruction [No] : - 'No\n', ...Dump intermediate arrays (merge later) [No] : - 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : - 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : - '%d\n', ...Max. threads to use for calculation [36] : + 'Yes\n', ...Dump intermediate arrays (merge later) [No] : + '%s/dump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : + '%s/dump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%d\n', ... Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... - symmetry,emc.pixel_size_angstroms, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); + symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache, iProc,tmpCache,iProc, n_threads_per_proc(iProc)); -fprintf(recScript, '\neof\n'); +fprintf(recScript, 'eof\n'); fclose(recScript); -system(sprintf('chmod a=wrx %s_rec.sh',output_prefix)); -system(sprintf('./%s_rec.sh',output_prefix)); +system(sprintf('chmod a=wrx %s_rec_%d.sh',output_prefix, iProc)); +pause(1); +if (iProc < n_recon_procs) + system(sprintf('./%s_rec_%d.sh 2>&1 > /dev/null &',output_prefix, iProc)); +else + system(sprintf('./%s_rec_%d.sh && wait',output_prefix, iProc)); +end + +end + + + +merge3d_name = sprintf('%s_merge3d.sh',output_prefix); +system(sprintf('rm -f %s',merge3d_name)); +merge3dScript = fopen(sprintf('%s',merge3d_name), 'w'); + +fprintf(merge3dScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s_rec1.mrc\n',... + '%s_rec2.mrc\n',... + '%s_recFilt.mrc\n',... + '%s_stats.txt\n',... + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '%s/dump_1_.dat\n', ... + '%s/dump_2_.dat\n', ... + '%d\n'], ... Number of dump files [8] : + getenv('EMC_MERGE3D'), ... + output_prefix, output_prefix, output_prefix, output_prefix, ... + emc.('particleMass')*10^3, ... + 0.0, mean(emc.('Ali_mRadius')), ... + tmpCache, tmpCache, n_recon_procs); + + +fprintf(merge3dScript, 'eof\n'); +fclose(merge3dScript); +system(sprintf('chmod a=wrx %s',merge3d_name)); +pause(1) +system(sprintf('./%s',merge3d_name)); +% clean up dumps +system(sprintf('rm %s/dump_?_*.dat',tmpCache)); + +% Get the FSC cutoff for refinement +fsc = importdata(sprintf('%s_stats.txt',output_prefix),' ',12); +fsc_cutoff = fsc.data(find(fsc.data(:,5) < 0.5,1),2) %%%%%%%%%%%%%%%%%%%%%%%%% % Refine -%%%%%%%%%%%%%%%%%%%%%%%%%%%% -system(sprintf('rm -f %s_ref.sh',output_prefix)); -refineScript = fopen(sprintf('%s_ref.sh',output_prefix), 'w'); +%%%%%%%%%%%%%%%%%%%%%%%%%%%%r +refine_shifts = sprintf('%s_ref_shifts.sh',output_prefix); +system(sprintf('rm -f %s',refine_shifts)); +refineScript = fopen(sprintf('%s',refine_shifts), 'w'); fprintf(refineScript,[ ... '#!/bin/bash\n\n', ... '%s << eof\n', ... @@ -885,7 +959,7 @@ '%3.3f\n', ... outermas ang '300.0\n',...Low resolution limit (A) [300.0] : '%3.3f\n',...High resolution limit (A) [8.0] : - '0.0\n',...Resolution limit for signed CC (A) (0.0 = max [0.0] : + '0.0\n',...Resolution limit for signerecon_21_25_17_stats_refined.txtd CC (A) (0.0 = max [0.0] : '0.0\n',...Res limit for classification (A) (0.0 = max) [0.0] : '0.0\n',...Mask radius for global search (A) (0.0 = max)[100.0] : '%3.3f\n',...Approx. resolution limit for search (A) [8] : @@ -919,68 +993,450 @@ ], getenv('EMC_REFINE3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... - resForFitting,resForFitting,maxThreads); + fsc_cutoff,fsc_cutoff,maxThreads); fprintf(refineScript, '\neof\n'); fclose(refineScript); -pause(3); -system(sprintf('chmod a=wrx %s_ref.sh',output_prefix)); -system(sprintf('./%s_ref.sh',output_prefix)); +pause(1); +system(sprintf('chmod a=wrx %s',refine_shifts)); +pause(1); +system(sprintf('./%s',refine_shifts)); %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Reconstruct refined %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -system(sprintf('rm -f %s_rec2.sh',output_prefix)); -recScript = fopen(sprintf('%s_rec2.sh',output_prefix), 'w'); -fprintf(recScript,[ ... +for iProc = 1:n_recon_procs + system(sprintf('rm -f %s_rec_%d.sh',output_prefix, iProc)); + recScript = fopen(sprintf('%s_rec_%d.sh',output_prefix, iProc), 'w'); + fprintf(recScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s_refined.star\n', ... sprintf('%s.star',output_prefix) + 'none.mrc\n', ... + '%s_rec1.mrc\n',... + '%s_rec2.mrc\n',... + '%s_recFilt_refined.mrc\n',... + '%s_stats_refined.txt\n',... + '%s\n', ... + '%d\n', ... + '%d\n', ... + '%3.3f\n', ... pixel size + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '0.0\n', ... rec res limit + '0.0\n', ... ref res limit + '5.0\n', ... Particle weighting factor (A^2) [5.0] + '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] + '1.0\n', ...Tuning parameter: smoothing factor [1.0] : + '1.0\n', ...Tuning parameters: padding factor [1.0] : + 'Yes\n', ...Normalize particles [Yes] : + 'No\n', ...Adjust scores for defocus dependence [no] : + 'No\n', ...Invert particle contrast [No] : + 'Yes\n', ...Exclude images with blank edges [yes] : + 'No\n', ...Crop particle images [no] : + 'Yes\n', ...FSC calculation with even/odd particles [Yes] : + 'No\n', ...Center mass [No] : + 'No\n', ...Apply likelihood blurring [No] : + 'No\n', ...Threshold input reconstruction [No] : + 'Yes\n', ...Dump intermediate arrays (merge later) [No] : + '%s/dump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : + '%s/dump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%d\n', ... Max. threads to use for calculation [36] : + ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... + symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')),tmpCache, iProc,tmpCache,iProc, n_threads_per_proc(iProc)); + + fprintf(recScript, 'eof\n'); + + fclose(recScript); + system(sprintf('chmod a=wrx %s_rec_%d.sh',output_prefix, iProc)); + pause(1); + if (iProc < n_recon_procs) + system(sprintf('./%s_rec_%d.sh 2>&1 > /dev/null &',output_prefix, iProc)); + else + system(sprintf('./%s_rec_%d.sh && wait',output_prefix, iProc)); + end + +end + + +merge3d_name = sprintf('%s_merge3d.sh',output_prefix); +system(sprintf('rm -f %s',merge3d_name)); +merge3dScript = fopen(sprintf('%s',merge3d_name), 'w'); + +fprintf(merge3dScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s_rec1.mrc\n',... + '%s_rec2.mrc\n',... + '%s_recFilt_refined.mrc\n',... + '%s_stats_refined.txt\n',... + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '%s/dump_1_.dat\n', ... + '%s/dump_2_.dat\n', ... + '%d\n'], ... Number of dump files [8] : + getenv('EMC_MERGE3D'), ... + output_prefix, output_prefix, output_prefix, output_prefix, ... + emc.('particleMass')*10^3, ... + 0.0, mean(emc.('Ali_mRadius')), ... + tmpCache, tmpCache, n_recon_procs); + + +fprintf(merge3dScript, 'eof\n'); +fclose(merge3dScript); +system(sprintf('chmod a=wrx %s',merge3d_name)); +pause(1) +system(sprintf('./%s',merge3d_name)); + +system(sprintf('rm %s/dump_?_*.dat',tmpCache)); + +% Get the FSC cutoff for refinement +fsc = importdata(sprintf('%s_stats_refined.txt',output_prefix),' ',12); +fsc_cutoff = fsc.data(find(fsc.data(:,5) < 0.5,1),2) + +%%%%%%%%%%%%%%%%%%%%%%%%% +% Refine +%%%%%%%%%%%%%%%%%%%%%%%%%%%%r +refine_angles = sprintf('%s_ref_angles.sh',output_prefix); +system(sprintf('rm -f %s',refine_angles)); +refineScript = fopen(sprintf('%s',refine_angles), 'w'); +fprintf(refineScript,[ ... '#!/bin/bash\n\n', ... '%s << eof\n', ... '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) - '%s_refined.star\n', ... sprintf('%s.star',output_prefix) - 'none.mrc\n', ... + '%s.star\n', ... sprintf('%s.star',output_prefix) + '%s_recFilt_refined.mrc\n',... + '%s_stats_refined.txt\n',... + 'yes\n',... Use statistics [Yes] : + 'my_projection_stack.mrc\n',... not going to be used : + '%s_refined2.star\n', ... + '%s_changes2.star\n', ...Output parameter changes + '%s\n',... Particle symmetry [C1] : + '1\n', ...First particle to refine (0 = first in stack) [1] : + '0\n', ...Last particle to refine (0 = last in stack) [0] : + '1.0\n',...Percent of particles to use (1 = all) [1.0] : + '%3.3f\n', ... pixel size + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '300.0\n',...Low resolution limit (A) [300.0] : + '%3.3f\n',...High resolution limit (A) [8.0] : + '0.0\n',...Resolution limit for signed CC (A) (0.0 = max [0.0] : + '0.0\n',...Res limit for classification (A) (0.0 = max) [0.0] : + '0.0\n',...Mask radius for global search (A) (0.0 = max)[100.0] : + '%3.3f\n',...Approx. resolution limit for search (A) [8] : + '0.0\n',...Angular step (0.0 = set automatically) [0.0] : + '20\n',...Number of top hits to refine [20] : + '10\n',...Search range in X (A) (0.0 = 0.5 * mask radius)[12] : + '10\n',...[12] : + '100.0\n',...2D mask X coordinate (A) [100.0] : + '100.0\n',...2D mask Y coordinate (A) [100.0] : + '100.0\n',...2D mask Z coordinate (A) [100.0] : + '100.0\n',...2D mask radius (A) [100.0] : + '500.0\n',...Defocus search range (A) [500.0] : + '50.0\n',...Defocus step (A) [50.0] : + '1.0\n',...Tuning parameters: padding factor [1.0] : + 'no\n',...Global search [No] : + 'yes\n',... Local refinement [Yes] : + 'yes\n',...Refine Psi [no] : + 'yes\n',...Refine Theta [no] : + 'yes\n',...Refine Phi [no] : + 'yes\n',...Refine ShiftX [Yes] : + 'yes\n',...Refine ShiftY [Yes] : + 'no\n',...Calculate matching projections [No] : + 'no\n',...Apply 2D masking [No] : + 'no\n',...Refine defocus [No] : + 'yes\n',...Normalize particles [Yes] : + 'no\n',...Invert particle contrast [No] : + 'yes\n',...Exclude images with blank edges [Yes] : + 'yes\n',...Normalize input reconstruction [Yes] : + 'no\n',...Threshold input reconstruction [No] : + '%2.2d\n', ...Max. threads to use for calculation [36] : + ], getenv('EMC_REFINE3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... + symmetry,emc.pixel_size_angstroms, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... + fsc_cutoff,fsc_cutoff,maxThreads); + +fprintf(refineScript, '\neof\n'); +fclose(refineScript); +pause(1); +system(sprintf('chmod a=wrx %s',refine_angles)); +pause(1); +system(sprintf('./%s',refine_angles)); + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Reconstruct refined +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +for iProc = 1:n_recon_procs + system(sprintf('rm -f %s_rec_%d.sh',output_prefix, iProc)); + recScript = fopen(sprintf('%s_rec_%d.sh',output_prefix, iProc), 'w'); + fprintf(recScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s_refined2.star\n', ... sprintf('%s.star',output_prefix) + 'none.mrc\n', ... + '%s_rec1.mrc\n',... + '%s_rec2.mrc\n',... + '%s_recFilt_refined2.mrc\n',... + '%s_stats_refined2.txt\n',... + '%s\n', ... + '%d\n', ... + '%d\n', ... + '%3.3f\n', ... pixel size + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '0.0\n', ... rec res limit + '0.0\n', ... ref res limit + '5.0\n', ... Particle weighting factor (A^2) [5.0] + '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] + '1.0\n', ...Tuning parameter: smoothing factor [1.0] : + '1.0\n', ...Tuning parameters: padding factor [1.0] : + 'Yes\n', ...Normalize particles [Yes] : + 'No\n', ...Adjust scores for defocus dependence [no] : + 'No\n', ...Invert particle contrast [No] : + 'Yes\n', ...Exclude images with blank edges [yes] : + 'No\n', ...Crop particle images [no] : + 'Yes\n', ...FSC calculation with even/odd particles [Yes] : + 'No\n', ...Center mass [No] : + 'No\n', ...Apply likelihood blurring [No] : + 'No\n', ...Threshold input reconstruction [No] : + 'Yes\n', ...Dump intermediate arrays (merge later) [No] : + '%s/dump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : + '%s/dump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%d\n', ... Max. threads to use for calculation [36] : + ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... + symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache, iProc,tmpCache,iProc, n_threads_per_proc(iProc)); + + fprintf(recScript, 'eof\n'); + + fclose(recScript); + system(sprintf('chmod a=wrx %s_rec_%d.sh',output_prefix, iProc)); + pause(1); + if (iProc < n_recon_procs) + system(sprintf('./%s_rec_%d.sh 2>&1 > /dev/null &',output_prefix, iProc)); + else + system(sprintf('./%s_rec_%d.sh && wait',output_prefix, iProc)); + end + +end + + +merge3d_name = sprintf('%s_merge3d.sh',output_prefix); +system(sprintf('rm -f %s',merge3d_name)); +merge3dScript = fopen(sprintf('%s',merge3d_name), 'w'); + +fprintf(merge3dScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... '%s_rec1.mrc\n',... '%s_rec2.mrc\n',... + '%s_recFilt_refined2.mrc\n',... + '%s_stats_refined2.txt\n',... + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '%s/dump_1_.dat\n', ... + '%s/dump_2_.dat\n', ... + '%d\n'], ... Number of dump files [8] : + getenv('EMC_MERGE3D'), ... + output_prefix, output_prefix, output_prefix, output_prefix, ... + emc.('particleMass')*10^3, ... + 0.0, mean(emc.('Ali_mRadius')), ... + tmpCache, tmpCache, n_recon_procs); + + +fprintf(merge3dScript, 'eof\n'); +fclose(merge3dScript); +system(sprintf('chmod a=wrx %s',merge3d_name)); +pause(1) +system(sprintf('./%s',merge3d_name)); + +system(sprintf('rm %s/dump_?_*.dat',tmpCache)); + +% Get the FSC cutoff for refinement +fsc = importdata(sprintf('%s_stats_refined2.txt',output_prefix),' ',12); +fsc_cutoff = fsc.data(find(fsc.data(:,5) < 0.5,1),2) +fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) + +% TODO dfocus refine if res high enough +% if (fsc_cutoff < 6.0) +if (false) +%%%%%%%%%%%%%%%%%%%%%%%%% +% Refine +%%%%%%%%%%%%%%%%%%%%%%%%%%%%r +refine_def = sprintf('%s_ref_def.sh',output_prefix); +system(sprintf('rm -f %s',refine_def)); +refineScript = fopen(sprintf('%s',refine_def), 'w'); +fprintf(refineScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s.star\n', ... sprintf('%s.star',output_prefix) '%s_recFilt_refined.mrc\n',... '%s_stats_refined.txt\n',... - '%s\n', ... - '1\n', ... - '0\n', ... + 'yes\n',... Use statistics [Yes] : + 'my_projection_stack.mrc\n',... not going to be used : + '%s_refined3.star\n', ... + '%s_changes3.star\n', ...Output parameter changes + '%s\n',... Particle symmetry [C1] : + '1\n', ...First particle to refine (0 = first in stack) [1] : + '0\n', ...Last particle to refine (0 = last in stack) [0] : + '1.0\n',...Percent of particles to use (1 = all) [1.0] : '%3.3f\n', ... pixel size '%4.4f\n', ... molecularMass' '%3.3f\n', ... inermask ang '%3.3f\n', ... outermas ang - '0.0\n', ... rec res limit - '0.0\n', ... ref res limit - '5.0\n', ... Particle weighting factor (A^2) [5.0] - '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] - '1.0\n', ...Tuning parameter: smoothing factor [1.0] : - '1.0\n', ...Tuning parameters: padding factor [1.0] : - 'Yes\n', ...Normalize particles [Yes] : - 'No\n', ...Adjust scores for defocus dependence [no] : - 'No\n', ...Invert particle contrast [No] : - 'Yes\n', ...Exclude images with blank edges [yes] : - 'No\n', ...Crop particle images [no] : - 'Yes\n', ...FSC calculation with even/odd particles [Yes] : - 'No\n', ...Center mass [No] : - 'No\n', ...Apply likelihood blurring [No] : - 'No\n', ...Threshold input reconstruction [No] : - 'No\n', ...Dump intermediate arrays (merge later) [No] : - 'dum_1.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : - 'dum_2.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '300.0\n',...Low resolution limit (A) [300.0] : + '%3.3f\n',...High resolution limit (A) [8.0] : + '0.0\n',...Resolution limit for signed CC (A) (0.0 = max [0.0] : + '0.0\n',...Res limit for classification (A) (0.0 = max) [0.0] : + '0.0\n',...Mask radius for global search (A) (0.0 = max)[100.0] : + '%3.3f\n',...Approx. resolution limit for search (A) [8] : + '0.0\n',...Angular step (0.0 = set automatically) [0.0] : + '20\n',...Number of top hits to refine [20] : + '10\n',...Search range in X (A) (0.0 = 0.5 * mask radius)[12] : + '10\n',...[12] : + '100.0\n',...2D mask X coordinate (A) [100.0] : + '100.0\n',...2D mask Y coordinate (A) [100.0] : + '100.0\n',...2D mask Z coordinate (A) [100.0] : + '100.0\n',...2D mask radius (A) [100.0] : + '5000.0\n',...Defocus search range (A) [500.0] : + '50.0\n',...Defocus step (A) [50.0] : + '1.0\n',...Tuning parameters: padding factor [1.0] : + 'no\n',...Global search [No] : + 'yes\n',... Local refinement [Yes] : + 'no\n',...Refine Psi [no] : + 'no\n',...Refine Theta [no] : + 'no\n',...Refine Phi [no] : + 'no\n',...Refine ShiftX [Yes] : + 'no\n',...Refine ShiftY [Yes] : + 'no\n',...Calculate matching projections [No] : + 'no\n',...Apply 2D masking [No] : + 'yes\n',...Refine defocus [No] : + 'yes\n',...Normalize particles [Yes] : + 'no\n',...Invert particle contrast [No] : + 'yes\n',...Exclude images with blank edges [Yes] : + 'yes\n',...Normalize input reconstruction [Yes] : + 'no\n',...Threshold input reconstruction [No] : '%2.2d\n', ...Max. threads to use for calculation [36] : - ], getenv('EMC_RECONSTRUCT3D'), output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... + ], getenv('EMC_REFINE3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,emc.pixel_size_angstroms, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), maxThreads); + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... + fsc_cutoff,fsc_cutoff,maxThreads); -fprintf(recScript, '\neof\n'); +fprintf(refineScript, '\neof\n'); +fclose(refineScript); +pause(1); +system(sprintf('chmod a=wrx %s',refine_def)); +pause(1); +system(sprintf('./%s',refine_def)); -fclose(recScript); -pause(2) -system(sprintf('chmod a=wrx %s_rec2.sh',output_prefix)); -pause(2) -system(sprintf('./%s_rec2.sh',output_prefix)); +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% Reconstruct refined +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +for iProc = 1:n_recon_procs + system(sprintf('rm -f %s_rec_%d.sh',output_prefix, iProc)); + recScript = fopen(sprintf('%s_rec_%d.sh',output_prefix, iProc), 'w'); + fprintf(recScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s_refined3.star\n', ... sprintf('%s.star',output_prefix) + 'none.mrc\n', ... + '%s_rec1.mrc\n',... + '%s_rec2.mrc\n',... + '%s_recFilt_refined3.mrc\n',... + '%s_stats_refined3.txt\n',... + '%s\n', ... + '%d\n', ... + '%d\n', ... + '%3.3f\n', ... pixel size + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '0.0\n', ... rec res limit + '0.0\n', ... ref res limit + '5.0\n', ... Particle weighting factor (A^2) [5.0] + '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] + '1.0\n', ...Tuning parameter: smoothing factor [1.0] : + '1.0\n', ...Tuning parameters: padding factor [1.0] : + 'Yes\n', ...Normalize particles [Yes] : + 'No\n', ...Adjust scores for defocus dependence [no] : + 'No\n', ...Invert particle contrast [No] : + 'Yes\n', ...Exclude images with blank edges [yes] : + 'No\n', ...Crop particle images [no] : + 'Yes\n', ...FSC calculation with even/odd particles [Yes] : + 'No\n', ...Center mass [No] : + 'No\n', ...Apply likelihood blurring [No] : + 'No\n', ...Threshold input reconstruction [No] : + 'Yes\n', ...Dump intermediate arrays (merge later) [No] : + '%s/dump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : + '%s/dump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%d\n', ... Max. threads to use for calculation [36] : + ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... + symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache, iProc,tmpCache,iProc, n_threads_per_proc(iProc)); + + fprintf(recScript, 'eof\n'); + + fclose(recScript); + system(sprintf('chmod a=wrx %s_rec_%d.sh',output_prefix, iProc)); + pause(1); + if (iProc < n_recon_procs) + system(sprintf('./%s_rec_%d.sh 2>&1 > /dev/null &',output_prefix, iProc)); + else + system(sprintf('./%s_rec_%d.sh && wait',output_prefix, iProc)); + end + +end + +merge3d_name = sprintf('%s_merge3d.sh',output_prefix); +system(sprintf('rm -f %s',merge3d_name)); +merge3dScript = fopen(sprintf('%s',merge3d_name), 'w'); + +fprintf(merge3dScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s_rec1.mrc\n',... + '%s_rec2.mrc\n',... + '%s_recFilt_refined3.mrc\n',... + '%s_stats_refined3.txt\n',... + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '%s/dump_1_.dat\n', ... + '%s/dump_2_.dat\n', ... + '%d\n'], ... Number of dump files [8] : + getenv('EMC_MERGE3D'), ... + output_prefix, output_prefix, output_prefix, output_prefix, ... + emc.('particleMass')*10^3, ... + 0.0, mean(emc.('Ali_mRadius')), ... + tmpCache, tmpCache, n_recon_procs); + + +fprintf(merge3dScript, 'eof\n'); +fclose(merge3dScript); +system(sprintf('chmod a=wrx %s',merge3d_name)); +pause(1) +system(sprintf('./%s',merge3d_name)); + +system(sprintf('rm %s/dump_?_*.dat',tmpCache)); + +fsc = importdata(sprintf('%s_stats_refined3.txt',output_prefix),' ',12); +fsc_cutoff = fsc.data(find(fsc.data(:,5) < 0.5,1),2) +fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) +end end From a0cd08708b492272fea8c84e5a4a15af5e6f0d09 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 22 Mar 2024 07:11:11 -0400 Subject: [PATCH 068/151] Updates to make pca with random subset work again, set boundary errors in cross-correlation to ignore (i'm processing more data by the edges) and other wip --- alignment/BH_runAutoAlign.m | 6 +- alignment/BH_templateSearch3d_2.m | 24 +- ctf/BH_ctf_Estimate.m | 218 +++++---------- ctf/BH_ctf_Updatefft.m | 2 +- logicals/EMC_assert_numeric.m | 2 +- metaData/BH_parseParameterFile.m | 14 + statistics/BH_multi_xcf_Translational.m | 29 +- statistics/BH_multi_xcf_Translational_2.m | 32 ++- statistics/BH_pcaPub.m | 16 +- synthetic/BH_synthetic_mapBack.m | 319 +++++++++++++--------- transformations/BH_average3d.m | 15 +- 11 files changed, 332 insertions(+), 345 deletions(-) diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index 46462708..28bde588 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -205,11 +205,11 @@ end inputStack = rotStack; clear rotStack imgRotation = imgRotation + 90; - SAVE_IMG(inputStack,sprintf('fixedStacks/%s.fixed',baseName),iPixelHeader,iOriginHeader); + SAVE_IMG(inputStack,{sprintf('fixedStacks/%s.fixed',baseName),'half'},iPixelHeader,iOriginHeader); elseif ( skip_tilts) % Originally saved in the skip_tilts block, but that is redundant if we % save in the switch_axes block in the new implementation. - SAVE_IMG(inputStack,sprintf('fixedStacks/%s.fixed',baseName),iPixelHeader,iOriginHeader); + SAVE_IMG(inputStack,{sprintf('fixedStacks/%s.fixed',baseName),'half'},iPixelHeader,iOriginHeader); else % No modifications, so just link to the original stack cd('fixedStacks'); @@ -240,7 +240,7 @@ inputStack(:,:,iPrj) = gather(tmpPrj); end -SAVE_IMG(inputStack,fixedName,emc.pixel_size_angstroms); +SAVE_IMG(inputStack,{fixedName,'half'},emc.pixel_size_angstroms); fprintf('finished preprocessing tilt-series\n'); clear tmpPrj inputStack diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 04837deb..66750add 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -60,24 +60,6 @@ tmpDecoy = 0 end -try - super_sample = emc.('super_sample'); - if (super_sample > 0) - [~,v] = system('cat $IMOD_DIR/VERSION'); - v = split(v,'.'); - if (EMC_str2double(v{1}) < 4 || (EMC_str2double(v{2}) <= 10 && EMC_str2double(v{3}) < 42)) - fprintf('Warning: imod version is too old for supersampling\n'); - super_sample = ''; - else - super_sample = sprintf(' -SuperSampleFactor %d',super_sample); - end - else - super_sample = ''; - end -catch - super_sample = ''; - expand_lines = ''; -end peakThreshold = emc.('Tmp_threshold'); @@ -1065,16 +1047,16 @@ resultsOUT = sprintf('./%s/%s_convmap.mrc',convTMPNAME,mapName); anglesOUT = sprintf('./%s/%s_angles.mrc',convTMPNAME,mapName); angleListOUT = sprintf('./%s/%s_angles.list',convTMPNAME,mapName); -SAVE_IMG(MRCImage(mag),resultsOUT); +SAVE_IMG(mag,{resultsOUT,'half'}); % SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); if ( tmpDecoy ) decoyOUT = sprintf('./%s/%s_decoy.mrc',convTMPNAME,mapName); - SAVE_IMG(MRCImage((RESULTS_decoy)),decoyOUT); + SAVE_IMG(RESULTS_decoy,{decoyOUT,'half'}); diffOUT = sprintf('./%s/%s_convmap-decoy.mrc',convTMPNAME,mapName); decoyLogical = mag < RESULTS_decoy; mag(decoyLogical) = 0; mag(~decoyLogical) = mag(~decoyLogical) - RESULTS_decoy(~decoyLogical); clear RESULTS_decoy - SAVE_IMG(MRCImage((mag)),diffOUT); + SAVE_IMG(mag,{diffOUT,'half'}); end angleFILE = fopen(angleListOUT,'w'); fprintf(angleFILE,'%2.2f\t%2.2f\t%2.2f\n', ANGLE_LIST'); diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 02705e6f..5978948c 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -320,174 +320,104 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% if ( resample_stack) + fprintf('Combining tranformations\n\n'); + % Load in the mapBack alignment + mbEST = load(sprintf('%s.xf',mapBackPrfx)); + mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); -fprintf('Combining tranformations\n\n'); -% Load in the mapBack alignment -mbEST = load(sprintf('%s.xf',mapBackPrfx)); -mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); + outputStackName = sprintf('aliStacks/%s%s',stackNameOUT,extension); -outputStackName = sprintf('aliStacks/%s%s',stackNameOUT,extension); - -try - erase_beads_after_ctf = emc.('erase_beads_after_ctf'); -catch - erase_beads_after_ctf = false; -end + try + erase_beads_after_ctf = emc.('erase_beads_after_ctf'); + catch + erase_beads_after_ctf = false; + end -if (erase_beads_after_ctf) - flgEraseBeads = 0; -else - if exist(sprintf('%s.erase',mapBackPrfx),'file') - flgEraseBeads = 1; - else + if (erase_beads_after_ctf) flgEraseBeads = 0; - fprintf('\nDid not find the gold bead file (%s) for erasing, will skip\n\n',sprintf('%s.erase',mapBackPrfx)); + else + if exist(sprintf('%s.erase',mapBackPrfx),'file') + flgEraseBeads = 1; + else + flgEraseBeads = 0; + fprintf('\nDid not find the gold bead file (%s) for erasing, will skip\n\n',sprintf('%s.erase',mapBackPrfx)); + end end -end - -sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); -sizeCropped(3) = d3; - -STACK = zeros(sizeCropped,'single'); -samplingMaskStack = zeros(sizeCropped,'single'); + sizeCropped = [d1,d2,d3]-(1-mod([d1,d2,d3],2)); + sizeCropped(3) = d3; + if (flgReOrderMapBack) + TLT = sortrows(TLT,1); + end -if (flgReOrderMapBack) - TLT = sortrows(TLT,1); -end - -% if any([d1,d2] > 4096) -% shiftMETHOD = 'cpu'; -% fprintf('transforming on cpu b/c > 4096\n') -% else -shiftMETHOD = 'GPU'; -% end - -% Redefine d3 incase views are ignored -d3 = size(TLT,1); - -osX = 1-mod(d1,2); osY = 1-mod(d2,2); + shiftMETHOD = 'GPU'; + % Redefine d3 incase views are ignored + d3 = size(TLT,1); + osX = 1-mod(d1,2); osY = 1-mod(d2,2); -for i = 1:d3 - % fprintf('Transforming prj %d in fourier space oversampled by 2x physical Nyquist\n',i); - - - % Stored in row order as output by imod, st transpose is needed. Inversion - % of the xform is handled in resample2d. - origXF = [1,0;0,1]; - - newXF = reshape(mbEST(TLT(i,23),1:4),2,2)'; - - - dXYZ = [(newXF*TLT(i,2:3)')' + mbEST(TLT(i,23),5:6) , 0]; - TLT(i,2:3) = dXYZ(1:2); - dXYZ = dXYZ ./ scalePixelsBy; - - combinedXF = reshape((newXF*origXF)',1,4); - TLT(i,7:10) = combinedXF; - - - - + for i = 1:d3 + % fprintf('Transforming prj %d in fourier space oversampled by 2x physical Nyquist\n',i); + origXF = [1,0;0,1]; + + newXF = reshape(mbEST(TLT(i,23),1:4),2,2)'; + + + dXYZ = [(newXF*TLT(i,2:3)')' + mbEST(TLT(i,23),5:6) , 0]; + TLT(i,2:3) = dXYZ(1:2); + dXYZ = dXYZ ./ scalePixelsBy; + + combinedXF = reshape((newXF*origXF)',1,4); + TLT(i,7:10) = combinedXF; sizeODD = [d1,d2]-[osX,osY]; - % If it is even sized, shift up one pixel so that the origin is in the middle - % of the odd output here we can just read it in this way, unlike super res. - - iProjection = ... - OPEN_IMG('single', iMrcObj,[1+osX,d1],[1+osY,d2],TLT(i,23),'keep'); - - iProjection = real(ifftn(fftn(iProjection).* BH_bandpass3d(1.*[d1-osX,d2-osY,1],0,0,0,'GPU','nyquistHigh'))); - - largeOutliersMean= mean(iProjection(:)); - largeOutliersSTD = std(iProjection(:)); - largeOutliersIDX = (iProjection < largeOutliersMean - 6*largeOutliersSTD | ... - iProjection > largeOutliersMean + 6*largeOutliersSTD); - iProjection(largeOutliersIDX) = (3*largeOutliersSTD).*randn([gather(sum(largeOutliersIDX(:))),1],'single'); - - largeOutliersIDX = []; - - - % Padding to avoid interpolation artifacts. For K3 images this can push - % a 2080 close to or over the limit, so it is been reduced to 1/4 (from - % 1) i.e. the image is paded to 1.25 x unless useFourierInterp is set > - % 1; - sizeSQ = floor(([1,1]+bh_global_do_2d_fourier_interp*0.25).*max(sizeODD)); - % sizeSQ = floor(([1,1]).*max(sizeODD)); - - padVal = BH_multi_padVal(sizeODD,sizeSQ); - trimVal = BH_multi_padVal(sizeSQ,sizeCropped(1:2)); - - - iProjection = iProjection - mean(iProjection(:)); - + % If it is even sized, shift up one pixel so that the origin is in the middle + % of the odd output here we can just read it in this way, unlike super res. + + end - iProjection = BH_padZeros3d(iProjection,padVal(1,:),padVal(2,:), ... - shiftMETHOD,'singleTaper'); - - if (i == 1 && bh_global_do_2d_fourier_interp) - bhF = fourierTransformer(iProjection,'OddSizeOversampled'); + base_cmd = sprintf('newstack -mode 12 -meansd 0,1 -xf %s.xf %s %s', mapBackPrfx, stackNameIN, outputStackName); + [ newstack_err ] = system(sprintf('%s > /dev/null',base_cmd)); + if (newstack_err) + system(base_cmd); + error('newstack failed'); end - - - - % Do the phase shift after rotating - need to invert the scaling since - % we are in reciprocal space - [imodMAG, imodStretch, imodSkewAngle, imodRot] = ... - BH_decomposeIMODxf(combinedXF); - - - - if (bh_global_do_2d_fourier_interp) - % combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(1/imodMAG); - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward'); - combinedInverted = combinedInverted([1,2,4,5]); - - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2), ... - 'Bah','GPU','forward',imodMAG,size(iProjection),bhF); - else - combinedInverted = BH_defineMatrix([imodRot,0,0],'Bah','forward').*(imodMAG); - combinedInverted = combinedInverted([1,2,4,5]); - iProjection = BH_resample2d(iProjection,combinedInverted,dXYZ(1:2),'Bah','GPU','forward',1.0,size(iProjection)); + + samplingMaskStack = ones(sizeCropped,'single'); + SAVE_IMG(samplingMaskStack,{sprintf('%s.samplingMask_pre',outputStackName), 'half'}, iPixelHeader,iOriginHeader); + + base_cmd = sprintf('newstack -mode 12 -fill 0 -xf %s.xf %s.samplingMask_pre %s.samplingMask',mapBackPrfx,outputStackName,outputStackName); + [ newstack_err ] = system(sprintf('%s > /dev/null',base_cmd)); + if (newstack_err) + system(base_cmd); + error('newstack failed'); end - - iSamplingMask = BH_resample2d(ones(sizeCropped(1:2),'single','gpuArray'),combinedXF,dXYZ(1:2),'Bah','GPU','forward',1.0,sizeCropped(1:2),NaN); - - iSamplingMask(isnan(iSamplingMask(:))) = 0; - samplingMaskStack(:,:,i) = (gather(real(iSamplingMask))); - iSamplingMask = []; - - % % % % % iProjection = real(fftshift(ifftn(ifftshift(iProjection)))); - STACK(:,:,i) = gather(real(BH_padZeros3d(iProjection, ... - trimVal(1,:),trimVal(2,:),... - shiftMETHOD,'single'))); - - + + system(sprintf('rm %s.samplingMask_pre',outputStackName)); end +STACK = gpuArray(OPEN_IMG('single',outputStackName)); if ( flgEraseBeads ) STACK = BH_eraseBeads(STACK,eraseRadius, fileName, scalePixelsBy,0,sortrows(TLT,1)); end - +samplingMaskStack = gpuArray(OPEN_IMG('single',sprintf('%s.samplingMask',outputStackName))); [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',100,emc.pixel_size_si*10^10,samplingMaskStack); +STACK = gather(STACK); +samplingMaskStack = gather(samplingMaskStack); +SAVE_IMG(STACK,{outputStackName, 'half'},iPixelHeader,iOriginHeader); +SAVE_IMG(samplingMaskStack,{sprintf('%s.samplingMask',outputStackName),'half'},iPixelHeader,iOriginHeader); -SAVE_IMG(MRCImage(STACK),outputStackName,iPixelHeader,iOriginHeader); -SAVE_IMG(MRCImage(samplingMaskStack),sprintf('%s.samplingMask',outputStackName),iPixelHeader,iOriginHeader); - -end if ~(flgSkip) gpuDevice(gpuIDX) [d1,d2,d3] = size(STACK); if (emc.pixel_size_si*10^10 < 0) - flgCrop = 1; [croppedIMG,pixelOUT] = cropIMG(STACK(:,:,1),emc.pixel_size_si*10^10); [d1C,d2C] = size(croppedIMG); clear croppedIMG @@ -498,15 +428,12 @@ % keep the optino for cropping. FIXED_FIRSTZERO = pixelOUT / 70 ; highCutoff = (pixelOUT*10^-10)/emc.('defCutOff'); - else - flgCrop = 0; pixelOUT = emc.pixel_size_si*10^10; d1C = d1; d2C = d2; tltForExp = TLT; - end d3 = size(STACK,3); @@ -600,16 +527,16 @@ tmpTile(:,:,3) = tmpTile(:,:,3) + thisTile; end - end - end - end + end % end of j + end % end of if iEvalMask + end % end of i fprintf('%d tiles at dZ= 0\t%d tiles at dZ > 0\t%d tiles at dZ < 0, after tilt %d\n',nT,nT2,nT3,k); % Apply the dose filter to the sum of each projection to save a bunch of % multiplicaiton psTile = psTile + tmpTile; - end + end % end of k clear tmpTile toc @@ -623,11 +550,7 @@ clear psTile if ~(skipFitting) - % % % % for iTile = 1:3 - % % % % rotAvgPowerSpec(:,:,iTile) = (fftshift(rotAvgPowerSpec(:,:,iTile))); - % % % % end AvgPowerSpec = rotAvgPowerSpec; - % TODO make a better rotational averaging funciton [rot1, rot2, ~, r1,r2, ~] = BH_multi_gridCoordinates(paddedSize.*[1,1], ... 'Cartesian','GPU', ... @@ -650,7 +573,6 @@ rotAvgPowerSpec = rotAvgPowerSpec ./ (720); clear a % rotAvgPowerSpec = rotAvgPowerSpec ./ (720.*(sqrt(fftshift(radialForCTF{1}.*(pixelOUT.*10^-10))))); clear a - is_a_bummer = ~isfinite(rotAvgPowerSpec); if sum(is_a_bummer,'all') > 0.5*numel(rotAvgPowerSpec) error('the rotated Avg power spectrum is more than half nan or inf'); diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 90225167..6c9df6e7 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -418,7 +418,7 @@ fprintf('Using an estimated thickenss of %3.3f nm for tilt-series %s\n',THICKNESS, STACK_PRFX); - + samplingMaskStack = gpuArray(OPEN_IMG('single',sprintf('%s.samplingMask',outputStackName))); [ STACK ] = BH_multi_loadAndMaskStack(STACK,TLT,'',THICKNESS,emc.pixel_size_angstroms,gpuArray(samplingMaskStack)); SAVE_IMG(STACK,{outputStackName,'half'},iPixelHeader,iOriginHeader); diff --git a/logicals/EMC_assert_numeric.m b/logicals/EMC_assert_numeric.m index 2c0d72e1..d806bc9e 100644 --- a/logicals/EMC_assert_numeric.m +++ b/logicals/EMC_assert_numeric.m @@ -43,7 +43,7 @@ function EMC_assert_numeric(input_val, varargin) end if ( assert_passed == false ) - error('EMC_assert_numeric: input is not numeric or has wrong number of elements'); + error('EMC_assert_numeric: input is not numeric or has wrong number of elements or is out of range'); end diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index f090c63b..3fcd1c32 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -420,12 +420,26 @@ emc.pixelMultiplier = -1; end +if isfield(emc, 'tomoCprLowPass') + EMC_assert_numeric(emc.tomoCprLowPass, 1); + if (emc.tomoCprLowPass < 20 || emc.tomoCprLowPass > 44) + fprintf('\n\n\tWARNING: tomoCprLowPass is outside the range of 20 to 44, this may be okay, but ... maybe not.\n\n'); + end +else + emc.tomoCprLowPass = 22; +end + if isfield(emc, 'tomoCPR_random_subset') EMC_assert_numeric(emc.tomoCPR_random_subset, 1); else emc.tomoCPR_random_subset = -1; end +if isfield(emc, 'tomoCPR_target_n_patches_x_y') + EMC_assert_numeric(emc.tomoCPR_target_n_patches_x_y, 2, [0, 100]); +else + emc.tomoCPR_target_n_patches_x_y = [0,0]; +end % I think this has been removed if isfield(emc, 'probabilityPeakiness') EMC_assert_numeric(emc.probabilityPeakiness, 1); diff --git a/statistics/BH_multi_xcf_Translational.m b/statistics/BH_multi_xcf_Translational.m index 25474c43..680682e6 100755 --- a/statistics/BH_multi_xcf_Translational.m +++ b/statistics/BH_multi_xcf_Translational.m @@ -68,24 +68,27 @@ % Center of mass calc only makes sense for positive values boX = boX - minVal; - catch - peakCOM = gather(peakCOM); - peakCoord = gather(peakCoord); - peakLOW = gather(peakLOW); - peakTOP = gather(peakTOP); - iCCCmap = gather(iCCCmap); - peakMask = gather(peakMask); - save('xfcCalcErr.mat','peakCOM','peakCoord','peakLOW','peakTOP',... - 'iCCCmap','peakMask'); - error('failed to box out COM calc, saving troubleshooting variables in xfcClacErr.mat') - end - - + + cMass = [ sum(sum(sum(boX.*cmX))) ; ... sum(sum(sum(boX.*cmY))) ; ... sum(sum(sum(boX.*cmZ))) ] ./ sum(boX(:)); PEAK_COORD = peakCoord + cMass'; + catch + PEAK_COORD = peakCoord; + % peakCOM = gather(peakCOM); + % peakCoord = gather(peakCoord); + % peakLOW = gather(peakLOW); + % peakTOP = gather(peakTOP); + % iCCCmap = gather(iCCCmap); + % peakMask = gather(peakMask); + % save('xfcCalcErr.mat','peakCOM','peakCoord','peakLOW','peakTOP',... + % 'iCCCmap','peakMask'); + % error('failed to box out COM calc, saving troubleshooting variables in xfcClacErr.mat') + end + + end end diff --git a/statistics/BH_multi_xcf_Translational_2.m b/statistics/BH_multi_xcf_Translational_2.m index 9580ffd6..55d2f221 100644 --- a/statistics/BH_multi_xcf_Translational_2.m +++ b/statistics/BH_multi_xcf_Translational_2.m @@ -96,24 +96,26 @@ boX = iCCCmap(peakLOW(1):peakTOP(1), ... peakLOW(2):peakTOP(2), ... peakLOW(3):peakTOP(3)); + + cMass = [ sum(sum(sum(boX.*cmX))) ; ... + sum(sum(sum(boX.*cmY))) ; ... + sum(sum(sum(boX.*cmZ))) ] ./ sum(boX(:)); + + PEAK_COORD = peakCoord + cMass'; catch - peakCOM = gather(peakCOM); - peakCoord = gather(peakCoord); - peakLOW = gather(peakLOW); - peakTOP = gather(peakTOP); - iCCCmap = gather(iCCCmap); - peakMask = gather(peakMask); - save('xfcCalcErr.mat','peakCOM','peakCoord','peakLOW','peakTOP',... - 'iCCCmap','peakMask'); - error('failed to box out COM calc, saving troubleshooting variables in xfcClacErr.mat') + PEAK_COORD = peakCoord; + % peakCOM = gather(peakCOM); + % peakCoord = gather(peakCoord); + % peakLOW = gather(peakLOW); + % peakTOP = gather(peakTOP); + % iCCCmap = gather(iCCCmap); + % peakMask = gather(peakMask); + % save('xfcCalcErr.mat','peakCOM','peakCoord','peakLOW','peakTOP',... + % 'iCCCmap','peakMask'); + % error('failed to box out COM calc, saving troubleshooting variables in xfcClacErr.mat') end - - cMass = [ sum(sum(sum(boX.*cmX))) ; ... - sum(sum(sum(boX.*cmY))) ; ... - sum(sum(sum(boX.*cmZ))) ] ./ sum(boX(:)); - - PEAK_COORD = peakCoord + cMass'; + end end diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 802f3791..70e8cd18 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -84,6 +84,12 @@ end + + + + + + % FIXME: % To test seeding the classification with existing classes, rather than always reverting to the global average, % use the mechanism in place to handle multiple references at different length scales derived from the global average, @@ -727,7 +733,7 @@ % classification will be done at a later stage after reducing to some % subset of peaks. FIXME includeParticle = positionList(iSubTomo, 8); - + particleIDX = positionList(iSubTomo, 4); % Same for all peaks iPeak=0; % make sure this exists if we are no including the particle if (includeParticle) make_sf3d = true; @@ -771,7 +777,6 @@ end using_this_subtomo = true; - particleIDX = positionList(iSubTomo, 4); % Same for all peaks if ~ischar(indVAL) % Read in and interpolate at single precision as the local values % in the interpolant suffer from any significant round off errors. @@ -894,7 +899,12 @@ end % end of loop over peaks else nIgnored = nIgnored + 1; - fprintf('Ignoring outside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + if ~(emc.Pca_randSubset) + fprintf('Ignoring outside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + end + % This could also probably go in the above not random_subset clause, but also + % i don't think will change anthything downstream as it shoul dbe ignored on the full run. + % Leaving it this way means the intermediate metadata is more accurately reflecting the process. eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; end % end of ignore if statment from extracted window out of bounds (ischar(indVAL)) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 6394b648..6759f891 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -17,7 +17,7 @@ % FIXME: is this even relevant any more? buildTomo=1;% % % % % % % - +save_diagnostic_ccf=0; % If false, this is faster, simplifies the code and permits defocus estimation % This will likely be removed in favor of deleting all the blockes under its control use_background_estimate = false; @@ -240,6 +240,7 @@ continue; end + mapBackRePrjSize = min(256,subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize')); % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); @@ -300,20 +301,8 @@ peak_search_radius = floor(emc.peak_mask_fraction .* particle_radius .* [1,1]); % FIXME: this should be in parseParameterFile - try - lowPassCutoff = emc.('tomoCprLowPass'); - fprintf('Using a user supplied lowpass cutoff of %3.3f Ang\n', lowPassCutoff); - catch - % TODO are these range limits okay? - lowPassCutoff = 1.5.*mean(subTomoMeta.currentResForDefocusError); - if (lowPassCutoff < 10) - lowPassCutoff = 10; - elseif (lowPassCutoff > 24) - lowPassCutoff = 24; - end - fprintf('Using an internatlly determined lowpass cutoff of %3.3f Ang\n',... - lowPassCutoff); - end + + lowPassCutoff = emc.('tomoCprLowPass'); % FIXME: this can also be in parseParameterFile if lowPassCutoff < 2* pixel_size @@ -336,7 +325,13 @@ end % TODO: these defaults should be re-examined - targetPatchSize = ceil(max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch)))); + if any(emc.tomoCPR_target_n_patches_x_y) + % This will be re-calculated once the tilt-series size is known. + targetPatchSize = emc.tomoCPR_target_n_patches_x_y; + else + targetPatchSize = ceil(max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch)))); + end + % The binned stacks should already exist, if not, this will re-create it in the cache dir. % Note that this will also be checked when reconstructing the full 3d background tomo. @@ -533,6 +528,11 @@ % FIXME, this isn't necessarily going to be the correct size fullTiltSizeXandY = [tiltHeader.nX,tiltHeader.nY].*samplingRate; + if any(emc.tomoCPR_target_n_patches_x_y) + % This will be re-calculated once the tilt-series size is known. + targetPatchSize = floor([tiltHeader.nX,tiltHeader.nY] ./ emc.tomoCPR_target_n_patches_x_y); + fprintf('\nUsing targetPatchSize of [%d,%d] for %s\n',targetPatchSize, tomoList{iTomo}); + end sTX = floor(tiltHeader.nX ); sTY = floor(tiltHeader.nY ); @@ -787,7 +787,7 @@ system(rotCMD); - % system(sprintf('rm %smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave)); + system(sprintf('rm %smapBack%d/%s.tmpTomo%d', mbOUT{1:3},iSave)); end clear avgTomo{1} wgt @@ -1096,8 +1096,12 @@ % peakMask(peakMask < 0.99) = 0; - peakMask = EMC_gaussianKernel(CTFSIZE,tileRadius./2,'gpu',{}); + dataMask = EMC_gaussianKernel(tileSize,particle_radius,'gpu',{}); + dataMask = dataMask ./ max(dataMask(:)); + + peakMask = EMC_gaussianKernel(CTFSIZE,particle_radius./3,'gpu',{}); peakMask = peakMask ./ max(peakMask(:)); + @@ -1114,7 +1118,7 @@ % look around +/- this value % globalPeak = max(2,ceil(10/pixel_size)); % globalPeak = globalPeak + mod(globalPeak,2); - globalPeak = floor(max(sTX,sTY).*0.25); + globalPeak = floor( min(max(sTX,sTY).*0.25, 120/pixel_size) ); globalPeak = globalPeak + mod(globalPeak,2); globalPeakMask = zeros([sTX,sTY,1],'single'); @@ -1179,7 +1183,7 @@ end - % for iPrj = 1:nPrjs + % for iPrj = 1:nPrjs % recert parfor iPrj = 1:nPrjs % % For some reason if these mrc objects are created before the parfor @@ -1322,7 +1326,7 @@ - cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).*conj(fftn(refPrj).*Hqz)))); + cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).*(conj(fftn(refPrj).*Hqz))))); % cccPrj = fftshift(real(ifftn(bandPassPrj.*fftn(dataPrj).*... % % conj(fftn(refPrj).*Hqz)))); @@ -1435,17 +1439,21 @@ continue end - dataTile = dataPrj(ox:ox+2.*tileRadius,oy:oy+2.*tileRadius); + dataTile = dataMask.*dataPrj(ox:ox+2.*tileRadius,oy:oy+2.*tileRadius); dataTile = dataTile - mean(dataTile(:)); - refTile = refPrj(ox:ox+2.*tileRadius,oy:oy+2.*tileRadius); + refTile = dataMask.*refPrj(ox:ox+2.*tileRadius,oy:oy+2.*tileRadius); refTile = refTile - mean(refTile(:)); dataTile = dataTile./rms(dataTile(:)); refTile = refTile ./ rms(refTile(:)); + if (save_diagnostic_ccf) + SAVE_IMG(dataTile, 'dataTile.mrc'); + SAVE_IMG(refTile, 'refTile.mrc'); + end dataTile = ctfMask.*BH_padZeros3d(dataTile,'fwd',padCTF, ... @@ -1454,8 +1462,11 @@ refTile = ctfMask.*BH_padZeros3d(refTile,'fwd',padCTF, ... 'GPU','singleTaper'); - - + if (save_diagnostic_ccf) + SAVE_IMG(dataTile, 'dataTile_pad.mrc'); + SAVE_IMG(refTile, 'refTile_pad.mrc'); + end + df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; @@ -1479,7 +1490,7 @@ % try iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')); cccMap = dataFT .* iRefCTF; - + if (use_PCF) cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.001); @@ -1487,6 +1498,10 @@ cccMap = peakMask.*real(bhF.invFFT(cccMap)); + if (save_diagnostic_ccf) + SAVE_IMG(cccMap, 'ccfMap.mrc'); + end + tmpOut(:,:,iFid) = cccMap; @@ -1553,7 +1568,9 @@ end cccMap = peakMask.*real(bhF.invFFT(cccMap)); - + if (save_diagnostic_ccf) + SAVE_IMG(cccMap, 'ccfMap_2.mrc'); + end [~,maxMap] = max(cccMap(:)); [mMx, mMy] = ind2sub(size(cccMap), maxMap); @@ -1572,6 +1589,9 @@ dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estimated_global_offset + [sx,sy]; end fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), dXY, wrkFid(iFid,5)); + if (save_diagnostic_ccf) + error('save_diagnostic_ccf'); + end end % end of loop over fiducials if (calcCTF) @@ -1616,9 +1636,6 @@ fidBin = fopen(sprintf('%smapBack%d/%s.coordBin%d',mbOUT{1:3},samplingRate),'w'); fidList = fidList(:,2:end); - % shifts/List col 1/4 should match - maybe add a check to be safe - size(fidShifts) - size(fidList) fCombine = [fidShifts(:,1),fidList(:,2:3)+fidShifts(:,2:3),fidShifts(:,4)]; fprintf('\n\n%d/%d pts ignored\n\n',sum(fCombine(:,4)==-9999),size(fCombine,1)); @@ -1666,7 +1683,8 @@ % write the com script for running tiltalign RotDef = 5; TltDef = 4; - aliCom = fopen(sprintf('%smapBack%d/%s.align',mbOUT{1:3}),'w'); + aliCom_name = sprintf('%smapBack%d/%s.align',mbOUT{1:3}); + aliCom = fopen(aliCom_name,'w'); if (emc.shift_z_to_to_centroid) final_line1 = 'ShiftZFromOriginal'; @@ -1687,107 +1705,152 @@ [~,tn2,tn3] = fileparts(iRawTltName); tilt_script_name = sprintf('cache/mapBack%d/%s%s',mbOUT{2},tn2,tn3); end - - % TODO: It looks like the output model file (3dmod) is the solved positions, - % but is saved at a pixel size of 1. Scaling by the sampling rate in all dimensions - % and then adding the origin (only for Z) places the coordinates back into the bin6 model - % I think we could get shifts from this - % TODO: could use ImageOriginXandY to accound for a diffence in origin due to binning - % as ImageSizeXandY given as binned size*sampling rate, which may not equal full size - % fullTiltSizeXandY,... - % unsampled_pixel_size,unsampled_pixel_size,... - % 'ImageSizeXandY %d,%d\n',... - % 'ImagePixelSizeXandY %f,%f\n',... - fprintf(aliCom,['#!/bin/bash\n\n',... - '#iTiltSeries %d\n',... - 'tiltalign -StandardInput << EOF\n',... - 'ModelFile %smapBack%d/%s_fit-full.fid\n',... - 'ImagesAreBinned 1\n',... - 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... - 'OutputResidualFile %smapBack%d/%s%s.resid\n',... - 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... - 'OutputTiltFile %smapBack%d/%s%s.tlt\n',... - 'OutputXAxisTiltFile %smapBack%d/%s%s.xtilt\n',... - 'OutputTransformFile %smapBack%d/%s%s.tltxf\n',... - 'RotationAngle 0.00\n',... % assumed to be rotated already - 'TiltFile %s\n',... - 'SurfacesToAnalyze %d\n',... - 'RotOption %d\n',... % def solve all rotations - 'RotDefaultGrouping 3\n',... % if rot option --> 5 use def group size - 'TiltOption %d\n',... % Tilts are harder use automapping - 'TiltDefaultGrouping %d\n',... - 'MagOption %d\n',... % def solve all mags - 'MagDefaultGrouping %d\n',... - 'XStretchOption 0\n',... - 'SkewOption 0\n',... - 'BeamTiltOption 0\n',... - 'XTiltOption 0\n',... - 'ResidualReportCriterion 0.001\n',... - 'RobustFitting\n',... - 'KFactorScaling %3.3f\n',... - 'LocalAlignments\n',... - 'LocalRotOption %d\n',... - 'LocalRotDefaultGrouping %d\n',... - 'LocalTiltOption %d\n',... - 'LocalTiltDefaultGrouping %d\n',... - 'LocalMagOption %d\n',... - 'LocalMagDefaultGrouping %d\n',... - 'OutputLocalFile %smapBack%d/%s%s.local\n',... - 'TargetPatchSizeXandY %d,%d\n', ... - 'MinFidsTotalAndEachSurface %d,%d\n',... - 'MinSizeOrOverlapXandY %f,%f\n',... - '%s\n',... - '%s\n',... - '%s\n',... - 'EOF'],... - iTiltSeries,... - mbOutAlt{1:3},... % for ModelFile - mbOutAlt{1:3},... - outCTF,... - mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... - mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... - tilt_script_name,... - n_surfaces, ... - emc.rot_option_global, ... - emc.tilt_option_global, ... - emc.tilt_default_grouping_global, ... - emc.mag_option_global, ... - emc.mag_default_grouping_global, ... - emc.k_factor_scaling, ... - emc.rot_option_local, ... - emc.rot_default_grouping_local, ... - emc.tilt_option_local, ... - emc.tilt_default_grouping_local, ... - emc.mag_option_local, ... - emc.mag_default_grouping_local, ... - mbOutAlt{1:3},outCTF,targetPatchSize, ... - targetPatchSize,... - nFiducialsPerPatch, ... - floor(nFiducialsPerPatch/3),... - emc.min_overlap,emc.min_overlap,... - final_line1,final_line2,final_line3); - % % % Assume that any backlash was solved well enough that there are no major - % % % discontinuities in the coarse alignment. Mag and rot are solved/ tilt in - % % % the global solution anyhow, so this shouldn't be a bit deal. - % % % 'SeparateGroup 1-%d\n',... - % % % iViewGroup, - % fprintf(aliCom,'\n\ngrep -A %d " At minimum tilt" ./mapBack%d/%s_ta.log > ./mapBack%d/tmp.log',nPrjs+2,mbOUT{1:3},mbOUT{1:3}); - % fprintf(aliCom,'\nawk ''{if(NR >3) print $5}'' ./mapBack%d/tmp.log > mapBack%d/%s.mag',mbOUT{1:3},mbOUT{1:3}); - + + % Testing local alignment with optimized parameters using the new imod options for leave out + fprintf(aliCom,['%smapBack%d/%s_fit-full.fid\n',... %1 + '%smapBack%d/%s%s.3dmod\n',... %2 + '%smapBack%d/%s%s.resid\n',... %3 + '%smapBack%d/%s%s.xyz\n',... %4 + '%smapBack%d/%s%s.tlt\n',... %5 + '%smapBack%d/%s%s.xtilt\n',... %6 + '%smapBack%d/%s%s.tltxf\n',... %7 + '%s\n',... %8 input tilt file + '%3.3f\n',... KFactorScaling %9 + '%smapBack%d/%s%s.local\n',... OutputLocalFile %10 + '%d\n%d\n', ...TargetPatchSizeXandY %11 12 + '%d\n%d\n',... MinFidsTotalAndEachSurface %13 14 + '%f\n%f\n',... MinSizeOrOverlapXandY %15 16 + '%smapBack%d/%s.align_ta.log\n',... output log file %17 + '%smapBack%d/%s.align_ta_optimizer.log\n'],... output log file for optimizer %18 + mbOutAlt{1:3},... % for ModelFile + mbOutAlt{1:3},outCTF,... %2 + mbOutAlt{1:3},outCTF,... %3 + mbOutAlt{1:3},outCTF,... %4 + mbOutAlt{1:3},outCTF,... %5 + mbOutAlt{1:3},outCTF,... %6 + mbOutAlt{1:3},outCTF, ... %7 + tilt_script_name,... + emc.k_factor_scaling, ... + mbOutAlt{1:3},outCTF, ... + targetPatchSize, ... + targetPatchSize,... + nFiducialsPerPatch, ... + floor(nFiducialsPerPatch/3),... + emc.min_overlap, ... + emc.min_overlap,... + mbOutAlt{1:3},... + mbOutAlt{1:3}); + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % % TODO: It looks like the output model file (3dmod) is the solved positions, + % % but is saved at a pixel size of 1. Scaling by the sampling rate in all dimensions + % % and then adding the origin (only for Z) places the coordinates back into the bin6 model + % % I think we could get shifts from this + % % TODO: could use ImageOriginXandY to accound for a diffence in origin due to binning + % % as ImageSizeXandY given as binned size*sampling rate, which may not equal full size + % % fullTiltSizeXandY,... + % % unsampled_pixel_size,unsampled_pixel_size,... + % % 'ImageSizeXandY %d,%d\n',... + % % 'ImagePixelSizeXandY %f,%f\n',... + % fprintf(aliCom,['#!/bin/bash\n\n',... + % '#iTiltSeries %d\n',... + % 'tiltalign -StandardInput << EOF\n',... + % 'ModelFile %smapBack%d/%s_fit-full.fid\n',... + % 'ImagesAreBinned 1\n',... + % 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... + % 'OutputResidualFile %smapBack%d/%s%s.resid\n',... + % 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... + % 'OutputTiltFile %smapBack%d/%s%s.tlt\n',... + % 'OutputXAxisTiltFile %smapBack%d/%s%s.xtilt\n',... + % 'OutputTransformFile %smapBack%d/%s%s.tltxf\n',... + % 'RotationAngle 0.00\n',... % assumed to be rotated already + % 'TiltFile %s\n',... + % 'SurfacesToAnalyze %d\n',... + % 'RotOption %d\n',... % def solve all rotations + % 'RotDefaultGrouping 3\n',... % if rot option --> 5 use def group size + % 'TiltOption %d\n',... % Tilts are harder use automapping + % 'TiltDefaultGrouping %d\n',... + % 'MagOption %d\n',... % def solve all mags + % 'MagDefaultGrouping %d\n',... + % 'XStretchOption 0\n',... + % 'SkewOption 0\n',... + % 'BeamTiltOption 0\n',... + % 'XTiltOption 0\n',... + % 'ResidualReportCriterion 0.001\n',... + % 'RobustFitting\n',... + % 'KFactorScaling %3.3f\n',... + % 'LocalAlignments\n',... + % 'LocalRotOption %d\n',... + % 'LocalRotDefaultGrouping %d\n',... + % 'LocalTiltOption %d\n',... + % 'LocalTiltDefaultGrouping %d\n',... + % 'LocalMagOption %d\n',... + % 'LocalMagDefaultGrouping %d\n',... + % 'OutputLocalFile %smapBack%d/%s%s.local\n',... + % 'TargetPatchSizeXandY %d,%d\n', ... + % 'MinFidsTotalAndEachSurface %d,%d\n',... + % 'MinSizeOrOverlapXandY %f,%f\n',... + % '%s\n',... + % '%s\n',... + % '%s\n',... + % 'EOF'],... + % iTiltSeries,... + % mbOutAlt{1:3},... % for ModelFile + % mbOutAlt{1:3},... + % outCTF,... + % mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... + % mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... + % tilt_script_name,... + % n_surfaces, ... + % emc.rot_option_global, ... + % emc.tilt_option_global, ... + % emc.tilt_default_grouping_global, ... + % emc.mag_option_global, ... + % emc.mag_default_grouping_global, ... + % emc.k_factor_scaling, ... + % emc.rot_option_local, ... + % emc.rot_default_grouping_local, ... + % emc.tilt_option_local, ... + % emc.tilt_default_grouping_local, ... + % emc.mag_option_local, ... + % emc.mag_default_grouping_local, ... + % mbOutAlt{1:3},outCTF,targetPatchSize, ... + % targetPatchSize,... + % nFiducialsPerPatch, ... + % floor(nFiducialsPerPatch/3),... + % emc.min_overlap,emc.min_overlap,... + % final_line1,final_line2,final_line3); + % % % % Assume that any backlash was solved well enough that there are no major + % % % % discontinuities in the coarse alignment. Mag and rot are solved/ tilt in + % % % % the global solution anyhow, so this shouldn't be a bit deal. + % % % % 'SeparateGroup 1-%d\n',... + % % % % iViewGroup, + % % fprintf(aliCom,'\n\ngrep -A %d " At minimum tilt" ./mapBack%d/%s_ta.log > ./mapBack%d/tmp.log',nPrjs+2,mbOUT{1:3},mbOUT{1:3}); + % % fprintf(aliCom,'\nawk ''{if(NR >3) print $5}'' ./mapBack%d/tmp.log > mapBack%d/%s.mag',mbOUT{1:3},mbOUT{1:3}); + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% fclose(aliCom); - system(sprintf('chmod a=wrx %smapBack%d/%s.align',mbOUT{1:3})); + % system(sprintf('chmod a=wrx %smapBack%d/%s.align',mbOUT{1:3})); if (is_first_run) + % if ( multi_node_run ) + % fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'w'); + % fprintf(fOUT,['%smapBack%d/%s.align > ',... + % '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); + % else + % fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}),'w'); + % fprintf(fOUT,['#!/bin/bash\n\n%smapBack%d/%s.align > ',... + % '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); + % end + if ( multi_node_run ) fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'w'); - fprintf(fOUT,['%smapBack%d/%s.align > ',... - '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); else fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}),'w'); - fprintf(fOUT,['#!/bin/bash\n\n%smapBack%d/%s.align > ',... - '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); + fprintf(fOUT,'#!/bin/bash\n\n'); end - + fprintf(fOUT,'cat %s | /scratch/etna/master_align.sh `xargs` &\n',aliCom_name); + % Since we send to the background in a shell, makes sure the % function waits on children. %if (iTiltSeries == nTiltSeries) @@ -1801,9 +1864,11 @@ else fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}), 'a'); end - fprintf(fOUT,['%smapBack%d/%s.align > ',... - '%smapBack%d/%s.align_ta.log &\n'], ... - mbOutAlt{1:3},mbOutAlt{1:3}); + % fprintf(fOUT,['%smapBack%d/%s.align > ',... + % '%smapBack%d/%s.align_ta.log &\n'], ... + % mbOutAlt{1:3},mbOutAlt{1:3}); + fprintf(fOUT,'cat %s | /scratch/etna/master_align.sh `xargs` &\n',aliCom_name); + % Since we send to the background in a shell, makes sure the % function waits on children. %if (iTiltSeries == nTiltSeries) diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 2b5c5c89..a3bcb88d 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -610,20 +610,9 @@ % fprintf('From %d possible volumes the total weight is %3.3e\n',nVolumes,addedWeight); end avgCCC = mean(cccVect); - - - - mean(wgtVect) - std(wgtVect) + maxCCC = max(cccVect); - mean(wgtVect) - median(wgtVect) - - % figure, hist(cccVect,29) - % figure, hist(wgtVect,29) - % figure, hist((wgtVect./median(wgtVect)).^weightScale,29) - % error('asdf') - + if (emc.track_stats) fprintf('Avgerage score is %3.3f, using a quality weight of %2.2f\n\n',avgCCC,emc.flgQualityWeight); else From e401a3bf2f3dc4b59dcd0a5b218f51c93a931f7f Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 22 Mar 2024 10:08:43 -0400 Subject: [PATCH 069/151] WIP adding gpu conversions to and from half --- alignment/BH_templateSearch3d_2.m | 7 +++- testScripts/emc_halfcast.m | 55 ++++++++++++++++++++++++++----- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 66750add..eb2f6a6f 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -9,7 +9,7 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - +test_half = false; ctf3dNoSubTomoMeta = true; if length(varargin) == 1 @@ -403,6 +403,11 @@ test_local scale_mip +wanted_storage_precision = 'single'; +if (test_half) + wanted_storage_precision = 'half'; +end + tomoStack = zeros([sizeChunk,nTomograms], 'single'); if test_local localStack = zeros([sizeChunk,nTomograms], 'single'); diff --git a/testScripts/emc_halfcast.m b/testScripts/emc_halfcast.m index 5e2834c6..f4e2f2fc 100644 --- a/testScripts/emc_halfcast.m +++ b/testScripts/emc_halfcast.m @@ -1,13 +1,52 @@ -function [output_vol] = emc_halfcast(input_vol) +function [output_vol] = emc_halfcast(input_vol, swap_host_device) - if isa(input_vol, 'uint16') - output_vol = zeros(size(input_vol), 'single'); - mexFP16(output_vol, input_vol, false); - elseif isa(input_vol, 'single') - output_vol = zeros(size(input_vol), 'uint16'); + if nargin < 2 + swap_host_device = false; + end + + to_gpu = false; + to_cpu = false; + to_half = false; + + % Determin if we are going to or from half based on the input precision. + switch underlyingType(input_vol) + case 'uint16' + to_half = false; + case 'single' + to_half = true; + otherwise + error('Unknown precision'); + end + + % By default, stay on cpu or gpu and just convert type. + % If swap_host_device is true, then we will swap to the other device. + if (swap_host_device) + if isa(input_vol, 'gpuArray') + to_cpu = true; + else + to_gpu = true; + end + end + + if (to_cpu && to_gpu) + error('Cannot convert to and from GPU at the same time'); + end + + + if (to_half) + if (to_gpu) + output_vol = zeros(size(input_vol), 'uint16', 'gpuArray'); + else + output_vol = zeros(size(input_vol), 'uint16'); + end mexFP16(input_vol, output_vol, true); - else - error('Unknown precision'); + else + if (to_gpu) + output_vol = zeros(size(input_vol), 'single', 'gpuArray'); + else + output_vol = zeros(size(input_vol), 'single'); + end + mexFP16(output_vol, input_vol, false); end end \ No newline at end of file From e25bbd4d835d601153d08ad201cad527fc3457c5 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 22 Mar 2024 10:20:29 -0400 Subject: [PATCH 070/151] wip fix tracking of mexfiles --- .gitignore | 2 ++ statistics/BH_pcaPub.m | 7 ++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 2aa87e89..d7fa4a60 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ mexFiles/ +!mexFiles/*.cu +!mexFiles/*.cpp mexFiles/compiled !mexFiles/compiled/emC_autoAlign.sh bin/ diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 70e8cd18..a425bdab 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -901,12 +901,9 @@ nIgnored = nIgnored + 1; if ~(emc.Pca_randSubset) fprintf('Ignoring outside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; + subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; end - % This could also probably go in the above not random_subset clause, but also - % i don't think will change anthything downstream as it shoul dbe ignored on the full run. - % Leaving it this way means the intermediate metadata is more accurately reflecting the process. - eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; - subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; end % end of ignore if statment from extracted window out of bounds (ischar(indVAL)) if ~rem(iSubTomo,100) fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... From e3958265267d6cb59a813cfe5d7e111858603ff9 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 22 Mar 2024 10:22:52 -0400 Subject: [PATCH 071/151] Fix non-tracking of mex files --- .gitignore | 3 - mexFiles/include/ieee-754-half/ChangeLog.txt | 213 + mexFiles/include/ieee-754-half/README.txt | 317 ++ mexFiles/include/ieee-754-half/half.hpp | 4601 ++++++++++++++++++ mexFiles/logFile/emClarity.logfile | 16 + mexFiles/mexFP16.cu | 167 + 6 files changed, 5314 insertions(+), 3 deletions(-) create mode 100644 mexFiles/include/ieee-754-half/ChangeLog.txt create mode 100644 mexFiles/include/ieee-754-half/README.txt create mode 100644 mexFiles/include/ieee-754-half/half.hpp create mode 100644 mexFiles/logFile/emClarity.logfile create mode 100644 mexFiles/mexFP16.cu diff --git a/.gitignore b/.gitignore index d7fa4a60..64ed841b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,3 @@ -mexFiles/ -!mexFiles/*.cu -!mexFiles/*.cpp mexFiles/compiled !mexFiles/compiled/emC_autoAlign.sh bin/ diff --git a/mexFiles/include/ieee-754-half/ChangeLog.txt b/mexFiles/include/ieee-754-half/ChangeLog.txt new file mode 100644 index 00000000..37f3dbf0 --- /dev/null +++ b/mexFiles/include/ieee-754-half/ChangeLog.txt @@ -0,0 +1,213 @@ +Release Notes {#changelog} +============= + +2.2.0 release (2021-06-12): +--------------------------- + +- Added `rsqrt` function for inverse square root. +- Improved performance of `pow` function. +- Fixed bug that forgot to include `` for F16C intrinsics. + + +2.1.0 release (2019-08-05): +--------------------------- + +- Added detection of IEEE floating-point exceptions to operators and functions. +- Added configuration options for automatic exception handling. +- Added functions for explicitly managing floating-point exception flags. +- Improved accuracy of `pow` and `atan2` functions. + + +2.0.0 release (2019-07-23): +--------------------------- + +- Made internal implementation independent from built-in floating point + facilities for increased reliability and IEEE-conformance. +- Changed default rounding mode to rounding to nearest. +- Always round ties to even when rounding to nearest. +- Extended `constexpr` support to comparison and classification functions. +- Added support for F16C compiler intrinsics for conversions. +- Enabled C++11 feature detection for Intel compilers. + + +1.12.0 release (2017-03-06): +---------------------------- + +- Changed behaviour of `half_cast` to perform conversions to/from `double` + and `long double` directly according to specified rounding mode, without an + intermediate `float` conversion. +- Added `noexcept` specifiers to constructors. +- Fixed minor portability problem with `logb` and `ilogb`. +- Tested for *VC++ 2015*. + + +1.11.0 release (2013-11-16): +---------------------------- + +- Made tie-breaking behaviour in round to nearest configurable by + `HALF_ROUND_TIES_TO_EVEN` macro. +- Completed support for all C++11 mathematical functions even if single- + precision versions from `` are unsupported. +- Fixed inability to disable support for C++11 mathematical functions on + *VC++ 2013*. + + +1.10.0 release (2013-11-09): +---------------------------- + +- Made default rounding mode configurable by `HALF_ROUND_STYLE` macro. +- Added support for non-IEEE single-precision implementations. +- Added `HALF_ENABLE_CPP11_TYPE_TRAITS` preprocessor flag for checking + support for C++11 type traits and TMP features. +- Restricted `half_cast` to support built-in arithmetic types only. +- Changed behaviour of `half_cast` to respect rounding mode when casting + to/from integer types. + + +1.9.2 release (2013-11-01): +--------------------------- + +- Tested for *gcc 4.8*. +- Tested and fixed for *VC++ 2013*. +- Removed unnecessary warnings in *MSVC*. + + +1.9.1 release (2013-08-08): +--------------------------- + +- Fixed problems with older gcc and MSVC versions. +- Small fix to non-C++11 implementations of `remainder` and `remquo`. + + +1.9.0 release (2013-08-07): +--------------------------- + +- Changed behaviour of `nearbyint`, `rint`, `lrint` and `llrint` to use + rounding mode of half-precision implementation (which is + truncating/indeterminate) instead of single-precision rounding mode. +- Added support for more C++11 mathematical functions even if single- + precision versions from `` are unsupported, in particular + `remainder`, `remquo` and `cbrt`. +- Minor implementation changes. + + +1.8.1 release (2013-01-22): +--------------------------- + +- Fixed bug resulting in multiple definitions of the `nanh` function due to + a missing `inline` specification. + + +1.8.0 release (2013-01-19): +--------------------------- + +- Added support for more C++11 mathematical functions even if single- + precision versions from `` are unsupported, in particular + exponential and logarithm functions, hyperbolic area functions and the + hypotenuse function. +- Made `fma` function use default implementation if single-precision version + from `` is not faster and thus `FP_FAST_FMAH` to be defined always. +- Fixed overload resolution issues when invoking certain mathematical + functions by unqualified calls. + + +1.7.0 release (2012-10-26): +--------------------------- + +- Added support for C++11 `noexcept` specifiers. +- Changed C++11 `long long` to be supported on *VC++ 2003* and up. + + +1.6.1 release (2012-09-13): +--------------------------- + +- Made `fma` and `fdim` functions available even if corresponding + single-precision functions are not. + + +1.6.0 release (2012-09-12): +--------------------------- + +- Added `HALF_ENABLE_CPP11_LONG_LONG` to control support for `long long` + integers and corresponding mathematical functions. +- Fixed C++98 compatibility on non-VC compilers. + + +1.5.1 release (2012-08-17): +--------------------------- + +- Recorrected `std::numeric_limits::round_style` to always return + `std::round_indeterminate`, due to overflow-handling deviating from + correct round-toward-zero behaviour. + + +1.5.0 release (2012-08-16): +--------------------------- + +- Added `half_cast` for explicitly casting between half and any type + convertible to/from `float` and allowing the explicit specification of + the rounding mode to use. + + +1.4.0 release (2012-08-12): +--------------------------- + +- Added support for C++11 generalized constant expressions (`constexpr`). + + +1.3.1 release (2012-08-11): +--------------------------- + +- Fixed requirement for `std::signbit` and `std::isnan` (even if C++11 + `` functions disabled) on non-VC compilers. + + +1.3.0 release (2012-08-10): +--------------------------- + +- Made requirement for `` and `static_assert` optional and thus + made the library C++98-compatible. +- Made support for C++11 features user-overridable through explicit + definition of corresponding preprocessor symbols to either 0 or 1. +- Renamed `HALF_ENABLE_HASH` to `HALF_ENABLE_CPP11_HASH` in correspondence + with other C++11 preprocessor symbols. + + +1.2.0 release (2012-08-07): +--------------------------- + +- Added proper preprocessor definitions for `HUGE_VALH` and `FP_FAST_FMAH` + in correspondence with their single-precision counterparts from ``. +- Fixed internal preprocessor macros to be properly undefined after use. + + +1.1.2 release (2012-08-07): +--------------------------- + +- Revised `std::numeric_limits::round_style` to return + `std::round_toward_zero` if the `float` version also does and + `std::round_indeterminate` otherwise. +- Fixed `std::numeric_limits::round_error` to reflect worst-case round + toward zero behaviour. + + +1.1.1 release (2012-08-06): +--------------------------- + +- Fixed `std::numeric_limits::min` to return smallest positive normal + number, instead of subnormal number. +- Fixed `std::numeric_limits::round_style` to return + `std::round_indeterminate` due to mixture of separately rounded + single-precision arithmetics with truncating single-to-half conversions. + + +1.1.0 release (2012-08-06): +--------------------------- + +- Added half-precision literals. + + +1.0.0 release (2012-08-05): +--------------------------- + +- First release. diff --git a/mexFiles/include/ieee-754-half/README.txt b/mexFiles/include/ieee-754-half/README.txt new file mode 100644 index 00000000..3dd0d1c2 --- /dev/null +++ b/mexFiles/include/ieee-754-half/README.txt @@ -0,0 +1,317 @@ +HALF-PRECISION FLOATING-POINT LIBRARY (Version 2.2.0) +----------------------------------------------------- + +This is a C++ header-only library to provide an IEEE 754 conformant 16-bit +half-precision floating-point type along with corresponding arithmetic +operators, type conversions and common mathematical functions. It aims for both +efficiency and ease of use, trying to accurately mimic the behaviour of the +built-in floating-point types at the best performance possible. + + +INSTALLATION AND REQUIREMENTS +----------------------------- + +Conveniently, the library consists of just a single header file containing all +the functionality, which can be directly included by your projects, without the +neccessity to build anything or link to anything. + +Whereas this library is fully C++98-compatible, it can profit from certain +C++11 features. Support for those features is checked automatically at compile +(or rather preprocessing) time, but can be explicitly enabled or disabled by +predefining the corresponding preprocessor symbols to either 1 or 0 yourself +before including half.hpp. This is useful when the automatic detection fails +(for more exotic implementations) or when a feature should be explicitly +disabled: + + - 'long long' integer type for mathematical functions returning 'long long' + results (enabled for VC++ 2003 and icc 11.1 and newer, gcc and clang, + overridable with 'HALF_ENABLE_CPP11_LONG_LONG'). + + - Static assertions for extended compile-time checks (enabled for VC++ 2010, + gcc 4.3, clang 2.9, icc 11.1 and newer, overridable with + 'HALF_ENABLE_CPP11_STATIC_ASSERT'). + + - Generalized constant expressions (enabled for VC++ 2015, gcc 4.6, clang 3.1, + icc 14.0 and newer, overridable with 'HALF_ENABLE_CPP11_CONSTEXPR'). + + - noexcept exception specifications (enabled for VC++ 2015, gcc 4.6, + clang 3.0, icc 14.0 and newer, overridable with 'HALF_ENABLE_CPP11_NOEXCEPT'). + + - User-defined literals for half-precision literals to work (enabled for + VC++ 2015, gcc 4.7, clang 3.1, icc 15.0 and newer, overridable with + 'HALF_ENABLE_CPP11_USER_LITERALS'). + + - Thread-local storage for per-thread floating-point exception flags (enabled + for VC++ 2015, gcc 4.8, clang 3.3, icc 15.0 and newer, overridable with + 'HALF_ENABLE_CPP11_THREAD_LOCAL'). + + - Type traits and template meta-programming features from + (enabled for VC++ 2010, libstdc++ 4.3, libc++ and newer, overridable with + 'HALF_ENABLE_CPP11_TYPE_TRAITS'). + + - Special integer types from (enabled for VC++ 2010, libstdc++ 4.3, + libc++ and newer, overridable with 'HALF_ENABLE_CPP11_CSTDINT'). + + - Certain C++11 single-precision mathematical functions from for + floating-point classification during conversions from higher precision types + (enabled for VC++ 2013, libstdc++ 4.3, libc++ and newer, overridable with + 'HALF_ENABLE_CPP11_CMATH'). + + - Floating-point environment control from for possible exception + propagation to the built-in floating-point platform (enabled for VC++ 2013, + libstdc++ 4.3, libc++ and newer, overridable with 'HALF_ENABLE_CPP11_CFENV'). + + - Hash functor 'std::hash' from (enabled for VC++ 2010, + libstdc++ 4.3, libc++ and newer, overridable with 'HALF_ENABLE_CPP11_HASH'). + +The library has been tested successfully with Visual C++ 2005-2015, gcc 4-8 +and clang 3-8 on 32- and 64-bit x86 systems. Please contact me if you have any +problems, suggestions or even just success testing it on other platforms. + + +DOCUMENTATION +------------- + +What follows are some general words about the usage of the library and its +implementation. For a complete documentation of its interface consult the +corresponding website http://half.sourceforge.net. You may also generate the +complete developer documentation from the library's only include file's doxygen +comments, but this is more relevant to developers rather than mere users. + +BASIC USAGE + +To make use of the library just include its only header file half.hpp, which +defines all half-precision functionality inside the 'half_float' namespace. The +actual 16-bit half-precision data type is represented by the 'half' type, which +uses the standard IEEE representation with 1 sign bit, 5 exponent bits and 11 +mantissa bits (including the hidden bit) and supports all types of special +values, like subnormal values, infinity and NaNs. This type behaves like the +built-in floating-point types as much as possible, supporting the usual +arithmetic, comparison and streaming operators, which makes its use pretty +straight-forward: + + using half_float::half; + half a(3.4), b(5); + half c = a * b; + c += 3; + if(c > a) + std::cout << c << std::endl; + +Additionally the 'half_float' namespace also defines half-precision versions +for all mathematical functions of the C++ standard library, which can be used +directly through ADL: + + half a(-3.14159); + half s = sin(abs(a)); + long l = lround(s); + +You may also specify explicit half-precision literals, since the library +provides a user-defined literal inside the 'half_float::literal' namespace, +which you just need to import (assuming support for C++11 user-defined literals): + + using namespace half_float::literal; + half x = 1.0_h; + +Furthermore the library provides proper specializations for +'std::numeric_limits', defining various implementation properties, and +'std::hash' for hashing half-precision numbers (assuming support for C++11 +'std::hash'). Similar to the corresponding preprocessor symbols from +the library also defines the 'HUGE_VALH' constant and maybe the 'FP_FAST_FMAH' +symbol. + +CONVERSIONS AND ROUNDING + +The half is explicitly constructible/convertible from a single-precision float +argument. Thus it is also explicitly constructible/convertible from any type +implicitly convertible to float, but constructing it from types like double or +int will involve the usual warnings arising when implicitly converting those to +float because of the lost precision. On the one hand those warnings are +intentional, because converting those types to half neccessarily also reduces +precision. But on the other hand they are raised for explicit conversions from +those types, when the user knows what he is doing. So if those warnings keep +bugging you, then you won't get around first explicitly converting to float +before converting to half, or use the 'half_cast' described below. In addition +you can also directly assign float values to halfs. + +In contrast to the float-to-half conversion, which reduces precision, the +conversion from half to float (and thus to any other type implicitly +convertible from float) is implicit, because all values represetable with +half-precision are also representable with single-precision. This way the +half-to-float conversion behaves similar to the builtin float-to-double +conversion and all arithmetic expressions involving both half-precision and +single-precision arguments will be of single-precision type. This way you can +also directly use the mathematical functions of the C++ standard library, +though in this case you will invoke the single-precision versions which will +also return single-precision values, which is (even if maybe performing the +exact same computation, see below) not as conceptually clean when working in a +half-precision environment. + +The default rounding mode for conversions between half and more precise types +as well as for rounding results of arithmetic operations and mathematical +functions rounds to the nearest representable value. But by predefining the +'HALF_ROUND_STYLE' preprocessor symbol this default can be overridden with one +of the other standard rounding modes using their respective constants or the +equivalent values of 'std::float_round_style' (it can even be synchronized with +the built-in single-precision implementation by defining it to +'std::numeric_limits::round_style'): + + - 'std::round_indeterminate' (-1) for the fastest rounding. + + - 'std::round_toward_zero' (0) for rounding toward zero. + + - 'std::round_to_nearest' (1) for rounding to the nearest value (default). + + - 'std::round_toward_infinity' (2) for rounding toward positive infinity. + + - 'std::round_toward_neg_infinity' (3) for rounding toward negative infinity. + +In addition to changing the overall default rounding mode one can also use the +'half_cast'. This converts between half and any built-in arithmetic type using +a configurable rounding mode (or the default rounding mode if none is +specified). In addition to a configurable rounding mode, 'half_cast' has +another big difference to a mere 'static_cast': Any conversions are performed +directly using the given rounding mode, without any intermediate conversion +to/from 'float'. This is especially relevant for conversions to integer types, +which don't necessarily truncate anymore. But also for conversions from +'double' or 'long double' this may produce more precise results than a +pre-conversion to 'float' using the single-precision implementation's current +rounding mode would. + + half a = half_cast(4.2); + half b = half_cast::round_style>(4.2f); + assert( half_cast( 0.7_h ) == 1 ); + assert( half_cast( 4097 ) == 4096.0_h ); + assert( half_cast( 4097 ) == 4100.0_h ); + assert( half_cast( std::numeric_limits::min() ) > 0.0_h ); + +ACCURACY AND PERFORMANCE + +From version 2.0 onward the library is implemented without employing the +underlying floating-point implementation of the system (except for conversions, +of course), providing an entirely self-contained half-precision implementation +with results independent from the system's existing single- or double-precision +implementation and its rounding behaviour. + +As to accuracy, many of the operators and functions provided by this library +are exact to rounding for all rounding modes, i.e. the error to the exact +result is at most 0.5 ULP (unit in the last place) for rounding to nearest and +less than 1 ULP for all other rounding modes. This holds for all the operations +required by the IEEE 754 standard and many more. Specifically the following +functions might exhibit a deviation from the correctly rounded exact result by +1 ULP for a select few input values: 'expm1', 'log1p', 'pow', 'atan2', 'erf', +'erfc', 'lgamma', 'tgamma' (for more details see the documentation of the +individual functions). All other functions and operators are always exact to +rounding or independent of the rounding mode altogether. + +The increased IEEE-conformance and cleanliness of this implementation comes +with a certain performance cost compared to doing computations and mathematical +functions in hardware-accelerated single-precision. On average and depending on +the platform, the arithemtic operators are about 75% as fast and the +mathematical functions about 33-50% as fast as performing the corresponding +operations in single-precision and converting between the inputs and outputs. +However, directly computing with half-precision values is a rather rare +use-case and usually using actual 'float' values for all computations and +temproraries and using 'half's only for storage is the recommended way. But +nevertheless the goal of this library was to provide a complete and +conceptually clean IEEE-confromant half-precision implementation and in the few +cases when you do need to compute directly in half-precision you do so for a +reason and want accurate results. + +If necessary, this internal implementation can be overridden by predefining the +'HALF_ARITHMETIC_TYPE' preprocessor symbol to one of the built-in +floating-point types ('float', 'double' or 'long double'), which will cause the +library to use this type for computing arithmetic operations and mathematical +functions (if available). However, due to using the platform's floating-point +implementation (and its rounding behaviour) internally, this might cause +results to deviate from the specified half-precision rounding mode. It will of +course also inhibit the automatic exception detection described below. + +The conversion operations between half-precision and single-precision types can +also make use of the F16C extension for x86 processors by using the +corresponding compiler intrinsics from . Support for this is +checked at compile-time by looking for the '__F16C__' macro which at least gcc +and clang define based on the target platform. It can also be enabled manually +by predefining the 'HALF_ENABLE_F16C_INTRINSICS' preprocessor symbol to 1, or 0 +for explicitly disabling it. However, this will directly use the corresponding +intrinsics for conversion without checking if they are available at runtime +(possibly crashing if they are not), so make sure they are supported on the +target platform before enabling this. + +EXCEPTION HANDLING + +The half-precision implementation supports all 5 required floating-point +exceptions from the IEEE standard to indicate erroneous inputs or inexact +results during operations. These are represented by exception flags which +actually use the same values as the corresponding 'FE_...' flags defined in +C++11's header if supported, specifically: + + - 'FE_INVALID' for invalid inputs to an operation. + - 'FE_DIVBYZERO' for finite inputs producing infinite results. + - 'FE_OVERFLOW' if a result is too large to represent finitely. + - 'FE_UNDERFLOW' for a subnormal or zero result after rounding. + - 'FE_INEXACT' if a result needed rounding to be representable. + - 'FE_ALL_EXCEPT' as a convenient OR of all possible exception flags. + +The internal exception flag state will start with all flags cleared and is +maintained per thread if C++11 thread-local storage is supported, otherwise it +will be maintained globally and will theoretically NOT be thread-safe (while +practically being as thread-safe as a simple integer variable can be). These +flags can be managed explicitly using the library's error handling functions, +which again try to mimic the built-in functions for handling floating-point +exceptions from . You can clear them with 'feclearexcept' (which is the +only way a flag can be cleared), test them with 'fetestexcept', explicitly +raise errors with 'feraiseexcept' and save and restore their state using +'fegetexceptflag' and 'fesetexceptflag'. You can also throw corresponding C++ +exceptions based on the current flag state using 'fethrowexcept'. + +However, any automatic exception detection and handling during half-precision +operations and functions is DISABLED by default, since it comes with a minor +performance overhead due to runtime checks, and reacting to IEEE floating-point +exceptions is rarely ever needed in application code. But the library fully +supports IEEE-conformant detection of floating-point exceptions and various +ways for handling them, which can be enabled by pre-defining the corresponding +preprocessor symbols to 1. They can be enabled individually or all at once and +they will be processed in the order they are listed here: + + - 'HALF_ERRHANDLING_FLAGS' sets the internal exception flags described above + whenever the corresponding exception occurs. + - 'HALF_ERRHANDLING_ERRNO' sets the value of 'errno' from similar to + the behaviour of the built-in floating-point types when 'MATH_ERRNO' is used. + - 'HALF_ERRHANDLING_FENV' will propagate exceptions to the built-in + floating-point implementation using 'std::feraiseexcept' if support for + C++11 floating-point control is enabled. However, this does not synchronize + exceptions: neither will clearing propagate nor will it work in reverse. + - 'HALF_ERRHANDLING_THROW_...' can be defined to a string literal which will + be used as description message for a C++ exception that is thrown whenever + a 'FE_...' exception occurs, similar to the behaviour of 'fethrowexcept'. + +If any of the above error handling is activated, non-quiet operations on +half-precision values will also raise a 'FE_INVALID' exception whenever +they encounter a signaling NaN value, in addition to transforming the value +into a quiet NaN. If error handling is disabled, signaling NaNs will be +treated like quiet NaNs (while still getting explicitly quieted if propagated +to the result). There can also be additional treatment of overflow and +underflow errors after they have been processed as above, which is ENABLED by +default (but of course only takes effect if any other exception handling is +activated) unless overridden by pre-defining the corresponding preprocessor +symbol to 0: + + - 'HALF_ERRHANDLING_OVERFLOW_TO_INEXACT' will cause overflow errors to also + raise a 'FE_INEXACT' exception. + - 'HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT' will cause underflow errors to also + raise a 'FE_INEXACT' exception. This will also slightly change the + behaviour of the underflow exception, which will ONLY be raised if the + result is actually inexact due to underflow. If this is disabled, underflow + exceptions will be raised for ANY (possibly exact) subnormal result. + + +CREDITS AND CONTACT +------------------- + +This library is developed by CHRISTIAN RAU and released under the MIT License +(see LICENSE.txt). If you have any questions or problems with it, feel free to +contact me at rauy@users.sourceforge.net. + +Additional credit goes to JEROEN VAN DER ZIJP for his paper on "Fast Half Float +Conversions", whose algorithms have been used in the library for converting +between half-precision and single-precision values. diff --git a/mexFiles/include/ieee-754-half/half.hpp b/mexFiles/include/ieee-754-half/half.hpp new file mode 100644 index 00000000..f4d86146 --- /dev/null +++ b/mexFiles/include/ieee-754-half/half.hpp @@ -0,0 +1,4601 @@ +// half - IEEE 754-based half-precision floating-point library. +// +// Copyright (c) 2012-2021 Christian Rau +// +// Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation +// files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, +// modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +// WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +// ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +// Version 2.2.0 + +/// \file +/// Main header file for half-precision functionality. + +#ifndef HALF_HALF_HPP +#define HALF_HALF_HPP + +#define HALF_GCC_VERSION (__GNUC__*100+__GNUC_MINOR__) + +#if defined(__INTEL_COMPILER) + #define HALF_ICC_VERSION __INTEL_COMPILER +#elif defined(__ICC) + #define HALF_ICC_VERSION __ICC +#elif defined(__ICL) + #define HALF_ICC_VERSION __ICL +#else + #define HALF_ICC_VERSION 0 +#endif + +// check C++11 language features +#if defined(__clang__) // clang + #if __has_feature(cxx_static_assert) && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if __has_feature(cxx_constexpr) && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if __has_feature(cxx_noexcept) && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if __has_feature(cxx_user_literals) && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if __has_feature(cxx_thread_local) && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) + #define HALF_ENABLE_CPP11_THREAD_LOCAL 1 + #endif + #if (defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L) && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif +#elif HALF_ICC_VERSION && defined(__INTEL_CXX11_MODE__) // Intel C++ + #if HALF_ICC_VERSION >= 1500 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) + #define HALF_ENABLE_CPP11_THREAD_LOCAL 1 + #endif + #if HALF_ICC_VERSION >= 1500 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if HALF_ICC_VERSION >= 1400 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if HALF_ICC_VERSION >= 1400 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if HALF_ICC_VERSION >= 1110 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if HALF_ICC_VERSION >= 1110 && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif +#elif defined(__GNUC__) // gcc + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103L + #if HALF_GCC_VERSION >= 408 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) + #define HALF_ENABLE_CPP11_THREAD_LOCAL 1 + #endif + #if HALF_GCC_VERSION >= 407 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if HALF_GCC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if HALF_GCC_VERSION >= 406 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif + #endif + #define HALF_TWOS_COMPLEMENT_INT 1 +#elif defined(_MSC_VER) // Visual C++ + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_THREAD_LOCAL) + #define HALF_ENABLE_CPP11_THREAD_LOCAL 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_USER_LITERALS) + #define HALF_ENABLE_CPP11_USER_LITERALS 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_CONSTEXPR) + #define HALF_ENABLE_CPP11_CONSTEXPR 1 + #endif + #if _MSC_VER >= 1900 && !defined(HALF_ENABLE_CPP11_NOEXCEPT) + #define HALF_ENABLE_CPP11_NOEXCEPT 1 + #endif + #if _MSC_VER >= 1600 && !defined(HALF_ENABLE_CPP11_STATIC_ASSERT) + #define HALF_ENABLE_CPP11_STATIC_ASSERT 1 + #endif + #if _MSC_VER >= 1310 && !defined(HALF_ENABLE_CPP11_LONG_LONG) + #define HALF_ENABLE_CPP11_LONG_LONG 1 + #endif + #define HALF_TWOS_COMPLEMENT_INT 1 + #define HALF_POP_WARNINGS 1 + #pragma warning(push) + #pragma warning(disable : 4099 4127 4146) //struct vs class, constant in if, negative unsigned +#endif + +// check C++11 library features +#include +#if defined(_LIBCPP_VERSION) // libc++ + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 + #ifndef HALF_ENABLE_CPP11_TYPE_TRAITS + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #ifndef HALF_ENABLE_CPP11_CSTDINT + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #ifndef HALF_ENABLE_CPP11_CMATH + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #ifndef HALF_ENABLE_CPP11_HASH + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #ifndef HALF_ENABLE_CPP11_CFENV + #define HALF_ENABLE_CPP11_CFENV 1 + #endif + #endif +#elif defined(__GLIBCXX__) // libstdc++ + #if defined(__GXX_EXPERIMENTAL_CXX0X__) || __cplusplus >= 201103 + #ifdef __clang__ + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #if __GLIBCXX__ >= 20080606 && !defined(HALF_ENABLE_CPP11_CFENV) + #define HALF_ENABLE_CPP11_CFENV 1 + #endif + #else + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #if HALF_GCC_VERSION >= 403 && !defined(HALF_ENABLE_CPP11_CFENV) + #define HALF_ENABLE_CPP11_CFENV 1 + #endif + #endif + #endif +#elif defined(_CPPLIB_VER) // Dinkumware/Visual C++ + #if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_TYPE_TRAITS) + #define HALF_ENABLE_CPP11_TYPE_TRAITS 1 + #endif + #if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_CSTDINT) + #define HALF_ENABLE_CPP11_CSTDINT 1 + #endif + #if _CPPLIB_VER >= 520 && !defined(HALF_ENABLE_CPP11_HASH) + #define HALF_ENABLE_CPP11_HASH 1 + #endif + #if _CPPLIB_VER >= 610 && !defined(HALF_ENABLE_CPP11_CMATH) + #define HALF_ENABLE_CPP11_CMATH 1 + #endif + #if _CPPLIB_VER >= 610 && !defined(HALF_ENABLE_CPP11_CFENV) + #define HALF_ENABLE_CPP11_CFENV 1 + #endif +#endif +#undef HALF_GCC_VERSION +#undef HALF_ICC_VERSION + +// any error throwing C++ exceptions? +#if defined(HALF_ERRHANDLING_THROW_INVALID) || defined(HALF_ERRHANDLING_THROW_DIVBYZERO) || defined(HALF_ERRHANDLING_THROW_OVERFLOW) || defined(HALF_ERRHANDLING_THROW_UNDERFLOW) || defined(HALF_ERRHANDLING_THROW_INEXACT) +#define HALF_ERRHANDLING_THROWS 1 +#endif + +// any error handling enabled? +#define HALF_ERRHANDLING (HALF_ERRHANDLING_FLAGS||HALF_ERRHANDLING_ERRNO||HALF_ERRHANDLING_FENV||HALF_ERRHANDLING_THROWS) + +#if HALF_ERRHANDLING + #define HALF_UNUSED_NOERR(name) name +#else + #define HALF_UNUSED_NOERR(name) +#endif + +// support constexpr +#if HALF_ENABLE_CPP11_CONSTEXPR + #define HALF_CONSTEXPR constexpr + #define HALF_CONSTEXPR_CONST constexpr + #if HALF_ERRHANDLING + #define HALF_CONSTEXPR_NOERR + #else + #define HALF_CONSTEXPR_NOERR constexpr + #endif +#else + #define HALF_CONSTEXPR + #define HALF_CONSTEXPR_CONST const + #define HALF_CONSTEXPR_NOERR +#endif + +// support noexcept +#if HALF_ENABLE_CPP11_NOEXCEPT + #define HALF_NOEXCEPT noexcept + #define HALF_NOTHROW noexcept +#else + #define HALF_NOEXCEPT + #define HALF_NOTHROW throw() +#endif + +// support thread storage +#if HALF_ENABLE_CPP11_THREAD_LOCAL + #define HALF_THREAD_LOCAL thread_local +#else + #define HALF_THREAD_LOCAL static +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if HALF_ENABLE_CPP11_TYPE_TRAITS + #include +#endif +#if HALF_ENABLE_CPP11_CSTDINT + #include +#endif +#if HALF_ERRHANDLING_ERRNO + #include +#endif +#if HALF_ENABLE_CPP11_CFENV + #include +#endif +#if HALF_ENABLE_CPP11_HASH + #include +#endif + + +#ifndef HALF_ENABLE_F16C_INTRINSICS + /// Enable F16C intruction set intrinsics. + /// Defining this to 1 enables the use of [F16C compiler intrinsics](https://en.wikipedia.org/wiki/F16C) for converting between + /// half-precision and single-precision values which may result in improved performance. This will not perform additional checks + /// for support of the F16C instruction set, so an appropriate target platform is required when enabling this feature. + /// + /// Unless predefined it will be enabled automatically when the `__F16C__` symbol is defined, which some compilers do on supporting platforms. + #define HALF_ENABLE_F16C_INTRINSICS __F16C__ +#endif +#if HALF_ENABLE_F16C_INTRINSICS + #include +#endif + +#ifdef HALF_DOXYGEN_ONLY +/// Type for internal floating-point computations. +/// This can be predefined to a built-in floating-point type (`float`, `double` or `long double`) to override the internal +/// half-precision implementation to use this type for computing arithmetic operations and mathematical function (if available). +/// This can result in improved performance for arithmetic operators and mathematical functions but might cause results to +/// deviate from the specified half-precision rounding mode and inhibits proper detection of half-precision exceptions. +#define HALF_ARITHMETIC_TYPE (undefined) + +/// Enable internal exception flags. +/// Defining this to 1 causes operations on half-precision values to raise internal floating-point exception flags according to +/// the IEEE 754 standard. These can then be cleared and checked with clearexcept(), testexcept(). +#define HALF_ERRHANDLING_FLAGS 0 + +/// Enable exception propagation to `errno`. +/// Defining this to 1 causes operations on half-precision values to propagate floating-point exceptions to +/// [errno](https://en.cppreference.com/w/cpp/error/errno) from ``. Specifically this will propagate domain errors as +/// [EDOM](https://en.cppreference.com/w/cpp/error/errno_macros) and pole, overflow and underflow errors as +/// [ERANGE](https://en.cppreference.com/w/cpp/error/errno_macros). Inexact errors won't be propagated. +#define HALF_ERRHANDLING_ERRNO 0 + +/// Enable exception propagation to built-in floating-point platform. +/// Defining this to 1 causes operations on half-precision values to propagate floating-point exceptions to the built-in +/// single- and double-precision implementation's exception flags using the +/// [C++11 floating-point environment control](https://en.cppreference.com/w/cpp/numeric/fenv) from ``. However, this +/// does not work in reverse and single- or double-precision exceptions will not raise the corresponding half-precision +/// exception flags, nor will explicitly clearing flags clear the corresponding built-in flags. +#define HALF_ERRHANDLING_FENV 0 + +/// Throw C++ exception on domain errors. +/// Defining this to a string literal causes operations on half-precision values to throw a +/// [std::domain_error](https://en.cppreference.com/w/cpp/error/domain_error) with the specified message on domain errors. +#define HALF_ERRHANDLING_THROW_INVALID (undefined) + +/// Throw C++ exception on pole errors. +/// Defining this to a string literal causes operations on half-precision values to throw a +/// [std::domain_error](https://en.cppreference.com/w/cpp/error/domain_error) with the specified message on pole errors. +#define HALF_ERRHANDLING_THROW_DIVBYZERO (undefined) + +/// Throw C++ exception on overflow errors. +/// Defining this to a string literal causes operations on half-precision values to throw a +/// [std::overflow_error](https://en.cppreference.com/w/cpp/error/overflow_error) with the specified message on overflows. +#define HALF_ERRHANDLING_THROW_OVERFLOW (undefined) + +/// Throw C++ exception on underflow errors. +/// Defining this to a string literal causes operations on half-precision values to throw a +/// [std::underflow_error](https://en.cppreference.com/w/cpp/error/underflow_error) with the specified message on underflows. +#define HALF_ERRHANDLING_THROW_UNDERFLOW (undefined) + +/// Throw C++ exception on rounding errors. +/// Defining this to 1 causes operations on half-precision values to throw a +/// [std::range_error](https://en.cppreference.com/w/cpp/error/range_error) with the specified message on general rounding errors. +#define HALF_ERRHANDLING_THROW_INEXACT (undefined) +#endif + +#ifndef HALF_ERRHANDLING_OVERFLOW_TO_INEXACT +/// Raise INEXACT exception on overflow. +/// Defining this to 1 (default) causes overflow errors to automatically raise inexact exceptions in addition. +/// These will be raised after any possible handling of the underflow exception. +#define HALF_ERRHANDLING_OVERFLOW_TO_INEXACT 1 +#endif + +#ifndef HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT +/// Raise INEXACT exception on underflow. +/// Defining this to 1 (default) causes underflow errors to automatically raise inexact exceptions in addition. +/// These will be raised after any possible handling of the underflow exception. +/// +/// **Note:** This will actually cause underflow (and the accompanying inexact) exceptions to be raised *only* when the result +/// is inexact, while if disabled bare underflow errors will be raised for *any* (possibly exact) subnormal result. +#define HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT 1 +#endif + +/// Default rounding mode. +/// This specifies the rounding mode used for all conversions between [half](\ref half_float::half)s and more precise types +/// (unless using half_cast() and specifying the rounding mode directly) as well as in arithmetic operations and mathematical +/// functions. It can be redefined (before including half.hpp) to one of the standard rounding modes using their respective +/// constants or the equivalent values of +/// [std::float_round_style](https://en.cppreference.com/w/cpp/types/numeric_limits/float_round_style): +/// +/// `std::float_round_style` | value | rounding +/// ---------------------------------|-------|------------------------- +/// `std::round_indeterminate` | -1 | fastest +/// `std::round_toward_zero` | 0 | toward zero +/// `std::round_to_nearest` | 1 | to nearest (default) +/// `std::round_toward_infinity` | 2 | toward positive infinity +/// `std::round_toward_neg_infinity` | 3 | toward negative infinity +/// +/// By default this is set to `1` (`std::round_to_nearest`), which rounds results to the nearest representable value. It can even +/// be set to [std::numeric_limits::round_style](https://en.cppreference.com/w/cpp/types/numeric_limits/round_style) to synchronize +/// the rounding mode with that of the built-in single-precision implementation (which is likely `std::round_to_nearest`, though). +#ifndef HALF_ROUND_STYLE + #define HALF_ROUND_STYLE 1 // = std::round_to_nearest +#endif + +/// Value signaling overflow. +/// In correspondence with `HUGE_VAL[F|L]` from `` this symbol expands to a positive value signaling the overflow of an +/// operation, in particular it just evaluates to positive infinity. +/// +/// **See also:** Documentation for [HUGE_VAL](https://en.cppreference.com/w/cpp/numeric/math/HUGE_VAL) +#define HUGE_VALH std::numeric_limits::infinity() + +/// Fast half-precision fma function. +/// This symbol is defined if the fma() function generally executes as fast as, or faster than, a separate +/// half-precision multiplication followed by an addition, which is always the case. +/// +/// **See also:** Documentation for [FP_FAST_FMA](https://en.cppreference.com/w/cpp/numeric/math/fma) +#define FP_FAST_FMAH 1 + +/// Half rounding mode. +/// In correspondence with `FLT_ROUNDS` from `` this symbol expands to the rounding mode used for +/// half-precision operations. It is an alias for [HALF_ROUND_STYLE](\ref HALF_ROUND_STYLE). +/// +/// **See also:** Documentation for [FLT_ROUNDS](https://en.cppreference.com/w/cpp/types/climits/FLT_ROUNDS) +#define HLF_ROUNDS HALF_ROUND_STYLE + +#ifndef FP_ILOGB0 + #define FP_ILOGB0 INT_MIN +#endif +#ifndef FP_ILOGBNAN + #define FP_ILOGBNAN INT_MAX +#endif +#ifndef FP_SUBNORMAL + #define FP_SUBNORMAL 0 +#endif +#ifndef FP_ZERO + #define FP_ZERO 1 +#endif +#ifndef FP_NAN + #define FP_NAN 2 +#endif +#ifndef FP_INFINITE + #define FP_INFINITE 3 +#endif +#ifndef FP_NORMAL + #define FP_NORMAL 4 +#endif + +#if !HALF_ENABLE_CPP11_CFENV && !defined(FE_ALL_EXCEPT) + #define FE_INVALID 0x10 + #define FE_DIVBYZERO 0x08 + #define FE_OVERFLOW 0x04 + #define FE_UNDERFLOW 0x02 + #define FE_INEXACT 0x01 + #define FE_ALL_EXCEPT (FE_INVALID|FE_DIVBYZERO|FE_OVERFLOW|FE_UNDERFLOW|FE_INEXACT) +#endif + + +/// Main namespace for half-precision functionality. +/// This namespace contains all the functionality provided by the library. +namespace half_float +{ + class half; + +#if HALF_ENABLE_CPP11_USER_LITERALS + /// Library-defined half-precision literals. + /// Import this namespace to enable half-precision floating-point literals: + /// ~~~~{.cpp} + /// using namespace half_float::literal; + /// half_float::half = 4.2_h; + /// ~~~~ + namespace literal + { + half operator "" _h(long double); + } +#endif + + /// \internal + /// \brief Implementation details. + namespace detail + { + #if HALF_ENABLE_CPP11_TYPE_TRAITS + /// Conditional type. + template struct conditional : std::conditional {}; + + /// Helper for tag dispatching. + template struct bool_type : std::integral_constant {}; + using std::true_type; + using std::false_type; + + /// Type traits for floating-point types. + template struct is_float : std::is_floating_point {}; + #else + /// Conditional type. + template struct conditional { typedef T type; }; + template struct conditional { typedef F type; }; + + /// Helper for tag dispatching. + template struct bool_type {}; + typedef bool_type true_type; + typedef bool_type false_type; + + /// Type traits for floating-point types. + template struct is_float : false_type {}; + template struct is_float : is_float {}; + template struct is_float : is_float {}; + template struct is_float : is_float {}; + template<> struct is_float : true_type {}; + template<> struct is_float : true_type {}; + template<> struct is_float : true_type {}; + #endif + + /// Type traits for floating-point bits. + template struct bits { typedef unsigned char type; }; + template struct bits : bits {}; + template struct bits : bits {}; + template struct bits : bits {}; + + #if HALF_ENABLE_CPP11_CSTDINT + /// Unsigned integer of (at least) 16 bits width. + typedef std::uint_least16_t uint16; + + /// Fastest unsigned integer of (at least) 32 bits width. + typedef std::uint_fast32_t uint32; + + /// Fastest signed integer of (at least) 32 bits width. + typedef std::int_fast32_t int32; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits { typedef std::uint_least32_t type; }; + + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { typedef std::uint_least64_t type; }; + #else + /// Unsigned integer of (at least) 16 bits width. + typedef unsigned short uint16; + + /// Fastest unsigned integer of (at least) 32 bits width. + typedef unsigned long uint32; + + /// Fastest unsigned integer of (at least) 32 bits width. + typedef long int32; + + /// Unsigned integer of (at least) 32 bits width. + template<> struct bits : conditional::digits>=32,unsigned int,unsigned long> {}; + + #if HALF_ENABLE_CPP11_LONG_LONG + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits : conditional::digits>=64,unsigned long,unsigned long long> {}; + #else + /// Unsigned integer of (at least) 64 bits width. + template<> struct bits { typedef unsigned long type; }; + #endif + #endif + + #ifdef HALF_ARITHMETIC_TYPE + /// Type to use for arithmetic computations and mathematic functions internally. + typedef HALF_ARITHMETIC_TYPE internal_t; + #endif + + /// Tag type for binary construction. + struct binary_t {}; + + /// Tag for binary construction. + HALF_CONSTEXPR_CONST binary_t binary = binary_t(); + + /// \name Implementation defined classification and arithmetic + /// \{ + + /// Check for infinity. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if infinity + /// \retval false else + template bool builtin_isinf(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::isinf(arg); + #elif defined(_MSC_VER) + return !::_finite(static_cast(arg)) && !::_isnan(static_cast(arg)); + #else + return arg == std::numeric_limits::infinity() || arg == -std::numeric_limits::infinity(); + #endif + } + + /// Check for NaN. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if not a number + /// \retval false else + template bool builtin_isnan(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::isnan(arg); + #elif defined(_MSC_VER) + return ::_isnan(static_cast(arg)) != 0; + #else + return arg != arg; + #endif + } + + /// Check sign. + /// \tparam T argument type (builtin floating-point type) + /// \param arg value to query + /// \retval true if signbit set + /// \retval false else + template bool builtin_signbit(T arg) + { + #if HALF_ENABLE_CPP11_CMATH + return std::signbit(arg); + #else + return arg < T() || (arg == T() && T(1)/arg < T()); + #endif + } + + /// Platform-independent sign mask. + /// \param arg integer value in two's complement + /// \retval -1 if \a arg negative + /// \retval 0 if \a arg positive + inline uint32 sign_mask(uint32 arg) + { + static const int N = std::numeric_limits::digits - 1; + #if HALF_TWOS_COMPLEMENT_INT + return static_cast(arg) >> N; + #else + return -((arg>>N)&1); + #endif + } + + /// Platform-independent arithmetic right shift. + /// \param arg integer value in two's complement + /// \param i shift amount (at most 31) + /// \return \a arg right shifted for \a i bits with possible sign extension + inline uint32 arithmetic_shift(uint32 arg, int i) + { + #if HALF_TWOS_COMPLEMENT_INT + return static_cast(arg) >> i; + #else + return static_cast(arg)/(static_cast(1)<>(std::numeric_limits::digits-1))&1); + #endif + } + + /// \} + /// \name Error handling + /// \{ + + /// Internal exception flags. + /// \return reference to global exception flags + inline int& errflags() { HALF_THREAD_LOCAL int flags = 0; return flags; } + + /// Raise floating-point exception. + /// \param flags exceptions to raise + /// \param cond condition to raise exceptions for + inline void raise(int HALF_UNUSED_NOERR(flags), bool HALF_UNUSED_NOERR(cond) = true) + { + #if HALF_ERRHANDLING + if(!cond) + return; + #if HALF_ERRHANDLING_FLAGS + errflags() |= flags; + #endif + #if HALF_ERRHANDLING_ERRNO + if(flags & FE_INVALID) + errno = EDOM; + else if(flags & (FE_DIVBYZERO|FE_OVERFLOW|FE_UNDERFLOW)) + errno = ERANGE; + #endif + #if HALF_ERRHANDLING_FENV && HALF_ENABLE_CPP11_CFENV + std::feraiseexcept(flags); + #endif + #ifdef HALF_ERRHANDLING_THROW_INVALID + if(flags & FE_INVALID) + throw std::domain_error(HALF_ERRHANDLING_THROW_INVALID); + #endif + #ifdef HALF_ERRHANDLING_THROW_DIVBYZERO + if(flags & FE_DIVBYZERO) + throw std::domain_error(HALF_ERRHANDLING_THROW_DIVBYZERO); + #endif + #ifdef HALF_ERRHANDLING_THROW_OVERFLOW + if(flags & FE_OVERFLOW) + throw std::overflow_error(HALF_ERRHANDLING_THROW_OVERFLOW); + #endif + #ifdef HALF_ERRHANDLING_THROW_UNDERFLOW + if(flags & FE_UNDERFLOW) + throw std::underflow_error(HALF_ERRHANDLING_THROW_UNDERFLOW); + #endif + #ifdef HALF_ERRHANDLING_THROW_INEXACT + if(flags & FE_INEXACT) + throw std::range_error(HALF_ERRHANDLING_THROW_INEXACT); + #endif + #if HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT + if((flags & FE_UNDERFLOW) && !(flags & FE_INEXACT)) + raise(FE_INEXACT); + #endif + #if HALF_ERRHANDLING_OVERFLOW_TO_INEXACT + if((flags & FE_OVERFLOW) && !(flags & FE_INEXACT)) + raise(FE_INEXACT); + #endif + #endif + } + + /// Check and signal for any NaN. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \retval true if either \a x or \a y is NaN + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool compsignal(unsigned int x, unsigned int y) + { + #if HALF_ERRHANDLING + raise(FE_INVALID, (x&0x7FFF)>0x7C00 || (y&0x7FFF)>0x7C00); + #endif + return (x&0x7FFF) > 0x7C00 || (y&0x7FFF) > 0x7C00; + } + + /// Signal and silence signaling NaN. + /// \param nan half-precision NaN value + /// \return quiet NaN + /// \exception FE_INVALID if \a nan is signaling NaN + inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int nan) + { + #if HALF_ERRHANDLING + raise(FE_INVALID, !(nan&0x200)); + #endif + return nan | 0x200; + } + + /// Signal and silence signaling NaNs. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \return quiet NaN + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int x, unsigned int y) + { + #if HALF_ERRHANDLING + raise(FE_INVALID, ((x&0x7FFF)>0x7C00 && !(x&0x200)) || ((y&0x7FFF)>0x7C00 && !(y&0x200))); + #endif + return ((x&0x7FFF)>0x7C00) ? (x|0x200) : (y|0x200); + } + + /// Signal and silence signaling NaNs. + /// \param x first half-precision value to check + /// \param y second half-precision value to check + /// \param z third half-precision value to check + /// \return quiet NaN + /// \exception FE_INVALID if \a x, \a y or \a z is signaling NaN + inline HALF_CONSTEXPR_NOERR unsigned int signal(unsigned int x, unsigned int y, unsigned int z) + { + #if HALF_ERRHANDLING + raise(FE_INVALID, ((x&0x7FFF)>0x7C00 && !(x&0x200)) || ((y&0x7FFF)>0x7C00 && !(y&0x200)) || ((z&0x7FFF)>0x7C00 && !(z&0x200))); + #endif + return ((x&0x7FFF)>0x7C00) ? (x|0x200) : ((y&0x7FFF)>0x7C00) ? (y|0x200) : (z|0x200); + } + + /// Select value or signaling NaN. + /// \param x preferred half-precision value + /// \param y ignored half-precision value except for signaling NaN + /// \return \a y if signaling NaN, \a x otherwise + /// \exception FE_INVALID if \a y is signaling NaN + inline HALF_CONSTEXPR_NOERR unsigned int select(unsigned int x, unsigned int HALF_UNUSED_NOERR(y)) + { + #if HALF_ERRHANDLING + return (((y&0x7FFF)>0x7C00) && !(y&0x200)) ? signal(y) : x; + #else + return x; + #endif + } + + /// Raise domain error and return NaN. + /// return quiet NaN + /// \exception FE_INVALID + inline HALF_CONSTEXPR_NOERR unsigned int invalid() + { + #if HALF_ERRHANDLING + raise(FE_INVALID); + #endif + return 0x7FFF; + } + + /// Raise pole error and return infinity. + /// \param sign half-precision value with sign bit only + /// \return half-precision infinity with sign of \a sign + /// \exception FE_DIVBYZERO + inline HALF_CONSTEXPR_NOERR unsigned int pole(unsigned int sign = 0) + { + #if HALF_ERRHANDLING + raise(FE_DIVBYZERO); + #endif + return sign | 0x7C00; + } + + /// Check value for underflow. + /// \param arg non-zero half-precision value to check + /// \return \a arg + /// \exception FE_UNDERFLOW if arg is subnormal + inline HALF_CONSTEXPR_NOERR unsigned int check_underflow(unsigned int arg) + { + #if HALF_ERRHANDLING && !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT + raise(FE_UNDERFLOW, !(arg&0x7C00)); + #endif + return arg; + } + + /// \} + /// \name Conversion and rounding + /// \{ + + /// Half-precision overflow. + /// \tparam R rounding mode to use + /// \param sign half-precision value with sign bit only + /// \return rounded overflowing half-precision value + /// \exception FE_OVERFLOW + template HALF_CONSTEXPR_NOERR unsigned int overflow(unsigned int sign = 0) + { + #if HALF_ERRHANDLING + raise(FE_OVERFLOW); + #endif + return (R==std::round_toward_infinity) ? (sign+0x7C00-(sign>>15)) : + (R==std::round_toward_neg_infinity) ? (sign+0x7BFF+(sign>>15)) : + (R==std::round_toward_zero) ? (sign|0x7BFF) : + (sign|0x7C00); + } + + /// Half-precision underflow. + /// \tparam R rounding mode to use + /// \param sign half-precision value with sign bit only + /// \return rounded underflowing half-precision value + /// \exception FE_UNDERFLOW + template HALF_CONSTEXPR_NOERR unsigned int underflow(unsigned int sign = 0) + { + #if HALF_ERRHANDLING + raise(FE_UNDERFLOW); + #endif + return (R==std::round_toward_infinity) ? (sign+1-(sign>>15)) : + (R==std::round_toward_neg_infinity) ? (sign+(sign>>15)) : + sign; + } + + /// Round half-precision number. + /// \tparam R rounding mode to use + /// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results + /// \param value finite half-precision number to round + /// \param g guard bit (most significant discarded bit) + /// \param s sticky bit (or of all but the most significant discarded bits) + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded or \a I is `true` + template HALF_CONSTEXPR_NOERR unsigned int rounded(unsigned int value, int g, int s) + { + #if HALF_ERRHANDLING + value += (R==std::round_to_nearest) ? (g&(s|value)) : + (R==std::round_toward_infinity) ? (~(value>>15)&(g|s)) : + (R==std::round_toward_neg_infinity) ? ((value>>15)&(g|s)) : 0; + if((value&0x7C00) == 0x7C00) + raise(FE_OVERFLOW); + else if(value & 0x7C00) + raise(FE_INEXACT, I || (g|s)!=0); + else + raise(FE_UNDERFLOW, !(HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT) || I || (g|s)!=0); + return value; + #else + return (R==std::round_to_nearest) ? (value+(g&(s|value))) : + (R==std::round_toward_infinity) ? (value+(~(value>>15)&(g|s))) : + (R==std::round_toward_neg_infinity) ? (value+((value>>15)&(g|s))) : + value; + #endif + } + + /// Round half-precision number to nearest integer value. + /// \tparam R rounding mode to use + /// \tparam E `true` for round to even, `false` for round away from zero + /// \tparam I `true` to raise INEXACT exception (if inexact), `false` to never raise it + /// \param value half-precision value to round + /// \return half-precision bits for nearest integral value + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded and \a I is `true` + template unsigned int integral(unsigned int value) + { + unsigned int abs = value & 0x7FFF; + if(abs < 0x3C00) + { + raise(FE_INEXACT, I); + return ((R==std::round_to_nearest) ? (0x3C00&-static_cast(abs>=(0x3800+E))) : + (R==std::round_toward_infinity) ? (0x3C00&-(~(value>>15)&(abs!=0))) : + (R==std::round_toward_neg_infinity) ? (0x3C00&-static_cast(value>0x8000)) : + 0) | (value&0x8000); + } + if(abs >= 0x6400) + return (abs>0x7C00) ? signal(value) : value; + unsigned int exp = 25 - (abs>>10), mask = (1<>exp)&E)) : + (R==std::round_toward_infinity) ? (mask&((value>>15)-1)) : + (R==std::round_toward_neg_infinity) ? (mask&-(value>>15)) : + 0) + value) & ~mask; + } + + /// Convert fixed point to half-precision floating-point. + /// \tparam R rounding mode to use + /// \tparam F number of fractional bits in [11,31] + /// \tparam S `true` for signed, `false` for unsigned + /// \tparam N `true` for additional normalization step, `false` if already normalized to 1.F + /// \tparam I `true` to always raise INEXACT exception, `false` to raise only for rounded results + /// \param m mantissa in Q1.F fixed point format + /// \param exp biased exponent - 1 + /// \param sign half-precision value with sign bit only + /// \param s sticky bit (or of all but the most significant already discarded bits) + /// \return value converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded or \a I is `true` + template unsigned int fixed2half(uint32 m, int exp = 14, unsigned int sign = 0, int s = 0) + { + if(S) + { + uint32 msign = sign_mask(m); + m = (m^msign) - msign; + sign = msign & 0x8000; + } + if(N) + for(; m<(static_cast(1)<(sign+(m>>(F-10-exp)), (m>>(F-11-exp))&1, s|((m&((static_cast(1)<<(F-11-exp))-1))!=0)); + return rounded(sign+(exp<<10)+(m>>(F-10)), (m>>(F-11))&1, s|((m&((static_cast(1)<<(F-11))-1))!=0)); + } + + /// Convert IEEE single-precision to half-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \tparam R rounding mode to use + /// \param value single-precision value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(float value, true_type) + { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtsi128_si32(_mm_cvtps_ph(_mm_set_ss(value), + (R==std::round_to_nearest) ? _MM_FROUND_TO_NEAREST_INT : + (R==std::round_toward_zero) ? _MM_FROUND_TO_ZERO : + (R==std::round_toward_infinity) ? _MM_FROUND_TO_POS_INF : + (R==std::round_toward_neg_infinity) ? _MM_FROUND_TO_NEG_INF : + _MM_FROUND_CUR_DIRECTION)); + #else + bits::type fbits; + std::memcpy(&fbits, &value, sizeof(float)); + #if 1 + unsigned int sign = (fbits>>16) & 0x8000; + fbits &= 0x7FFFFFFF; + if(fbits >= 0x7F800000) + return sign | 0x7C00 | ((fbits>0x7F800000) ? (0x200|((fbits>>13)&0x3FF)) : 0); + if(fbits >= 0x47800000) + return overflow(sign); + if(fbits >= 0x38800000) + return rounded(sign|(((fbits>>23)-112)<<10)|((fbits>>13)&0x3FF), (fbits>>12)&1, (fbits&0xFFF)!=0); + if(fbits >= 0x33000000) + { + int i = 125 - (fbits>>23); + fbits = (fbits&0x7FFFFF) | 0x800000; + return rounded(sign|(fbits>>(i+1)), (fbits>>i)&1, (fbits&((static_cast(1)<(sign); + return sign; + #else + static const uint16 base_table[512] = { + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, + 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0000, 0x0001, 0x0002, 0x0004, 0x0008, 0x0010, 0x0020, 0x0040, 0x0080, 0x0100, + 0x0200, 0x0400, 0x0800, 0x0C00, 0x1000, 0x1400, 0x1800, 0x1C00, 0x2000, 0x2400, 0x2800, 0x2C00, 0x3000, 0x3400, 0x3800, 0x3C00, + 0x4000, 0x4400, 0x4800, 0x4C00, 0x5000, 0x5400, 0x5800, 0x5C00, 0x6000, 0x6400, 0x6800, 0x6C00, 0x7000, 0x7400, 0x7800, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, + 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7BFF, 0x7C00, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, + 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8002, 0x8004, 0x8008, 0x8010, 0x8020, 0x8040, 0x8080, 0x8100, + 0x8200, 0x8400, 0x8800, 0x8C00, 0x9000, 0x9400, 0x9800, 0x9C00, 0xA000, 0xA400, 0xA800, 0xAC00, 0xB000, 0xB400, 0xB800, 0xBC00, + 0xC000, 0xC400, 0xC800, 0xCC00, 0xD000, 0xD400, 0xD800, 0xDC00, 0xE000, 0xE400, 0xE800, 0xEC00, 0xF000, 0xF400, 0xF800, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, + 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFBFF, 0xFC00 }; + static const unsigned char shift_table[256] = { + 24, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, 25, + 25, 25, 25, 25, 25, 25, 24, 23, 22, 21, 20, 19, 18, 17, 16, 15, 14, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, + 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 13, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, + 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 13 }; + int sexp = fbits >> 23, exp = sexp & 0xFF, i = shift_table[exp]; + fbits &= 0x7FFFFF; + uint32 m = (fbits|((exp!=0)<<23)) & -static_cast(exp!=0xFF); + return rounded(base_table[sexp]+(fbits>>i), (m>>(i-1))&1, (((static_cast(1)<<(i-1))-1)&m)!=0); + #endif + #endif + } + + /// Convert IEEE double-precision to half-precision. + /// \tparam R rounding mode to use + /// \param value double-precision value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(double value, true_type) + { + #if HALF_ENABLE_F16C_INTRINSICS + if(R == std::round_indeterminate) + return _mm_cvtsi128_si32(_mm_cvtps_ph(_mm_cvtpd_ps(_mm_set_sd(value)), _MM_FROUND_CUR_DIRECTION)); + #endif + bits::type dbits; + std::memcpy(&dbits, &value, sizeof(double)); + uint32 hi = dbits >> 32, lo = dbits & 0xFFFFFFFF; + unsigned int sign = (hi>>16) & 0x8000; + hi &= 0x7FFFFFFF; + if(hi >= 0x7FF00000) + return sign | 0x7C00 | ((dbits&0xFFFFFFFFFFFFF) ? (0x200|((hi>>10)&0x3FF)) : 0); + if(hi >= 0x40F00000) + return overflow(sign); + if(hi >= 0x3F100000) + return rounded(sign|(((hi>>20)-1008)<<10)|((hi>>10)&0x3FF), (hi>>9)&1, ((hi&0x1FF)|lo)!=0); + if(hi >= 0x3E600000) + { + int i = 1018 - (hi>>20); + hi = (hi&0xFFFFF) | 0x100000; + return rounded(sign|(hi>>(i+1)), (hi>>i)&1, ((hi&((static_cast(1)<(sign); + return sign; + } + + /// Convert non-IEEE floating-point to half-precision. + /// \tparam R rounding mode to use + /// \tparam T source type (builtin floating-point type) + /// \param value floating-point value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half_impl(T value, ...) + { + unsigned int hbits = static_cast(builtin_signbit(value)) << 15; + if(value == T()) + return hbits; + if(builtin_isnan(value)) + return hbits | 0x7FFF; + if(builtin_isinf(value)) + return hbits | 0x7C00; + int exp; + std::frexp(value, &exp); + if(exp > 16) + return overflow(hbits); + if(exp < -13) + value = std::ldexp(value, 25); + else + { + value = std::ldexp(value, 12-exp); + hbits |= ((exp+13)<<10); + } + T ival, frac = std::modf(value, &ival); + int m = std::abs(static_cast(ival)); + return rounded(hbits+(m>>1), m&1, frac!=T()); + } + + /// Convert floating-point to half-precision. + /// \tparam R rounding mode to use + /// \tparam T source type (builtin floating-point type) + /// \param value floating-point value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int float2half(T value) + { + return float2half_impl(value, bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); + } + + /// Convert integer to half-precision floating-point. + /// \tparam R rounding mode to use + /// \tparam T type to convert (builtin integer type) + /// \param value integral value to convert + /// \return rounded half-precision value + /// \exception FE_OVERFLOW on overflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int int2half(T value) + { + unsigned int bits = static_cast(value<0) << 15; + if(!value) + return bits; + if(bits) + value = -value; + if(value > 0xFFFF) + return overflow(bits); + unsigned int m = static_cast(value), exp = 24; + for(; m<0x400; m<<=1,--exp) ; + for(; m>0x7FF; m>>=1,++exp) ; + bits |= (exp<<10) + m; + return (exp>24) ? rounded(bits, (value>>(exp-25))&1, (((1<<(exp-25))-1)&value)!=0) : bits; + } + + /// Convert half-precision to IEEE single-precision. + /// Credit for this goes to [Jeroen van der Zijp](ftp://ftp.fox-toolkit.org/pub/fasthalffloatconversion.pdf). + /// \param value half-precision value to convert + /// \return single-precision value + inline float half2float_impl(unsigned int value, float, true_type) + { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtss_f32(_mm_cvtph_ps(_mm_cvtsi32_si128(value))); + #else + #if 0 + bits::type fbits = static_cast::type>(value&0x8000) << 16; + int abs = value & 0x7FFF; + if(abs) + { + fbits |= 0x38000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,fbits-=0x800000) ; + fbits += static_cast::type>(abs) << 13; + } + #else + static const bits::type mantissa_table[2048] = { + 0x00000000, 0x33800000, 0x34000000, 0x34400000, 0x34800000, 0x34A00000, 0x34C00000, 0x34E00000, 0x35000000, 0x35100000, 0x35200000, 0x35300000, 0x35400000, 0x35500000, 0x35600000, 0x35700000, + 0x35800000, 0x35880000, 0x35900000, 0x35980000, 0x35A00000, 0x35A80000, 0x35B00000, 0x35B80000, 0x35C00000, 0x35C80000, 0x35D00000, 0x35D80000, 0x35E00000, 0x35E80000, 0x35F00000, 0x35F80000, + 0x36000000, 0x36040000, 0x36080000, 0x360C0000, 0x36100000, 0x36140000, 0x36180000, 0x361C0000, 0x36200000, 0x36240000, 0x36280000, 0x362C0000, 0x36300000, 0x36340000, 0x36380000, 0x363C0000, + 0x36400000, 0x36440000, 0x36480000, 0x364C0000, 0x36500000, 0x36540000, 0x36580000, 0x365C0000, 0x36600000, 0x36640000, 0x36680000, 0x366C0000, 0x36700000, 0x36740000, 0x36780000, 0x367C0000, + 0x36800000, 0x36820000, 0x36840000, 0x36860000, 0x36880000, 0x368A0000, 0x368C0000, 0x368E0000, 0x36900000, 0x36920000, 0x36940000, 0x36960000, 0x36980000, 0x369A0000, 0x369C0000, 0x369E0000, + 0x36A00000, 0x36A20000, 0x36A40000, 0x36A60000, 0x36A80000, 0x36AA0000, 0x36AC0000, 0x36AE0000, 0x36B00000, 0x36B20000, 0x36B40000, 0x36B60000, 0x36B80000, 0x36BA0000, 0x36BC0000, 0x36BE0000, + 0x36C00000, 0x36C20000, 0x36C40000, 0x36C60000, 0x36C80000, 0x36CA0000, 0x36CC0000, 0x36CE0000, 0x36D00000, 0x36D20000, 0x36D40000, 0x36D60000, 0x36D80000, 0x36DA0000, 0x36DC0000, 0x36DE0000, + 0x36E00000, 0x36E20000, 0x36E40000, 0x36E60000, 0x36E80000, 0x36EA0000, 0x36EC0000, 0x36EE0000, 0x36F00000, 0x36F20000, 0x36F40000, 0x36F60000, 0x36F80000, 0x36FA0000, 0x36FC0000, 0x36FE0000, + 0x37000000, 0x37010000, 0x37020000, 0x37030000, 0x37040000, 0x37050000, 0x37060000, 0x37070000, 0x37080000, 0x37090000, 0x370A0000, 0x370B0000, 0x370C0000, 0x370D0000, 0x370E0000, 0x370F0000, + 0x37100000, 0x37110000, 0x37120000, 0x37130000, 0x37140000, 0x37150000, 0x37160000, 0x37170000, 0x37180000, 0x37190000, 0x371A0000, 0x371B0000, 0x371C0000, 0x371D0000, 0x371E0000, 0x371F0000, + 0x37200000, 0x37210000, 0x37220000, 0x37230000, 0x37240000, 0x37250000, 0x37260000, 0x37270000, 0x37280000, 0x37290000, 0x372A0000, 0x372B0000, 0x372C0000, 0x372D0000, 0x372E0000, 0x372F0000, + 0x37300000, 0x37310000, 0x37320000, 0x37330000, 0x37340000, 0x37350000, 0x37360000, 0x37370000, 0x37380000, 0x37390000, 0x373A0000, 0x373B0000, 0x373C0000, 0x373D0000, 0x373E0000, 0x373F0000, + 0x37400000, 0x37410000, 0x37420000, 0x37430000, 0x37440000, 0x37450000, 0x37460000, 0x37470000, 0x37480000, 0x37490000, 0x374A0000, 0x374B0000, 0x374C0000, 0x374D0000, 0x374E0000, 0x374F0000, + 0x37500000, 0x37510000, 0x37520000, 0x37530000, 0x37540000, 0x37550000, 0x37560000, 0x37570000, 0x37580000, 0x37590000, 0x375A0000, 0x375B0000, 0x375C0000, 0x375D0000, 0x375E0000, 0x375F0000, + 0x37600000, 0x37610000, 0x37620000, 0x37630000, 0x37640000, 0x37650000, 0x37660000, 0x37670000, 0x37680000, 0x37690000, 0x376A0000, 0x376B0000, 0x376C0000, 0x376D0000, 0x376E0000, 0x376F0000, + 0x37700000, 0x37710000, 0x37720000, 0x37730000, 0x37740000, 0x37750000, 0x37760000, 0x37770000, 0x37780000, 0x37790000, 0x377A0000, 0x377B0000, 0x377C0000, 0x377D0000, 0x377E0000, 0x377F0000, + 0x37800000, 0x37808000, 0x37810000, 0x37818000, 0x37820000, 0x37828000, 0x37830000, 0x37838000, 0x37840000, 0x37848000, 0x37850000, 0x37858000, 0x37860000, 0x37868000, 0x37870000, 0x37878000, + 0x37880000, 0x37888000, 0x37890000, 0x37898000, 0x378A0000, 0x378A8000, 0x378B0000, 0x378B8000, 0x378C0000, 0x378C8000, 0x378D0000, 0x378D8000, 0x378E0000, 0x378E8000, 0x378F0000, 0x378F8000, + 0x37900000, 0x37908000, 0x37910000, 0x37918000, 0x37920000, 0x37928000, 0x37930000, 0x37938000, 0x37940000, 0x37948000, 0x37950000, 0x37958000, 0x37960000, 0x37968000, 0x37970000, 0x37978000, + 0x37980000, 0x37988000, 0x37990000, 0x37998000, 0x379A0000, 0x379A8000, 0x379B0000, 0x379B8000, 0x379C0000, 0x379C8000, 0x379D0000, 0x379D8000, 0x379E0000, 0x379E8000, 0x379F0000, 0x379F8000, + 0x37A00000, 0x37A08000, 0x37A10000, 0x37A18000, 0x37A20000, 0x37A28000, 0x37A30000, 0x37A38000, 0x37A40000, 0x37A48000, 0x37A50000, 0x37A58000, 0x37A60000, 0x37A68000, 0x37A70000, 0x37A78000, + 0x37A80000, 0x37A88000, 0x37A90000, 0x37A98000, 0x37AA0000, 0x37AA8000, 0x37AB0000, 0x37AB8000, 0x37AC0000, 0x37AC8000, 0x37AD0000, 0x37AD8000, 0x37AE0000, 0x37AE8000, 0x37AF0000, 0x37AF8000, + 0x37B00000, 0x37B08000, 0x37B10000, 0x37B18000, 0x37B20000, 0x37B28000, 0x37B30000, 0x37B38000, 0x37B40000, 0x37B48000, 0x37B50000, 0x37B58000, 0x37B60000, 0x37B68000, 0x37B70000, 0x37B78000, + 0x37B80000, 0x37B88000, 0x37B90000, 0x37B98000, 0x37BA0000, 0x37BA8000, 0x37BB0000, 0x37BB8000, 0x37BC0000, 0x37BC8000, 0x37BD0000, 0x37BD8000, 0x37BE0000, 0x37BE8000, 0x37BF0000, 0x37BF8000, + 0x37C00000, 0x37C08000, 0x37C10000, 0x37C18000, 0x37C20000, 0x37C28000, 0x37C30000, 0x37C38000, 0x37C40000, 0x37C48000, 0x37C50000, 0x37C58000, 0x37C60000, 0x37C68000, 0x37C70000, 0x37C78000, + 0x37C80000, 0x37C88000, 0x37C90000, 0x37C98000, 0x37CA0000, 0x37CA8000, 0x37CB0000, 0x37CB8000, 0x37CC0000, 0x37CC8000, 0x37CD0000, 0x37CD8000, 0x37CE0000, 0x37CE8000, 0x37CF0000, 0x37CF8000, + 0x37D00000, 0x37D08000, 0x37D10000, 0x37D18000, 0x37D20000, 0x37D28000, 0x37D30000, 0x37D38000, 0x37D40000, 0x37D48000, 0x37D50000, 0x37D58000, 0x37D60000, 0x37D68000, 0x37D70000, 0x37D78000, + 0x37D80000, 0x37D88000, 0x37D90000, 0x37D98000, 0x37DA0000, 0x37DA8000, 0x37DB0000, 0x37DB8000, 0x37DC0000, 0x37DC8000, 0x37DD0000, 0x37DD8000, 0x37DE0000, 0x37DE8000, 0x37DF0000, 0x37DF8000, + 0x37E00000, 0x37E08000, 0x37E10000, 0x37E18000, 0x37E20000, 0x37E28000, 0x37E30000, 0x37E38000, 0x37E40000, 0x37E48000, 0x37E50000, 0x37E58000, 0x37E60000, 0x37E68000, 0x37E70000, 0x37E78000, + 0x37E80000, 0x37E88000, 0x37E90000, 0x37E98000, 0x37EA0000, 0x37EA8000, 0x37EB0000, 0x37EB8000, 0x37EC0000, 0x37EC8000, 0x37ED0000, 0x37ED8000, 0x37EE0000, 0x37EE8000, 0x37EF0000, 0x37EF8000, + 0x37F00000, 0x37F08000, 0x37F10000, 0x37F18000, 0x37F20000, 0x37F28000, 0x37F30000, 0x37F38000, 0x37F40000, 0x37F48000, 0x37F50000, 0x37F58000, 0x37F60000, 0x37F68000, 0x37F70000, 0x37F78000, + 0x37F80000, 0x37F88000, 0x37F90000, 0x37F98000, 0x37FA0000, 0x37FA8000, 0x37FB0000, 0x37FB8000, 0x37FC0000, 0x37FC8000, 0x37FD0000, 0x37FD8000, 0x37FE0000, 0x37FE8000, 0x37FF0000, 0x37FF8000, + 0x38000000, 0x38004000, 0x38008000, 0x3800C000, 0x38010000, 0x38014000, 0x38018000, 0x3801C000, 0x38020000, 0x38024000, 0x38028000, 0x3802C000, 0x38030000, 0x38034000, 0x38038000, 0x3803C000, + 0x38040000, 0x38044000, 0x38048000, 0x3804C000, 0x38050000, 0x38054000, 0x38058000, 0x3805C000, 0x38060000, 0x38064000, 0x38068000, 0x3806C000, 0x38070000, 0x38074000, 0x38078000, 0x3807C000, + 0x38080000, 0x38084000, 0x38088000, 0x3808C000, 0x38090000, 0x38094000, 0x38098000, 0x3809C000, 0x380A0000, 0x380A4000, 0x380A8000, 0x380AC000, 0x380B0000, 0x380B4000, 0x380B8000, 0x380BC000, + 0x380C0000, 0x380C4000, 0x380C8000, 0x380CC000, 0x380D0000, 0x380D4000, 0x380D8000, 0x380DC000, 0x380E0000, 0x380E4000, 0x380E8000, 0x380EC000, 0x380F0000, 0x380F4000, 0x380F8000, 0x380FC000, + 0x38100000, 0x38104000, 0x38108000, 0x3810C000, 0x38110000, 0x38114000, 0x38118000, 0x3811C000, 0x38120000, 0x38124000, 0x38128000, 0x3812C000, 0x38130000, 0x38134000, 0x38138000, 0x3813C000, + 0x38140000, 0x38144000, 0x38148000, 0x3814C000, 0x38150000, 0x38154000, 0x38158000, 0x3815C000, 0x38160000, 0x38164000, 0x38168000, 0x3816C000, 0x38170000, 0x38174000, 0x38178000, 0x3817C000, + 0x38180000, 0x38184000, 0x38188000, 0x3818C000, 0x38190000, 0x38194000, 0x38198000, 0x3819C000, 0x381A0000, 0x381A4000, 0x381A8000, 0x381AC000, 0x381B0000, 0x381B4000, 0x381B8000, 0x381BC000, + 0x381C0000, 0x381C4000, 0x381C8000, 0x381CC000, 0x381D0000, 0x381D4000, 0x381D8000, 0x381DC000, 0x381E0000, 0x381E4000, 0x381E8000, 0x381EC000, 0x381F0000, 0x381F4000, 0x381F8000, 0x381FC000, + 0x38200000, 0x38204000, 0x38208000, 0x3820C000, 0x38210000, 0x38214000, 0x38218000, 0x3821C000, 0x38220000, 0x38224000, 0x38228000, 0x3822C000, 0x38230000, 0x38234000, 0x38238000, 0x3823C000, + 0x38240000, 0x38244000, 0x38248000, 0x3824C000, 0x38250000, 0x38254000, 0x38258000, 0x3825C000, 0x38260000, 0x38264000, 0x38268000, 0x3826C000, 0x38270000, 0x38274000, 0x38278000, 0x3827C000, + 0x38280000, 0x38284000, 0x38288000, 0x3828C000, 0x38290000, 0x38294000, 0x38298000, 0x3829C000, 0x382A0000, 0x382A4000, 0x382A8000, 0x382AC000, 0x382B0000, 0x382B4000, 0x382B8000, 0x382BC000, + 0x382C0000, 0x382C4000, 0x382C8000, 0x382CC000, 0x382D0000, 0x382D4000, 0x382D8000, 0x382DC000, 0x382E0000, 0x382E4000, 0x382E8000, 0x382EC000, 0x382F0000, 0x382F4000, 0x382F8000, 0x382FC000, + 0x38300000, 0x38304000, 0x38308000, 0x3830C000, 0x38310000, 0x38314000, 0x38318000, 0x3831C000, 0x38320000, 0x38324000, 0x38328000, 0x3832C000, 0x38330000, 0x38334000, 0x38338000, 0x3833C000, + 0x38340000, 0x38344000, 0x38348000, 0x3834C000, 0x38350000, 0x38354000, 0x38358000, 0x3835C000, 0x38360000, 0x38364000, 0x38368000, 0x3836C000, 0x38370000, 0x38374000, 0x38378000, 0x3837C000, + 0x38380000, 0x38384000, 0x38388000, 0x3838C000, 0x38390000, 0x38394000, 0x38398000, 0x3839C000, 0x383A0000, 0x383A4000, 0x383A8000, 0x383AC000, 0x383B0000, 0x383B4000, 0x383B8000, 0x383BC000, + 0x383C0000, 0x383C4000, 0x383C8000, 0x383CC000, 0x383D0000, 0x383D4000, 0x383D8000, 0x383DC000, 0x383E0000, 0x383E4000, 0x383E8000, 0x383EC000, 0x383F0000, 0x383F4000, 0x383F8000, 0x383FC000, + 0x38400000, 0x38404000, 0x38408000, 0x3840C000, 0x38410000, 0x38414000, 0x38418000, 0x3841C000, 0x38420000, 0x38424000, 0x38428000, 0x3842C000, 0x38430000, 0x38434000, 0x38438000, 0x3843C000, + 0x38440000, 0x38444000, 0x38448000, 0x3844C000, 0x38450000, 0x38454000, 0x38458000, 0x3845C000, 0x38460000, 0x38464000, 0x38468000, 0x3846C000, 0x38470000, 0x38474000, 0x38478000, 0x3847C000, + 0x38480000, 0x38484000, 0x38488000, 0x3848C000, 0x38490000, 0x38494000, 0x38498000, 0x3849C000, 0x384A0000, 0x384A4000, 0x384A8000, 0x384AC000, 0x384B0000, 0x384B4000, 0x384B8000, 0x384BC000, + 0x384C0000, 0x384C4000, 0x384C8000, 0x384CC000, 0x384D0000, 0x384D4000, 0x384D8000, 0x384DC000, 0x384E0000, 0x384E4000, 0x384E8000, 0x384EC000, 0x384F0000, 0x384F4000, 0x384F8000, 0x384FC000, + 0x38500000, 0x38504000, 0x38508000, 0x3850C000, 0x38510000, 0x38514000, 0x38518000, 0x3851C000, 0x38520000, 0x38524000, 0x38528000, 0x3852C000, 0x38530000, 0x38534000, 0x38538000, 0x3853C000, + 0x38540000, 0x38544000, 0x38548000, 0x3854C000, 0x38550000, 0x38554000, 0x38558000, 0x3855C000, 0x38560000, 0x38564000, 0x38568000, 0x3856C000, 0x38570000, 0x38574000, 0x38578000, 0x3857C000, + 0x38580000, 0x38584000, 0x38588000, 0x3858C000, 0x38590000, 0x38594000, 0x38598000, 0x3859C000, 0x385A0000, 0x385A4000, 0x385A8000, 0x385AC000, 0x385B0000, 0x385B4000, 0x385B8000, 0x385BC000, + 0x385C0000, 0x385C4000, 0x385C8000, 0x385CC000, 0x385D0000, 0x385D4000, 0x385D8000, 0x385DC000, 0x385E0000, 0x385E4000, 0x385E8000, 0x385EC000, 0x385F0000, 0x385F4000, 0x385F8000, 0x385FC000, + 0x38600000, 0x38604000, 0x38608000, 0x3860C000, 0x38610000, 0x38614000, 0x38618000, 0x3861C000, 0x38620000, 0x38624000, 0x38628000, 0x3862C000, 0x38630000, 0x38634000, 0x38638000, 0x3863C000, + 0x38640000, 0x38644000, 0x38648000, 0x3864C000, 0x38650000, 0x38654000, 0x38658000, 0x3865C000, 0x38660000, 0x38664000, 0x38668000, 0x3866C000, 0x38670000, 0x38674000, 0x38678000, 0x3867C000, + 0x38680000, 0x38684000, 0x38688000, 0x3868C000, 0x38690000, 0x38694000, 0x38698000, 0x3869C000, 0x386A0000, 0x386A4000, 0x386A8000, 0x386AC000, 0x386B0000, 0x386B4000, 0x386B8000, 0x386BC000, + 0x386C0000, 0x386C4000, 0x386C8000, 0x386CC000, 0x386D0000, 0x386D4000, 0x386D8000, 0x386DC000, 0x386E0000, 0x386E4000, 0x386E8000, 0x386EC000, 0x386F0000, 0x386F4000, 0x386F8000, 0x386FC000, + 0x38700000, 0x38704000, 0x38708000, 0x3870C000, 0x38710000, 0x38714000, 0x38718000, 0x3871C000, 0x38720000, 0x38724000, 0x38728000, 0x3872C000, 0x38730000, 0x38734000, 0x38738000, 0x3873C000, + 0x38740000, 0x38744000, 0x38748000, 0x3874C000, 0x38750000, 0x38754000, 0x38758000, 0x3875C000, 0x38760000, 0x38764000, 0x38768000, 0x3876C000, 0x38770000, 0x38774000, 0x38778000, 0x3877C000, + 0x38780000, 0x38784000, 0x38788000, 0x3878C000, 0x38790000, 0x38794000, 0x38798000, 0x3879C000, 0x387A0000, 0x387A4000, 0x387A8000, 0x387AC000, 0x387B0000, 0x387B4000, 0x387B8000, 0x387BC000, + 0x387C0000, 0x387C4000, 0x387C8000, 0x387CC000, 0x387D0000, 0x387D4000, 0x387D8000, 0x387DC000, 0x387E0000, 0x387E4000, 0x387E8000, 0x387EC000, 0x387F0000, 0x387F4000, 0x387F8000, 0x387FC000, + 0x38000000, 0x38002000, 0x38004000, 0x38006000, 0x38008000, 0x3800A000, 0x3800C000, 0x3800E000, 0x38010000, 0x38012000, 0x38014000, 0x38016000, 0x38018000, 0x3801A000, 0x3801C000, 0x3801E000, + 0x38020000, 0x38022000, 0x38024000, 0x38026000, 0x38028000, 0x3802A000, 0x3802C000, 0x3802E000, 0x38030000, 0x38032000, 0x38034000, 0x38036000, 0x38038000, 0x3803A000, 0x3803C000, 0x3803E000, + 0x38040000, 0x38042000, 0x38044000, 0x38046000, 0x38048000, 0x3804A000, 0x3804C000, 0x3804E000, 0x38050000, 0x38052000, 0x38054000, 0x38056000, 0x38058000, 0x3805A000, 0x3805C000, 0x3805E000, + 0x38060000, 0x38062000, 0x38064000, 0x38066000, 0x38068000, 0x3806A000, 0x3806C000, 0x3806E000, 0x38070000, 0x38072000, 0x38074000, 0x38076000, 0x38078000, 0x3807A000, 0x3807C000, 0x3807E000, + 0x38080000, 0x38082000, 0x38084000, 0x38086000, 0x38088000, 0x3808A000, 0x3808C000, 0x3808E000, 0x38090000, 0x38092000, 0x38094000, 0x38096000, 0x38098000, 0x3809A000, 0x3809C000, 0x3809E000, + 0x380A0000, 0x380A2000, 0x380A4000, 0x380A6000, 0x380A8000, 0x380AA000, 0x380AC000, 0x380AE000, 0x380B0000, 0x380B2000, 0x380B4000, 0x380B6000, 0x380B8000, 0x380BA000, 0x380BC000, 0x380BE000, + 0x380C0000, 0x380C2000, 0x380C4000, 0x380C6000, 0x380C8000, 0x380CA000, 0x380CC000, 0x380CE000, 0x380D0000, 0x380D2000, 0x380D4000, 0x380D6000, 0x380D8000, 0x380DA000, 0x380DC000, 0x380DE000, + 0x380E0000, 0x380E2000, 0x380E4000, 0x380E6000, 0x380E8000, 0x380EA000, 0x380EC000, 0x380EE000, 0x380F0000, 0x380F2000, 0x380F4000, 0x380F6000, 0x380F8000, 0x380FA000, 0x380FC000, 0x380FE000, + 0x38100000, 0x38102000, 0x38104000, 0x38106000, 0x38108000, 0x3810A000, 0x3810C000, 0x3810E000, 0x38110000, 0x38112000, 0x38114000, 0x38116000, 0x38118000, 0x3811A000, 0x3811C000, 0x3811E000, + 0x38120000, 0x38122000, 0x38124000, 0x38126000, 0x38128000, 0x3812A000, 0x3812C000, 0x3812E000, 0x38130000, 0x38132000, 0x38134000, 0x38136000, 0x38138000, 0x3813A000, 0x3813C000, 0x3813E000, + 0x38140000, 0x38142000, 0x38144000, 0x38146000, 0x38148000, 0x3814A000, 0x3814C000, 0x3814E000, 0x38150000, 0x38152000, 0x38154000, 0x38156000, 0x38158000, 0x3815A000, 0x3815C000, 0x3815E000, + 0x38160000, 0x38162000, 0x38164000, 0x38166000, 0x38168000, 0x3816A000, 0x3816C000, 0x3816E000, 0x38170000, 0x38172000, 0x38174000, 0x38176000, 0x38178000, 0x3817A000, 0x3817C000, 0x3817E000, + 0x38180000, 0x38182000, 0x38184000, 0x38186000, 0x38188000, 0x3818A000, 0x3818C000, 0x3818E000, 0x38190000, 0x38192000, 0x38194000, 0x38196000, 0x38198000, 0x3819A000, 0x3819C000, 0x3819E000, + 0x381A0000, 0x381A2000, 0x381A4000, 0x381A6000, 0x381A8000, 0x381AA000, 0x381AC000, 0x381AE000, 0x381B0000, 0x381B2000, 0x381B4000, 0x381B6000, 0x381B8000, 0x381BA000, 0x381BC000, 0x381BE000, + 0x381C0000, 0x381C2000, 0x381C4000, 0x381C6000, 0x381C8000, 0x381CA000, 0x381CC000, 0x381CE000, 0x381D0000, 0x381D2000, 0x381D4000, 0x381D6000, 0x381D8000, 0x381DA000, 0x381DC000, 0x381DE000, + 0x381E0000, 0x381E2000, 0x381E4000, 0x381E6000, 0x381E8000, 0x381EA000, 0x381EC000, 0x381EE000, 0x381F0000, 0x381F2000, 0x381F4000, 0x381F6000, 0x381F8000, 0x381FA000, 0x381FC000, 0x381FE000, + 0x38200000, 0x38202000, 0x38204000, 0x38206000, 0x38208000, 0x3820A000, 0x3820C000, 0x3820E000, 0x38210000, 0x38212000, 0x38214000, 0x38216000, 0x38218000, 0x3821A000, 0x3821C000, 0x3821E000, + 0x38220000, 0x38222000, 0x38224000, 0x38226000, 0x38228000, 0x3822A000, 0x3822C000, 0x3822E000, 0x38230000, 0x38232000, 0x38234000, 0x38236000, 0x38238000, 0x3823A000, 0x3823C000, 0x3823E000, + 0x38240000, 0x38242000, 0x38244000, 0x38246000, 0x38248000, 0x3824A000, 0x3824C000, 0x3824E000, 0x38250000, 0x38252000, 0x38254000, 0x38256000, 0x38258000, 0x3825A000, 0x3825C000, 0x3825E000, + 0x38260000, 0x38262000, 0x38264000, 0x38266000, 0x38268000, 0x3826A000, 0x3826C000, 0x3826E000, 0x38270000, 0x38272000, 0x38274000, 0x38276000, 0x38278000, 0x3827A000, 0x3827C000, 0x3827E000, + 0x38280000, 0x38282000, 0x38284000, 0x38286000, 0x38288000, 0x3828A000, 0x3828C000, 0x3828E000, 0x38290000, 0x38292000, 0x38294000, 0x38296000, 0x38298000, 0x3829A000, 0x3829C000, 0x3829E000, + 0x382A0000, 0x382A2000, 0x382A4000, 0x382A6000, 0x382A8000, 0x382AA000, 0x382AC000, 0x382AE000, 0x382B0000, 0x382B2000, 0x382B4000, 0x382B6000, 0x382B8000, 0x382BA000, 0x382BC000, 0x382BE000, + 0x382C0000, 0x382C2000, 0x382C4000, 0x382C6000, 0x382C8000, 0x382CA000, 0x382CC000, 0x382CE000, 0x382D0000, 0x382D2000, 0x382D4000, 0x382D6000, 0x382D8000, 0x382DA000, 0x382DC000, 0x382DE000, + 0x382E0000, 0x382E2000, 0x382E4000, 0x382E6000, 0x382E8000, 0x382EA000, 0x382EC000, 0x382EE000, 0x382F0000, 0x382F2000, 0x382F4000, 0x382F6000, 0x382F8000, 0x382FA000, 0x382FC000, 0x382FE000, + 0x38300000, 0x38302000, 0x38304000, 0x38306000, 0x38308000, 0x3830A000, 0x3830C000, 0x3830E000, 0x38310000, 0x38312000, 0x38314000, 0x38316000, 0x38318000, 0x3831A000, 0x3831C000, 0x3831E000, + 0x38320000, 0x38322000, 0x38324000, 0x38326000, 0x38328000, 0x3832A000, 0x3832C000, 0x3832E000, 0x38330000, 0x38332000, 0x38334000, 0x38336000, 0x38338000, 0x3833A000, 0x3833C000, 0x3833E000, + 0x38340000, 0x38342000, 0x38344000, 0x38346000, 0x38348000, 0x3834A000, 0x3834C000, 0x3834E000, 0x38350000, 0x38352000, 0x38354000, 0x38356000, 0x38358000, 0x3835A000, 0x3835C000, 0x3835E000, + 0x38360000, 0x38362000, 0x38364000, 0x38366000, 0x38368000, 0x3836A000, 0x3836C000, 0x3836E000, 0x38370000, 0x38372000, 0x38374000, 0x38376000, 0x38378000, 0x3837A000, 0x3837C000, 0x3837E000, + 0x38380000, 0x38382000, 0x38384000, 0x38386000, 0x38388000, 0x3838A000, 0x3838C000, 0x3838E000, 0x38390000, 0x38392000, 0x38394000, 0x38396000, 0x38398000, 0x3839A000, 0x3839C000, 0x3839E000, + 0x383A0000, 0x383A2000, 0x383A4000, 0x383A6000, 0x383A8000, 0x383AA000, 0x383AC000, 0x383AE000, 0x383B0000, 0x383B2000, 0x383B4000, 0x383B6000, 0x383B8000, 0x383BA000, 0x383BC000, 0x383BE000, + 0x383C0000, 0x383C2000, 0x383C4000, 0x383C6000, 0x383C8000, 0x383CA000, 0x383CC000, 0x383CE000, 0x383D0000, 0x383D2000, 0x383D4000, 0x383D6000, 0x383D8000, 0x383DA000, 0x383DC000, 0x383DE000, + 0x383E0000, 0x383E2000, 0x383E4000, 0x383E6000, 0x383E8000, 0x383EA000, 0x383EC000, 0x383EE000, 0x383F0000, 0x383F2000, 0x383F4000, 0x383F6000, 0x383F8000, 0x383FA000, 0x383FC000, 0x383FE000, + 0x38400000, 0x38402000, 0x38404000, 0x38406000, 0x38408000, 0x3840A000, 0x3840C000, 0x3840E000, 0x38410000, 0x38412000, 0x38414000, 0x38416000, 0x38418000, 0x3841A000, 0x3841C000, 0x3841E000, + 0x38420000, 0x38422000, 0x38424000, 0x38426000, 0x38428000, 0x3842A000, 0x3842C000, 0x3842E000, 0x38430000, 0x38432000, 0x38434000, 0x38436000, 0x38438000, 0x3843A000, 0x3843C000, 0x3843E000, + 0x38440000, 0x38442000, 0x38444000, 0x38446000, 0x38448000, 0x3844A000, 0x3844C000, 0x3844E000, 0x38450000, 0x38452000, 0x38454000, 0x38456000, 0x38458000, 0x3845A000, 0x3845C000, 0x3845E000, + 0x38460000, 0x38462000, 0x38464000, 0x38466000, 0x38468000, 0x3846A000, 0x3846C000, 0x3846E000, 0x38470000, 0x38472000, 0x38474000, 0x38476000, 0x38478000, 0x3847A000, 0x3847C000, 0x3847E000, + 0x38480000, 0x38482000, 0x38484000, 0x38486000, 0x38488000, 0x3848A000, 0x3848C000, 0x3848E000, 0x38490000, 0x38492000, 0x38494000, 0x38496000, 0x38498000, 0x3849A000, 0x3849C000, 0x3849E000, + 0x384A0000, 0x384A2000, 0x384A4000, 0x384A6000, 0x384A8000, 0x384AA000, 0x384AC000, 0x384AE000, 0x384B0000, 0x384B2000, 0x384B4000, 0x384B6000, 0x384B8000, 0x384BA000, 0x384BC000, 0x384BE000, + 0x384C0000, 0x384C2000, 0x384C4000, 0x384C6000, 0x384C8000, 0x384CA000, 0x384CC000, 0x384CE000, 0x384D0000, 0x384D2000, 0x384D4000, 0x384D6000, 0x384D8000, 0x384DA000, 0x384DC000, 0x384DE000, + 0x384E0000, 0x384E2000, 0x384E4000, 0x384E6000, 0x384E8000, 0x384EA000, 0x384EC000, 0x384EE000, 0x384F0000, 0x384F2000, 0x384F4000, 0x384F6000, 0x384F8000, 0x384FA000, 0x384FC000, 0x384FE000, + 0x38500000, 0x38502000, 0x38504000, 0x38506000, 0x38508000, 0x3850A000, 0x3850C000, 0x3850E000, 0x38510000, 0x38512000, 0x38514000, 0x38516000, 0x38518000, 0x3851A000, 0x3851C000, 0x3851E000, + 0x38520000, 0x38522000, 0x38524000, 0x38526000, 0x38528000, 0x3852A000, 0x3852C000, 0x3852E000, 0x38530000, 0x38532000, 0x38534000, 0x38536000, 0x38538000, 0x3853A000, 0x3853C000, 0x3853E000, + 0x38540000, 0x38542000, 0x38544000, 0x38546000, 0x38548000, 0x3854A000, 0x3854C000, 0x3854E000, 0x38550000, 0x38552000, 0x38554000, 0x38556000, 0x38558000, 0x3855A000, 0x3855C000, 0x3855E000, + 0x38560000, 0x38562000, 0x38564000, 0x38566000, 0x38568000, 0x3856A000, 0x3856C000, 0x3856E000, 0x38570000, 0x38572000, 0x38574000, 0x38576000, 0x38578000, 0x3857A000, 0x3857C000, 0x3857E000, + 0x38580000, 0x38582000, 0x38584000, 0x38586000, 0x38588000, 0x3858A000, 0x3858C000, 0x3858E000, 0x38590000, 0x38592000, 0x38594000, 0x38596000, 0x38598000, 0x3859A000, 0x3859C000, 0x3859E000, + 0x385A0000, 0x385A2000, 0x385A4000, 0x385A6000, 0x385A8000, 0x385AA000, 0x385AC000, 0x385AE000, 0x385B0000, 0x385B2000, 0x385B4000, 0x385B6000, 0x385B8000, 0x385BA000, 0x385BC000, 0x385BE000, + 0x385C0000, 0x385C2000, 0x385C4000, 0x385C6000, 0x385C8000, 0x385CA000, 0x385CC000, 0x385CE000, 0x385D0000, 0x385D2000, 0x385D4000, 0x385D6000, 0x385D8000, 0x385DA000, 0x385DC000, 0x385DE000, + 0x385E0000, 0x385E2000, 0x385E4000, 0x385E6000, 0x385E8000, 0x385EA000, 0x385EC000, 0x385EE000, 0x385F0000, 0x385F2000, 0x385F4000, 0x385F6000, 0x385F8000, 0x385FA000, 0x385FC000, 0x385FE000, + 0x38600000, 0x38602000, 0x38604000, 0x38606000, 0x38608000, 0x3860A000, 0x3860C000, 0x3860E000, 0x38610000, 0x38612000, 0x38614000, 0x38616000, 0x38618000, 0x3861A000, 0x3861C000, 0x3861E000, + 0x38620000, 0x38622000, 0x38624000, 0x38626000, 0x38628000, 0x3862A000, 0x3862C000, 0x3862E000, 0x38630000, 0x38632000, 0x38634000, 0x38636000, 0x38638000, 0x3863A000, 0x3863C000, 0x3863E000, + 0x38640000, 0x38642000, 0x38644000, 0x38646000, 0x38648000, 0x3864A000, 0x3864C000, 0x3864E000, 0x38650000, 0x38652000, 0x38654000, 0x38656000, 0x38658000, 0x3865A000, 0x3865C000, 0x3865E000, + 0x38660000, 0x38662000, 0x38664000, 0x38666000, 0x38668000, 0x3866A000, 0x3866C000, 0x3866E000, 0x38670000, 0x38672000, 0x38674000, 0x38676000, 0x38678000, 0x3867A000, 0x3867C000, 0x3867E000, + 0x38680000, 0x38682000, 0x38684000, 0x38686000, 0x38688000, 0x3868A000, 0x3868C000, 0x3868E000, 0x38690000, 0x38692000, 0x38694000, 0x38696000, 0x38698000, 0x3869A000, 0x3869C000, 0x3869E000, + 0x386A0000, 0x386A2000, 0x386A4000, 0x386A6000, 0x386A8000, 0x386AA000, 0x386AC000, 0x386AE000, 0x386B0000, 0x386B2000, 0x386B4000, 0x386B6000, 0x386B8000, 0x386BA000, 0x386BC000, 0x386BE000, + 0x386C0000, 0x386C2000, 0x386C4000, 0x386C6000, 0x386C8000, 0x386CA000, 0x386CC000, 0x386CE000, 0x386D0000, 0x386D2000, 0x386D4000, 0x386D6000, 0x386D8000, 0x386DA000, 0x386DC000, 0x386DE000, + 0x386E0000, 0x386E2000, 0x386E4000, 0x386E6000, 0x386E8000, 0x386EA000, 0x386EC000, 0x386EE000, 0x386F0000, 0x386F2000, 0x386F4000, 0x386F6000, 0x386F8000, 0x386FA000, 0x386FC000, 0x386FE000, + 0x38700000, 0x38702000, 0x38704000, 0x38706000, 0x38708000, 0x3870A000, 0x3870C000, 0x3870E000, 0x38710000, 0x38712000, 0x38714000, 0x38716000, 0x38718000, 0x3871A000, 0x3871C000, 0x3871E000, + 0x38720000, 0x38722000, 0x38724000, 0x38726000, 0x38728000, 0x3872A000, 0x3872C000, 0x3872E000, 0x38730000, 0x38732000, 0x38734000, 0x38736000, 0x38738000, 0x3873A000, 0x3873C000, 0x3873E000, + 0x38740000, 0x38742000, 0x38744000, 0x38746000, 0x38748000, 0x3874A000, 0x3874C000, 0x3874E000, 0x38750000, 0x38752000, 0x38754000, 0x38756000, 0x38758000, 0x3875A000, 0x3875C000, 0x3875E000, + 0x38760000, 0x38762000, 0x38764000, 0x38766000, 0x38768000, 0x3876A000, 0x3876C000, 0x3876E000, 0x38770000, 0x38772000, 0x38774000, 0x38776000, 0x38778000, 0x3877A000, 0x3877C000, 0x3877E000, + 0x38780000, 0x38782000, 0x38784000, 0x38786000, 0x38788000, 0x3878A000, 0x3878C000, 0x3878E000, 0x38790000, 0x38792000, 0x38794000, 0x38796000, 0x38798000, 0x3879A000, 0x3879C000, 0x3879E000, + 0x387A0000, 0x387A2000, 0x387A4000, 0x387A6000, 0x387A8000, 0x387AA000, 0x387AC000, 0x387AE000, 0x387B0000, 0x387B2000, 0x387B4000, 0x387B6000, 0x387B8000, 0x387BA000, 0x387BC000, 0x387BE000, + 0x387C0000, 0x387C2000, 0x387C4000, 0x387C6000, 0x387C8000, 0x387CA000, 0x387CC000, 0x387CE000, 0x387D0000, 0x387D2000, 0x387D4000, 0x387D6000, 0x387D8000, 0x387DA000, 0x387DC000, 0x387DE000, + 0x387E0000, 0x387E2000, 0x387E4000, 0x387E6000, 0x387E8000, 0x387EA000, 0x387EC000, 0x387EE000, 0x387F0000, 0x387F2000, 0x387F4000, 0x387F6000, 0x387F8000, 0x387FA000, 0x387FC000, 0x387FE000 }; + static const bits::type exponent_table[64] = { + 0x00000000, 0x00800000, 0x01000000, 0x01800000, 0x02000000, 0x02800000, 0x03000000, 0x03800000, 0x04000000, 0x04800000, 0x05000000, 0x05800000, 0x06000000, 0x06800000, 0x07000000, 0x07800000, + 0x08000000, 0x08800000, 0x09000000, 0x09800000, 0x0A000000, 0x0A800000, 0x0B000000, 0x0B800000, 0x0C000000, 0x0C800000, 0x0D000000, 0x0D800000, 0x0E000000, 0x0E800000, 0x0F000000, 0x47800000, + 0x80000000, 0x80800000, 0x81000000, 0x81800000, 0x82000000, 0x82800000, 0x83000000, 0x83800000, 0x84000000, 0x84800000, 0x85000000, 0x85800000, 0x86000000, 0x86800000, 0x87000000, 0x87800000, + 0x88000000, 0x88800000, 0x89000000, 0x89800000, 0x8A000000, 0x8A800000, 0x8B000000, 0x8B800000, 0x8C000000, 0x8C800000, 0x8D000000, 0x8D800000, 0x8E000000, 0x8E800000, 0x8F000000, 0xC7800000 }; + static const unsigned short offset_table[64] = { + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, + 0, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024, 1024 }; + bits::type fbits = mantissa_table[offset_table[value>>10]+(value&0x3FF)] + exponent_table[value>>10]; + #endif + float out; + std::memcpy(&out, &fbits, sizeof(float)); + return out; + #endif + } + + /// Convert half-precision to IEEE double-precision. + /// \param value half-precision value to convert + /// \return double-precision value + inline double half2float_impl(unsigned int value, double, true_type) + { + #if HALF_ENABLE_F16C_INTRINSICS + return _mm_cvtsd_f64(_mm_cvtps_pd(_mm_cvtph_ps(_mm_cvtsi32_si128(value)))); + #else + uint32 hi = static_cast(value&0x8000) << 16; + unsigned int abs = value & 0x7FFF; + if(abs) + { + hi |= 0x3F000000 << static_cast(abs>=0x7C00); + for(; abs<0x400; abs<<=1,hi-=0x100000) ; + hi += static_cast(abs) << 10; + } + bits::type dbits = static_cast::type>(hi) << 32; + double out; + std::memcpy(&out, &dbits, sizeof(double)); + return out; + #endif + } + + /// Convert half-precision to non-IEEE floating-point. + /// \tparam T type to convert to (builtin integer type) + /// \param value half-precision value to convert + /// \return floating-point value + template T half2float_impl(unsigned int value, T, ...) + { + T out; + unsigned int abs = value & 0x7FFF; + if(abs > 0x7C00) + out = (std::numeric_limits::has_signaling_NaN && !(abs&0x200)) ? std::numeric_limits::signaling_NaN() : + std::numeric_limits::has_quiet_NaN ? std::numeric_limits::quiet_NaN() : T(); + else if(abs == 0x7C00) + out = std::numeric_limits::has_infinity ? std::numeric_limits::infinity() : std::numeric_limits::max(); + else if(abs > 0x3FF) + out = std::ldexp(static_cast((abs&0x3FF)|0x400), (abs>>10)-25); + else + out = std::ldexp(static_cast(abs), -24); + return (value&0x8000) ? -out : out; + } + + /// Convert half-precision to floating-point. + /// \tparam T type to convert to (builtin integer type) + /// \param value half-precision value to convert + /// \return floating-point value + template T half2float(unsigned int value) + { + return half2float_impl(value, T(), bool_type::is_iec559&&sizeof(typename bits::type)==sizeof(T)>()); + } + + /// Convert half-precision floating-point to integer. + /// \tparam R rounding mode to use + /// \tparam E `true` for round to even, `false` for round away from zero + /// \tparam I `true` to raise INEXACT exception (if inexact), `false` to never raise it + /// \tparam T type to convert to (buitlin integer type with at least 16 bits precision, excluding any implicit sign bits) + /// \param value half-precision value to convert + /// \return rounded integer value + /// \exception FE_INVALID if value is not representable in type \a T + /// \exception FE_INEXACT if value had to be rounded and \a I is `true` + template T half2int(unsigned int value) + { + unsigned int abs = value & 0x7FFF; + if(abs >= 0x7C00) + { + raise(FE_INVALID); + return (value&0x8000) ? std::numeric_limits::min() : std::numeric_limits::max(); + } + if(abs < 0x3800) + { + raise(FE_INEXACT, I); + return (R==std::round_toward_infinity) ? T(~(value>>15)&(abs!=0)) : + (R==std::round_toward_neg_infinity) ? -T(value>0x8000) : + T(); + } + int exp = 25 - (abs>>10); + unsigned int m = (value&0x3FF) | 0x400; + int32 i = static_cast((exp<=0) ? (m<<-exp) : ((m+( + (R==std::round_to_nearest) ? ((1<<(exp-1))-(~(m>>exp)&E)) : + (R==std::round_toward_infinity) ? (((1<>15)-1)) : + (R==std::round_toward_neg_infinity) ? (((1<>15)) : 0))>>exp)); + if((!std::numeric_limits::is_signed && (value&0x8000)) || (std::numeric_limits::digits<16 && + ((value&0x8000) ? (-i::min()) : (i>std::numeric_limits::max())))) + raise(FE_INVALID); + else if(I && exp > 0 && (m&((1<((value&0x8000) ? -i : i); + } + + /// \} + /// \name Mathematics + /// \{ + + /// upper part of 64-bit multiplication. + /// \tparam R rounding mode to use + /// \param x first factor + /// \param y second factor + /// \return upper 32 bit of \a x * \a y + template uint32 mulhi(uint32 x, uint32 y) + { + uint32 xy = (x>>16) * (y&0xFFFF), yx = (x&0xFFFF) * (y>>16), c = (xy&0xFFFF) + (yx&0xFFFF) + (((x&0xFFFF)*(y&0xFFFF))>>16); + return (x>>16)*(y>>16) + (xy>>16) + (yx>>16) + (c>>16) + + ((R==std::round_to_nearest) ? ((c>>15)&1) : (R==std::round_toward_infinity) ? ((c&0xFFFF)!=0) : 0); + } + + /// 64-bit multiplication. + /// \param x first factor + /// \param y second factor + /// \return upper 32 bit of \a x * \a y rounded to nearest + inline uint32 multiply64(uint32 x, uint32 y) + { + #if HALF_ENABLE_CPP11_LONG_LONG + return static_cast((static_cast(x)*static_cast(y)+0x80000000)>>32); + #else + return mulhi(x, y); + #endif + } + + /// 64-bit division. + /// \param x upper 32 bit of dividend + /// \param y divisor + /// \param s variable to store sticky bit for rounding + /// \return (\a x << 32) / \a y + inline uint32 divide64(uint32 x, uint32 y, int &s) + { + #if HALF_ENABLE_CPP11_LONG_LONG + unsigned long long xx = static_cast(x) << 32; + return s = (xx%y!=0), static_cast(xx/y); + #else + y >>= 1; + uint32 rem = x, div = 0; + for(unsigned int i=0; i<32; ++i) + { + div <<= 1; + if(rem >= y) + { + rem -= y; + div |= 1; + } + rem <<= 1; + } + return s = rem > 1, div; + #endif + } + + /// Half precision positive modulus. + /// \tparam Q `true` to compute full quotient, `false` else + /// \tparam R `true` to compute signed remainder, `false` for positive remainder + /// \param x first operand as positive finite half-precision value + /// \param y second operand as positive finite half-precision value + /// \param quo adress to store quotient at, `nullptr` if \a Q `false` + /// \return modulus of \a x / \a y + template unsigned int mod(unsigned int x, unsigned int y, int *quo = NULL) + { + unsigned int q = 0; + if(x > y) + { + int absx = x, absy = y, expx = 0, expy = 0; + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + expx += absx >> 10; + expy += absy >> 10; + int mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + for(int d=expx-expy; d; --d) + { + if(!Q && mx == my) + return 0; + if(mx >= my) + { + mx -= my; + q += Q; + } + mx <<= 1; + q <<= static_cast(Q); + } + if(!Q && mx == my) + return 0; + if(mx >= my) + { + mx -= my; + ++q; + } + if(Q) + { + q &= (1<<(std::numeric_limits::digits-1)) - 1; + if(!mx) + return *quo = q, 0; + } + for(; mx<0x400; mx<<=1,--expy) ; + x = (expy>0) ? ((expy<<10)|(mx&0x3FF)) : (mx>>(1-expy)); + } + if(R) + { + unsigned int a, b; + if(y < 0x800) + { + a = (x<0x400) ? (x<<1) : (x+0x400); + b = y; + } + else + { + a = x; + b = y - 0x400; + } + if(a > b || (a == b && (q&1))) + { + int exp = (y>>10) + (y<=0x3FF), d = exp - (x>>10) - (x<=0x3FF); + int m = (((y&0x3FF)|((y>0x3FF)<<10))<<1) - (((x&0x3FF)|((x>0x3FF)<<10))<<(1-d)); + for(; m<0x800 && exp>1; m<<=1,--exp) ; + x = 0x8000 + ((exp-1)<<10) + (m>>1); + q += Q; + } + } + if(Q) + *quo = q; + return x; + } + + /// Fixed point square root. + /// \tparam F number of fractional bits + /// \param r radicand in Q1.F fixed point format + /// \param exp exponent + /// \return square root as Q1.F/2 + template uint32 sqrt(uint32 &r, int &exp) + { + int i = exp & 1; + r <<= i; + exp = (exp-i) / 2; + uint32 m = 0; + for(uint32 bit=static_cast(1)<>=2) + { + if(r < m+bit) + m >>= 1; + else + { + r -= m + bit; + m = (m>>1) + bit; + } + } + return m; + } + + /// Fixed point binary exponential. + /// This uses the BKM algorithm in E-mode. + /// \param m exponent in [0,1) as Q0.31 + /// \param n number of iterations (at most 32) + /// \return 2 ^ \a m as Q1.31 + inline uint32 exp2(uint32 m, unsigned int n = 32) + { + static const uint32 logs[] = { + 0x80000000, 0x4AE00D1D, 0x2934F098, 0x15C01A3A, 0x0B31FB7D, 0x05AEB4DD, 0x02DCF2D1, 0x016FE50B, + 0x00B84E23, 0x005C3E10, 0x002E24CA, 0x001713D6, 0x000B8A47, 0x0005C53B, 0x0002E2A3, 0x00017153, + 0x0000B8AA, 0x00005C55, 0x00002E2B, 0x00001715, 0x00000B8B, 0x000005C5, 0x000002E3, 0x00000171, + 0x000000B9, 0x0000005C, 0x0000002E, 0x00000017, 0x0000000C, 0x00000006, 0x00000003, 0x00000001 }; + if(!m) + return 0x80000000; + uint32 mx = 0x80000000, my = 0; + for(unsigned int i=1; i> i; + } + } + return mx; + } + + /// Fixed point binary logarithm. + /// This uses the BKM algorithm in L-mode. + /// \param m mantissa in [1,2) as Q1.30 + /// \param n number of iterations (at most 32) + /// \return log2(\a m) as Q0.31 + inline uint32 log2(uint32 m, unsigned int n = 32) + { + static const uint32 logs[] = { + 0x80000000, 0x4AE00D1D, 0x2934F098, 0x15C01A3A, 0x0B31FB7D, 0x05AEB4DD, 0x02DCF2D1, 0x016FE50B, + 0x00B84E23, 0x005C3E10, 0x002E24CA, 0x001713D6, 0x000B8A47, 0x0005C53B, 0x0002E2A3, 0x00017153, + 0x0000B8AA, 0x00005C55, 0x00002E2B, 0x00001715, 0x00000B8B, 0x000005C5, 0x000002E3, 0x00000171, + 0x000000B9, 0x0000005C, 0x0000002E, 0x00000017, 0x0000000C, 0x00000006, 0x00000003, 0x00000001 }; + if(m == 0x40000000) + return 0; + uint32 mx = 0x40000000, my = 0; + for(unsigned int i=1; i>i); + if(mz <= m) + { + mx = mz; + my += logs[i]; + } + } + return my; + } + + /// Fixed point sine and cosine. + /// This uses the CORDIC algorithm in rotation mode. + /// \param mz angle in [-pi/2,pi/2] as Q1.30 + /// \param n number of iterations (at most 31) + /// \return sine and cosine of \a mz as Q1.30 + inline std::pair sincos(uint32 mz, unsigned int n = 31) + { + static const uint32 angles[] = { + 0x3243F6A9, 0x1DAC6705, 0x0FADBAFD, 0x07F56EA7, 0x03FEAB77, 0x01FFD55C, 0x00FFFAAB, 0x007FFF55, + 0x003FFFEB, 0x001FFFFD, 0x00100000, 0x00080000, 0x00040000, 0x00020000, 0x00010000, 0x00008000, + 0x00004000, 0x00002000, 0x00001000, 0x00000800, 0x00000400, 0x00000200, 0x00000100, 0x00000080, + 0x00000040, 0x00000020, 0x00000010, 0x00000008, 0x00000004, 0x00000002, 0x00000001 }; + uint32 mx = 0x26DD3B6A, my = 0; + for(unsigned int i=0; i0x3FF)<<10); + int exp = (abs>>10) + (abs<=0x3FF) - 15; + if(abs < 0x3A48) + return k = 0, m << (exp+20); + #if HALF_ENABLE_CPP11_LONG_LONG + unsigned long long y = m * 0xA2F9836E4E442, mask = (1ULL<<(62-exp)) - 1, yi = (y+(mask>>1)) & ~mask, f = y - yi; + uint32 sign = -static_cast(f>>63); + k = static_cast(yi>>(62-exp)); + return (multiply64(static_cast((sign ? -f : f)>>(31-exp)), 0xC90FDAA2)^sign) - sign; + #else + uint32 yh = m*0xA2F98 + mulhi(m, 0x36E4E442), yl = (m*0x36E4E442) & 0xFFFFFFFF; + uint32 mask = (static_cast(1)<<(30-exp)) - 1, yi = (yh+(mask>>1)) & ~mask, sign = -static_cast(yi>yh); + k = static_cast(yi>>(30-exp)); + uint32 fh = (yh^sign) + (yi^~sign) - ~sign, fl = (yl^sign) - sign; + return (multiply64((exp>-1) ? (((fh<<(1+exp))&0xFFFFFFFF)|((fl&0xFFFFFFFF)>>(31-exp))) : fh, 0xC90FDAA2)^sign) - sign; + #endif + } + + /// Get arguments for atan2 function. + /// \param abs half-precision floating-point value + /// \return \a abs and sqrt(1 - \a abs^2) as Q0.30 + inline std::pair atan2_args(unsigned int abs) + { + int exp = -15; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + uint32 my = ((abs&0x3FF)|0x400) << 5, r = my * my; + int rexp = 2 * exp; + r = 0x40000000 - ((rexp>-31) ? ((r>>-rexp)|((r&((static_cast(1)<<-rexp)-1))!=0)) : 1); + for(rexp=0; r<0x40000000; r<<=1,--rexp) ; + uint32 mx = sqrt<30>(r, rexp); + int d = exp - rexp; + if(d < 0) + return std::make_pair((d<-14) ? ((my>>(-d-14))+((my>>(-d-15))&1)) : (my<<(14+d)), (mx<<14)+(r<<13)/mx); + if(d > 0) + return std::make_pair(my<<14, (d>14) ? ((mx>>(d-14))+((mx>>(d-15))&1)) : ((d==14) ? mx : ((mx<<(14-d))+(r<<(13-d))/mx))); + return std::make_pair(my<<13, (mx<<13)+(r<<12)/mx); + } + + /// Get exponentials for hyperbolic computation + /// \param abs half-precision floating-point value + /// \param exp variable to take unbiased exponent of larger result + /// \param n number of BKM iterations (at most 32) + /// \return exp(abs) and exp(-\a abs) as Q1.31 with same exponent + inline std::pair hyperbolic_args(unsigned int abs, int &exp, unsigned int n = 32) + { + uint32 mx = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29), my; + int e = (abs>>10) + (abs<=0x3FF); + if(e < 14) + { + exp = 0; + mx >>= 14 - e; + } + else + { + exp = mx >> (45-e); + mx = (mx<<(e-14)) & 0x7FFFFFFF; + } + mx = exp2(mx, n); + int d = exp << 1, s; + if(mx > 0x80000000) + { + my = divide64(0x80000000, mx, s); + my |= s; + ++d; + } + else + my = mx; + return std::make_pair(mx, (d<31) ? ((my>>d)|((my&((static_cast(1)< unsigned int exp2_post(uint32 m, int exp, bool esign, unsigned int sign = 0, unsigned int n = 32) + { + if(esign) + { + exp = -exp - (m!=0); + if(exp < -25) + return underflow(sign); + else if(exp == -25) + return rounded(sign, 1, m!=0); + } + else if(exp > 15) + return overflow(sign); + if(!m) + return sign | (((exp+=15)>0) ? (exp<<10) : check_underflow(0x200>>-exp)); + m = exp2(m, n); + int s = 0; + if(esign) + m = divide64(0x80000000, m, s); + return fixed2half(m, exp+14, sign, s); + } + + /// Postprocessing for binary logarithm. + /// \tparam R rounding mode to use + /// \tparam L logarithm for base transformation as Q1.31 + /// \param m fractional part of logarithm as Q0.31 + /// \param ilog signed integer part of logarithm + /// \param exp biased exponent of result + /// \param sign sign bit of result + /// \return value base-transformed and converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int log2_post(uint32 m, int ilog, int exp, unsigned int sign = 0) + { + uint32 msign = sign_mask(ilog); + m = (((static_cast(ilog)<<27)+(m>>4))^msign) - msign; + if(!m) + return 0; + for(; m<0x80000000; m<<=1,--exp) ; + int i = m >= L, s; + exp += i; + m >>= 1 + i; + sign ^= msign & 0x8000; + if(exp < -11) + return underflow(sign); + m = divide64(m, L, s); + return fixed2half(m, exp, sign, 1); + } + + /// Hypotenuse square root and postprocessing. + /// \tparam R rounding mode to use + /// \param r mantissa as Q2.30 + /// \param exp biased exponent + /// \return square root converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if value had to be rounded + template unsigned int hypot_post(uint32 r, int exp) + { + int i = r >> 31; + if((exp+=i) > 46) + return overflow(); + if(exp < -34) + return underflow(); + r = (r>>i) | (r&i); + uint32 m = sqrt<30>(r, exp+=15); + return fixed2half(m, exp-1, 0, r!=0); + } + + /// Division and postprocessing for tangents. + /// \tparam R rounding mode to use + /// \param my dividend as Q1.31 + /// \param mx divisor as Q1.31 + /// \param exp biased exponent of result + /// \param sign sign bit of result + /// \return quotient converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int tangent_post(uint32 my, uint32 mx, int exp, unsigned int sign = 0) + { + int i = my >= mx, s; + exp += i; + if(exp > 29) + return overflow(sign); + if(exp < -11) + return underflow(sign); + uint32 m = divide64(my>>(i+1), mx, s); + return fixed2half(m, exp, sign, s); + } + + /// Area function and postprocessing. + /// This computes the value directly in Q2.30 using the representation `asinh|acosh(x) = log(x+sqrt(x^2+|-1))`. + /// \tparam R rounding mode to use + /// \tparam S `true` for asinh, `false` for acosh + /// \param arg half-precision argument + /// \return asinh|acosh(\a arg) converted to half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int area(unsigned int arg) + { + int abs = arg & 0x7FFF, expx = (abs>>10) + (abs<=0x3FF) - 15, expy = -15, ilog, i; + uint32 mx = static_cast((abs&0x3FF)|((abs>0x3FF)<<10)) << 20, my, r; + for(; abs<0x400; abs<<=1,--expy) ; + expy += abs >> 10; + r = ((abs&0x3FF)|0x400) << 5; + r *= r; + i = r >> 31; + expy = 2*expy + i; + r >>= i; + if(S) + { + if(expy < 0) + { + r = 0x40000000 + ((expy>-30) ? ((r>>-expy)|((r&((static_cast(1)<<-expy)-1))!=0)) : 1); + expy = 0; + } + else + { + r += 0x40000000 >> expy; + i = r >> 31; + r = (r>>i) | (r&i); + expy += i; + } + } + else + { + r -= 0x40000000 >> expy; + for(; r<0x40000000; r<<=1,--expy) ; + } + my = sqrt<30>(r, expy); + my = (my<<15) + (r<<14)/my; + if(S) + { + mx >>= expy - expx; + ilog = expy; + } + else + { + my >>= expx - expy; + ilog = expx; + } + my += mx; + i = my >> 31; + static const int G = S && (R==std::round_to_nearest); + return log2_post(log2(my>>i, 26+S+G)+(G<<3), ilog+i, 17, arg&(static_cast(S)<<15)); + } + + /// Class for 1.31 unsigned floating-point computation + struct f31 + { + /// Constructor. + /// \param mant mantissa as 1.31 + /// \param e exponent + HALF_CONSTEXPR f31(uint32 mant, int e) : m(mant), exp(e) {} + + /// Constructor. + /// \param abs unsigned half-precision value + f31(unsigned int abs) : exp(-15) + { + for(; abs<0x400; abs<<=1,--exp) ; + m = static_cast((abs&0x3FF)|0x400) << 21; + exp += (abs>>10); + } + + /// Addition operator. + /// \param a first operand + /// \param b second operand + /// \return \a a + \a b + friend f31 operator+(f31 a, f31 b) + { + if(b.exp > a.exp) + std::swap(a, b); + int d = a.exp - b.exp; + uint32 m = a.m + ((d<32) ? (b.m>>d) : 0); + int i = (m&0xFFFFFFFF) < a.m; + return f31(((m+i)>>i)|0x80000000, a.exp+i); + } + + /// Subtraction operator. + /// \param a first operand + /// \param b second operand + /// \return \a a - \a b + friend f31 operator-(f31 a, f31 b) + { + int d = a.exp - b.exp, exp = a.exp; + uint32 m = a.m - ((d<32) ? (b.m>>d) : 0); + if(!m) + return f31(0, -32); + for(; m<0x80000000; m<<=1,--exp) ; + return f31(m, exp); + } + + /// Multiplication operator. + /// \param a first operand + /// \param b second operand + /// \return \a a * \a b + friend f31 operator*(f31 a, f31 b) + { + uint32 m = multiply64(a.m, b.m); + int i = m >> 31; + return f31(m<<(1-i), a.exp + b.exp + i); + } + + /// Division operator. + /// \param a first operand + /// \param b second operand + /// \return \a a / \a b + friend f31 operator/(f31 a, f31 b) + { + int i = a.m >= b.m, s; + uint32 m = divide64((a.m+i)>>i, b.m, s); + return f31(m, a.exp - b.exp + i - 1); + } + + uint32 m; ///< mantissa as 1.31. + int exp; ///< exponent. + }; + + /// Error function and postprocessing. + /// This computes the value directly in Q1.31 using the approximations given + /// [here](https://en.wikipedia.org/wiki/Error_function#Approximation_with_elementary_functions). + /// \tparam R rounding mode to use + /// \tparam C `true` for comlementary error function, `false` else + /// \param arg half-precision function argument + /// \return approximated value of error function in half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if no other exception occurred + template unsigned int erf(unsigned int arg) + { + unsigned int abs = arg & 0x7FFF, sign = arg & 0x8000; + f31 x(abs), x2 = x * x * f31(0xB8AA3B29, 0), t = f31(0x80000000, 0) / (f31(0x80000000, 0)+f31(0xA7BA054A, -2)*x), t2 = t * t; + f31 e = ((f31(0x87DC2213, 0)*t2+f31(0xB5F0E2AE, 0))*t2+f31(0x82790637, -2)-(f31(0xBA00E2B8, 0)*t2+f31(0x91A98E62, -2))*t) * t / + ((x2.exp<0) ? f31(exp2((x2.exp>-32) ? (x2.m>>-x2.exp) : 0, 30), 0) : f31(exp2((x2.m<>(31-x2.exp))); + return (!C || sign) ? fixed2half(0x80000000-(e.m>>(C-e.exp)), 14+C, sign&(C-1U)) : + (e.exp<-25) ? underflow() : fixed2half(e.m>>1, e.exp+14, 0, e.m&1); + } + + /// Gamma function and postprocessing. + /// This approximates the value of either the gamma function or its logarithm directly in Q1.31. + /// \tparam R rounding mode to use + /// \tparam L `true` for lograithm of gamma function, `false` for gamma function + /// \param arg half-precision floating-point value + /// \return lgamma/tgamma(\a arg) in half-precision + /// \exception FE_OVERFLOW on overflows + /// \exception FE_UNDERFLOW on underflows + /// \exception FE_INEXACT if \a arg is not a positive integer + template unsigned int gamma(unsigned int arg) + { +/* static const double p[] ={ 2.50662827563479526904, 225.525584619175212544, -268.295973841304927459, 80.9030806934622512966, -5.00757863970517583837, 0.0114684895434781459556 }; + double t = arg + 4.65, s = p[0]; + for(unsigned int i=0; i<5; ++i) + s += p[i+1] / (arg+i); + return std::log(s) + (arg-0.5)*std::log(t) - t; +*/ static const f31 pi(0xC90FDAA2, 1), lbe(0xB8AA3B29, 0); + unsigned int abs = arg & 0x7FFF, sign = arg & 0x8000; + bool bsign = sign != 0; + f31 z(abs), x = sign ? (z+f31(0x80000000, 0)) : z, t = x + f31(0x94CCCCCD, 2), s = + f31(0xA06C9901, 1) + f31(0xBBE654E2, -7)/(x+f31(0x80000000, 2)) + f31(0xA1CE6098, 6)/(x+f31(0x80000000, 1)) + + f31(0xE1868CB7, 7)/x - f31(0x8625E279, 8)/(x+f31(0x80000000, 0)) - f31(0xA03E158F, 2)/(x+f31(0xC0000000, 1)); + int i = (s.exp>=2) + (s.exp>=4) + (s.exp>=8) + (s.exp>=16); + s = f31((static_cast(s.exp)<<(31-i))+(log2(s.m>>1, 28)>>i), i) / lbe; + if(x.exp != -1 || x.m != 0x80000000) + { + i = (t.exp>=2) + (t.exp>=4) + (t.exp>=8); + f31 l = f31((static_cast(t.exp)<<(31-i))+(log2(t.m>>1, 30)>>i), i) / lbe; + s = (x.exp<-1) ? (s-(f31(0x80000000, -1)-x)*l) : (s+(x-f31(0x80000000, -1))*l); + } + s = x.exp ? (s-t) : (t-s); + if(bsign) + { + if(z.exp >= 0) + { + sign &= (L|((z.m>>(31-z.exp))&1)) - 1; + for(z=f31((z.m<<(1+z.exp))&0xFFFFFFFF, -1); z.m<0x80000000; z.m<<=1,--z.exp) ; + } + if(z.exp == -1) + z = f31(0x80000000, 0) - z; + if(z.exp < -1) + { + z = z * pi; + z.m = sincos(z.m>>(1-z.exp), 30).first; + for(z.exp=1; z.m<0x80000000; z.m<<=1,--z.exp) ; + } + else + z = f31(0x80000000, 0); + } + if(L) + { + if(bsign) + { + f31 l(0x92868247, 0); + if(z.exp < 0) + { + uint32 m = log2((z.m+1)>>1, 27); + z = f31(-((static_cast(z.exp)<<26)+(m>>5)), 5); + for(; z.m<0x80000000; z.m<<=1,--z.exp) ; + l = l + z / lbe; + } + sign = static_cast(x.exp&&(l.exp(x.exp==0) << 15; + if(s.exp < -24) + return underflow(sign); + if(s.exp > 15) + return overflow(sign); + } + } + else + { + s = s * lbe; + uint32 m; + if(s.exp < 0) + { + m = s.m >> -s.exp; + s.exp = 0; + } + else + { + m = (s.m<>(31-s.exp)); + } + s.m = exp2(m, 27); + if(!x.exp) + s = f31(0x80000000, 0) / s; + if(bsign) + { + if(z.exp < 0) + s = s * z; + s = pi / s; + if(s.exp < -24) + return underflow(sign); + } + else if(z.exp > 0 && !(z.m&((1<<(31-z.exp))-1))) + return ((s.exp+14)<<10) + (s.m>>21); + if(s.exp > 15) + return overflow(sign); + } + return fixed2half(s.m, s.exp+14, sign); + } + /// \} + + template struct half_caster; + } + + /// Half-precision floating-point type. + /// This class implements an IEEE-conformant half-precision floating-point type with the usual arithmetic + /// operators and conversions. It is implicitly convertible to single-precision floating-point, which makes artihmetic + /// expressions and functions with mixed-type operands to be of the most precise operand type. + /// + /// According to the C++98/03 definition, the half type is not a POD type. But according to C++11's less strict and + /// extended definitions it is both a standard layout type and a trivially copyable type (even if not a POD type), which + /// means it can be standard-conformantly copied using raw binary copies. But in this context some more words about the + /// actual size of the type. Although the half is representing an IEEE 16-bit type, it does not neccessarily have to be of + /// exactly 16-bits size. But on any reasonable implementation the actual binary representation of this type will most + /// probably not ivolve any additional "magic" or padding beyond the simple binary representation of the underlying 16-bit + /// IEEE number, even if not strictly guaranteed by the standard. But even then it only has an actual size of 16 bits if + /// your C++ implementation supports an unsigned integer type of exactly 16 bits width. But this should be the case on + /// nearly any reasonable platform. + /// + /// So if your C++ implementation is not totally exotic or imposes special alignment requirements, it is a reasonable + /// assumption that the data of a half is just comprised of the 2 bytes of the underlying IEEE representation. + class half + { + public: + /// \name Construction and assignment + /// \{ + + /// Default constructor. + /// This initializes the half to 0. Although this does not match the builtin types' default-initialization semantics + /// and may be less efficient than no initialization, it is needed to provide proper value-initialization semantics. + HALF_CONSTEXPR half() HALF_NOEXCEPT : data_() {} + + /// Conversion constructor. + /// \param rhs float to convert + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + explicit half(float rhs) : data_(static_cast(detail::float2half(rhs))) {} + + /// Conversion to single-precision. + /// \return single precision value representing expression value + operator float() const { return detail::half2float(data_); } + + /// Assignment operator. + /// \param rhs single-precision value to copy from + /// \return reference to this half + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + half& operator=(float rhs) { data_ = static_cast(detail::float2half(rhs)); return *this; } + + /// \} + /// \name Arithmetic updates + /// \{ + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to add + /// \return reference to this half + /// \exception FE_... according to operator+(half,half) + half& operator+=(half rhs) { return *this = *this + rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to subtract + /// \return reference to this half + /// \exception FE_... according to operator-(half,half) + half& operator-=(half rhs) { return *this = *this - rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to multiply with + /// \return reference to this half + /// \exception FE_... according to operator*(half,half) + half& operator*=(half rhs) { return *this = *this * rhs; } + + /// Arithmetic assignment. + /// \tparam T type of concrete half expression + /// \param rhs half expression to divide by + /// \return reference to this half + /// \exception FE_... according to operator/(half,half) + half& operator/=(half rhs) { return *this = *this / rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to add + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator+=(float rhs) { return *this = *this + rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to subtract + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator-=(float rhs) { return *this = *this - rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to multiply with + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator*=(float rhs) { return *this = *this * rhs; } + + /// Arithmetic assignment. + /// \param rhs single-precision value to divide by + /// \return reference to this half + /// \exception FE_... according to operator=() + half& operator/=(float rhs) { return *this = *this / rhs; } + + /// \} + /// \name Increment and decrement + /// \{ + + /// Prefix increment. + /// \return incremented half value + /// \exception FE_... according to operator+(half,half) + half& operator++() { return *this = *this + half(detail::binary, 0x3C00); } + + /// Prefix decrement. + /// \return decremented half value + /// \exception FE_... according to operator-(half,half) + half& operator--() { return *this = *this + half(detail::binary, 0xBC00); } + + /// Postfix increment. + /// \return non-incremented half value + /// \exception FE_... according to operator+(half,half) + half operator++(int) { half out(*this); ++*this; return out; } + + /// Postfix decrement. + /// \return non-decremented half value + /// \exception FE_... according to operator-(half,half) + half operator--(int) { half out(*this); --*this; return out; } + /// \} + + private: + /// Rounding mode to use + static const std::float_round_style round_style = (std::float_round_style)(HALF_ROUND_STYLE); + + /// Constructor. + /// \param bits binary representation to set half to + HALF_CONSTEXPR half(detail::binary_t, unsigned int bits) HALF_NOEXCEPT : data_(static_cast(bits)) {} + + /// Internal binary representation + detail::uint16 data_; + + #ifndef HALF_DOXYGEN_ONLY + friend HALF_CONSTEXPR_NOERR bool operator==(half, half); + friend HALF_CONSTEXPR_NOERR bool operator!=(half, half); + friend HALF_CONSTEXPR_NOERR bool operator<(half, half); + friend HALF_CONSTEXPR_NOERR bool operator>(half, half); + friend HALF_CONSTEXPR_NOERR bool operator<=(half, half); + friend HALF_CONSTEXPR_NOERR bool operator>=(half, half); + friend HALF_CONSTEXPR half operator-(half); + friend half operator+(half, half); + friend half operator-(half, half); + friend half operator*(half, half); + friend half operator/(half, half); + template friend std::basic_ostream& operator<<(std::basic_ostream&, half); + template friend std::basic_istream& operator>>(std::basic_istream&, half&); + friend HALF_CONSTEXPR half fabs(half); + friend half fmod(half, half); + friend half remainder(half, half); + friend half remquo(half, half, int*); + friend half fma(half, half, half); + friend HALF_CONSTEXPR_NOERR half fmax(half, half); + friend HALF_CONSTEXPR_NOERR half fmin(half, half); + friend half fdim(half, half); + friend half nanh(const char*); + friend half exp(half); + friend half exp2(half); + friend half expm1(half); + friend half log(half); + friend half log10(half); + friend half log2(half); + friend half log1p(half); + friend half sqrt(half); + friend half rsqrt(half); + friend half cbrt(half); + friend half hypot(half, half); + friend half hypot(half, half, half); + friend half pow(half, half); + friend void sincos(half, half*, half*); + friend half sin(half); + friend half cos(half); + friend half tan(half); + friend half asin(half); + friend half acos(half); + friend half atan(half); + friend half atan2(half, half); + friend half sinh(half); + friend half cosh(half); + friend half tanh(half); + friend half asinh(half); + friend half acosh(half); + friend half atanh(half); + friend half erf(half); + friend half erfc(half); + friend half lgamma(half); + friend half tgamma(half); + friend half ceil(half); + friend half floor(half); + friend half trunc(half); + friend half round(half); + friend long lround(half); + friend half rint(half); + friend long lrint(half); + friend half nearbyint(half); + #ifdef HALF_ENABLE_CPP11_LONG_LONG + friend long long llround(half); + friend long long llrint(half); + #endif + friend half frexp(half, int*); + friend half scalbln(half, long); + friend half modf(half, half*); + friend int ilogb(half); + friend half logb(half); + friend half nextafter(half, half); + friend half nexttoward(half, long double); + friend HALF_CONSTEXPR half copysign(half, half); + friend HALF_CONSTEXPR int fpclassify(half); + friend HALF_CONSTEXPR bool isfinite(half); + friend HALF_CONSTEXPR bool isinf(half); + friend HALF_CONSTEXPR bool isnan(half); + friend HALF_CONSTEXPR bool isnormal(half); + friend HALF_CONSTEXPR bool signbit(half); + friend HALF_CONSTEXPR bool isgreater(half, half); + friend HALF_CONSTEXPR bool isgreaterequal(half, half); + friend HALF_CONSTEXPR bool isless(half, half); + friend HALF_CONSTEXPR bool islessequal(half, half); + friend HALF_CONSTEXPR bool islessgreater(half, half); + template friend struct detail::half_caster; + friend class std::numeric_limits; + #if HALF_ENABLE_CPP11_HASH + friend struct std::hash; + #endif + #if HALF_ENABLE_CPP11_USER_LITERALS + friend half literal::operator "" _h(long double); + #endif + #endif + }; + +#if HALF_ENABLE_CPP11_USER_LITERALS + namespace literal + { + /// Half literal. + /// While this returns a properly rounded half-precision value, half literals can unfortunately not be constant + /// expressions due to rather involved conversions. So don't expect this to be a literal literal without involving + /// conversion operations at runtime. It is a convenience feature, not a performance optimization. + /// \param value literal value + /// \return half with of given value (possibly rounded) + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator "" _h(long double value) { return half(detail::binary, detail::float2half(value)); } + } +#endif + + namespace detail + { + /// Helper class for half casts. + /// This class template has to be specialized for all valid cast arguments to define an appropriate static + /// `cast` member function and a corresponding `type` member denoting its return type. + /// \tparam T destination type + /// \tparam U source type + /// \tparam R rounding mode to use + template struct half_caster {}; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast from non-arithmetic type unsupported"); + #endif + + static half cast(U arg) { return cast_impl(arg, is_float()); }; + + private: + static half cast_impl(U arg, true_type) { return half(binary, float2half(arg)); } + static half cast_impl(U arg, false_type) { return half(binary, int2half(arg)); } + }; + template struct half_caster + { + #if HALF_ENABLE_CPP11_STATIC_ASSERT && HALF_ENABLE_CPP11_TYPE_TRAITS + static_assert(std::is_arithmetic::value, "half_cast to non-arithmetic type unsupported"); + #endif + + static T cast(half arg) { return cast_impl(arg, is_float()); } + + private: + static T cast_impl(half arg, true_type) { return half2float(arg.data_); } + static T cast_impl(half arg, false_type) { return half2int(arg.data_); } + }; + template struct half_caster + { + static half cast(half arg) { return arg; } + }; + } +} + +/// Extensions to the C++ standard library. +namespace std +{ + /// Numeric limits for half-precision floats. + /// **See also:** Documentation for [std::numeric_limits](https://en.cppreference.com/w/cpp/types/numeric_limits) + template<> class numeric_limits + { + public: + /// Is template specialization. + static HALF_CONSTEXPR_CONST bool is_specialized = true; + + /// Supports signed values. + static HALF_CONSTEXPR_CONST bool is_signed = true; + + /// Is not an integer type. + static HALF_CONSTEXPR_CONST bool is_integer = false; + + /// Is not exact. + static HALF_CONSTEXPR_CONST bool is_exact = false; + + /// Doesn't provide modulo arithmetic. + static HALF_CONSTEXPR_CONST bool is_modulo = false; + + /// Has a finite set of values. + static HALF_CONSTEXPR_CONST bool is_bounded = true; + + /// IEEE conformant. + static HALF_CONSTEXPR_CONST bool is_iec559 = true; + + /// Supports infinity. + static HALF_CONSTEXPR_CONST bool has_infinity = true; + + /// Supports quiet NaNs. + static HALF_CONSTEXPR_CONST bool has_quiet_NaN = true; + + /// Supports signaling NaNs. + static HALF_CONSTEXPR_CONST bool has_signaling_NaN = true; + + /// Supports subnormal values. + static HALF_CONSTEXPR_CONST float_denorm_style has_denorm = denorm_present; + + /// Supports no denormalization detection. + static HALF_CONSTEXPR_CONST bool has_denorm_loss = false; + + #if HALF_ERRHANDLING_THROWS + static HALF_CONSTEXPR_CONST bool traps = true; + #else + /// Traps only if [HALF_ERRHANDLING_THROW_...](\ref HALF_ERRHANDLING_THROW_INVALID) is acitvated. + static HALF_CONSTEXPR_CONST bool traps = false; + #endif + + /// Does not support no pre-rounding underflow detection. + static HALF_CONSTEXPR_CONST bool tinyness_before = false; + + /// Rounding mode. + static HALF_CONSTEXPR_CONST float_round_style round_style = half_float::half::round_style; + + /// Significant digits. + static HALF_CONSTEXPR_CONST int digits = 11; + + /// Significant decimal digits. + static HALF_CONSTEXPR_CONST int digits10 = 3; + + /// Required decimal digits to represent all possible values. + static HALF_CONSTEXPR_CONST int max_digits10 = 5; + + /// Number base. + static HALF_CONSTEXPR_CONST int radix = 2; + + /// One more than smallest exponent. + static HALF_CONSTEXPR_CONST int min_exponent = -13; + + /// Smallest normalized representable power of 10. + static HALF_CONSTEXPR_CONST int min_exponent10 = -4; + + /// One more than largest exponent + static HALF_CONSTEXPR_CONST int max_exponent = 16; + + /// Largest finitely representable power of 10. + static HALF_CONSTEXPR_CONST int max_exponent10 = 4; + + /// Smallest positive normal value. + static HALF_CONSTEXPR half_float::half min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0400); } + + /// Smallest finite value. + static HALF_CONSTEXPR half_float::half lowest() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0xFBFF); } + + /// Largest finite value. + static HALF_CONSTEXPR half_float::half max() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7BFF); } + + /// Difference between 1 and next representable value. + static HALF_CONSTEXPR half_float::half epsilon() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x1400); } + + /// Maximum rounding error in ULP (units in the last place). + static HALF_CONSTEXPR half_float::half round_error() HALF_NOTHROW + { return half_float::half(half_float::detail::binary, (round_style==std::round_to_nearest) ? 0x3800 : 0x3C00); } + + /// Positive infinity. + static HALF_CONSTEXPR half_float::half infinity() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7C00); } + + /// Quiet NaN. + static HALF_CONSTEXPR half_float::half quiet_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7FFF); } + + /// Signaling NaN. + static HALF_CONSTEXPR half_float::half signaling_NaN() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x7DFF); } + + /// Smallest positive subnormal value. + static HALF_CONSTEXPR half_float::half denorm_min() HALF_NOTHROW { return half_float::half(half_float::detail::binary, 0x0001); } + }; + +#if HALF_ENABLE_CPP11_HASH + /// Hash function for half-precision floats. + /// This is only defined if C++11 `std::hash` is supported and enabled. + /// + /// **See also:** Documentation for [std::hash](https://en.cppreference.com/w/cpp/utility/hash) + template<> struct hash + { + /// Type of function argument. + typedef half_float::half argument_type; + + /// Function return type. + typedef size_t result_type; + + /// Compute hash function. + /// \param arg half to hash + /// \return hash value + result_type operator()(argument_type arg) const { return hash()(arg.data_&-static_cast(arg.data_!=0x8000)); } + }; +#endif +} + +namespace half_float +{ + /// \anchor compop + /// \name Comparison operators + /// \{ + + /// Comparison for equality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands equal + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator==(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && (x.data_==y.data_ || !((x.data_|y.data_)&0x7FFF)); + } + + /// Comparison for inequality. + /// \param x first operand + /// \param y second operand + /// \retval true if operands not equal + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator!=(half x, half y) + { + return detail::compsignal(x.data_, y.data_) || (x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF)); + } + + /// Comparison for less than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator<(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) < ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + + /// Comparison for greater than. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator>(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) > ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + + /// Comparison for less equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator<=(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) <= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + + /// Comparison for greater equal. + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else + /// \exception FE_INVALID if \a x or \a y is NaN + inline HALF_CONSTEXPR_NOERR bool operator>=(half x, half y) + { + return !detail::compsignal(x.data_, y.data_) && + ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) >= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)); + } + + /// \} + /// \anchor arithmetics + /// \name Arithmetic operators + /// \{ + + /// Identity. + /// \param arg operand + /// \return unchanged operand + inline HALF_CONSTEXPR half operator+(half arg) { return arg; } + + /// Negation. + /// \param arg operand + /// \return negated operand + inline HALF_CONSTEXPR half operator-(half arg) { return half(detail::binary, arg.data_^0x8000); } + + /// Addition. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return sum of half expressions + /// \exception FE_INVALID if \a x and \a y are infinities with different signs or signaling NaNs + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator+(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)+detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF; + bool sub = ((x.data_^y.data_)&0x8000) != 0; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : (absy!=0x7C00) ? x.data_ : + (sub && absx==0x7C00) ? detail::invalid() : y.data_); + if(!absx) + return absy ? y : half(detail::binary, (half::round_style==std::round_toward_neg_infinity) ? (x.data_|y.data_) : (x.data_&y.data_)); + if(!absy) + return x; + unsigned int sign = ((sub && absy>absx) ? y.data_ : x.data_) & 0x8000; + if(absy > absx) + std::swap(absx, absy); + int exp = (absx>>10) + (absx<=0x3FF), d = exp - (absy>>10) - (absy<=0x3FF), mx = ((absx&0x3FF)|((absx>0x3FF)<<10)) << 3, my; + if(d < 13) + { + my = ((absy&0x3FF)|((absy>0x3FF)<<10)) << 3; + my = (my>>d) | ((my&((1<(half::round_style==std::round_toward_neg_infinity)<<15); + for(; mx<0x2000 && exp>1; mx<<=1,--exp) ; + } + else + { + mx += my; + int i = mx >> 14; + if((exp+=i) > 30) + return half(detail::binary, detail::overflow(sign)); + mx = (mx>>i) | (mx&i); + } + return half(detail::binary, detail::rounded(sign+((exp-1)<<10)+(mx>>3), (mx>>2)&1, (mx&0x3)!=0)); + #endif + } + + /// Subtraction. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return difference of half expressions + /// \exception FE_INVALID if \a x and \a y are infinities with equal signs or signaling NaNs + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator-(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)-detail::half2float(y.data_))); + #else + return x + -y; + #endif + } + + /// Multiplication. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return product of half expressions + /// \exception FE_INVALID if multiplying 0 with infinity or if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator*(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)*detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = -16; + unsigned int sign = (x.data_^y.data_) & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + ((absx==0x7C00 && !absy)||(absy==0x7C00 && !absx)) ? detail::invalid() : (sign|0x7C00)); + if(!absx || !absy) + return half(detail::binary, sign); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + detail::uint32 m = static_cast((absx&0x3FF)|0x400) * static_cast((absy&0x3FF)|0x400); + int i = m >> 21, s = m & i; + exp += (absx>>10) + (absy>>10) + i; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -11) + return half(detail::binary, detail::underflow(sign)); + return half(detail::binary, detail::fixed2half(m>>i, exp, sign, s)); + #endif + } + + /// Division. + /// This operation is exact to rounding for all rounding modes. + /// \param x left operand + /// \param y right operand + /// \return quotient of half expressions + /// \exception FE_INVALID if dividing 0s or infinities with each other or if \a x or \a y is signaling NaN + /// \exception FE_DIVBYZERO if dividing finite value by 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half operator/(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::half2float(x.data_)/detail::half2float(y.data_))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = 14; + unsigned int sign = (x.data_^y.data_) & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==absy) ? detail::invalid() : (sign|((absx==0x7C00) ? 0x7C00 : 0))); + if(!absx) + return half(detail::binary, absy ? sign : detail::invalid()); + if(!absy) + return half(detail::binary, detail::pole(sign)); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,++exp) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + int i = mx < my; + exp += (absx>>10) - (absy>>10) - i; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -11) + return half(detail::binary, detail::underflow(sign)); + mx <<= 12 + i; + my <<= 1; + return half(detail::binary, detail::fixed2half(mx/my, exp, sign, mx%my!=0)); + #endif + } + + /// \} + /// \anchor streaming + /// \name Input and output + /// \{ + + /// Output operator. + /// This uses the built-in functionality for streaming out floating-point numbers. + /// \param out output stream to write into + /// \param arg half expression to write + /// \return reference to output stream + template std::basic_ostream& operator<<(std::basic_ostream &out, half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return out << detail::half2float(arg.data_); + #else + return out << detail::half2float(arg.data_); + #endif + } + + /// Input operator. + /// This uses the built-in functionality for streaming in floating-point numbers, specifically double precision floating + /// point numbers (unless overridden with [HALF_ARITHMETIC_TYPE](\ref HALF_ARITHMETIC_TYPE)). So the input string is first + /// rounded to double precision using the underlying platform's current floating-point rounding mode before being rounded + /// to half-precision using the library's half-precision rounding mode. + /// \param in input stream to read from + /// \param arg half to read into + /// \return reference to input stream + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template std::basic_istream& operator>>(std::basic_istream &in, half &arg) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t f; + #else + double f; + #endif + if(in >> f) + arg.data_ = detail::float2half(f); + return in; + } + + /// \} + /// \anchor basic + /// \name Basic mathematical operations + /// \{ + + /// Absolute value. + /// **See also:** Documentation for [std::fabs](https://en.cppreference.com/w/cpp/numeric/math/fabs). + /// \param arg operand + /// \return absolute value of \a arg + inline HALF_CONSTEXPR half fabs(half arg) { return half(detail::binary, arg.data_&0x7FFF); } + + /// Absolute value. + /// **See also:** Documentation for [std::abs](https://en.cppreference.com/w/cpp/numeric/math/fabs). + /// \param arg operand + /// \return absolute value of \a arg + inline HALF_CONSTEXPR half abs(half arg) { return fabs(arg); } + + /// Remainder of division. + /// **See also:** Documentation for [std::fmod](https://en.cppreference.com/w/cpp/numeric/math/fmod). + /// \param x first operand + /// \param y second operand + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half fmod(half x, half y) + { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, sign = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : x.data_); + if(!absy) + return half(detail::binary, detail::invalid()); + if(!absx) + return x; + if(absx == absy) + return half(detail::binary, sign); + return half(detail::binary, sign|detail::mod(absx, absy)); + } + + /// Remainder of division. + /// **See also:** Documentation for [std::remainder](https://en.cppreference.com/w/cpp/numeric/math/remainder). + /// \param x first operand + /// \param y second operand + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half remainder(half x, half y) + { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, sign = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : x.data_); + if(!absy) + return half(detail::binary, detail::invalid()); + if(absx == absy) + return half(detail::binary, sign); + return half(detail::binary, sign^detail::mod(absx, absy)); + } + + /// Remainder of division. + /// **See also:** Documentation for [std::remquo](https://en.cppreference.com/w/cpp/numeric/math/remquo). + /// \param x first operand + /// \param y second operand + /// \param quo address to store some bits of quotient at + /// \return remainder of floating-point division. + /// \exception FE_INVALID if \a x is infinite or \a y is 0 or if \a x or \a y is signaling NaN + inline half remquo(half x, half y, int *quo) + { + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, value = x.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absx==0x7C00) ? detail::invalid() : (*quo = 0, x.data_)); + if(!absy) + return half(detail::binary, detail::invalid()); + bool qsign = ((value^y.data_)&0x8000) != 0; + int q = 1; + if(absx != absy) + value ^= detail::mod(absx, absy, &q); + return *quo = qsign ? -q : q, half(detail::binary, value); + } + + /// Fused multiply add. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::fma](https://en.cppreference.com/w/cpp/numeric/math/fma). + /// \param x first operand + /// \param y second operand + /// \param z third operand + /// \return ( \a x * \a y ) + \a z rounded as one operation. + /// \exception FE_INVALID according to operator*() and operator+() unless any argument is a quiet NaN and no argument is a signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding the final addition + inline half fma(half x, half y, half z) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_), fz = detail::half2float(z.data_); + #if HALF_ENABLE_CPP11_CMATH && FP_FAST_FMA + return half(detail::binary, detail::float2half(std::fma(fx, fy, fz))); + #else + return half(detail::binary, detail::float2half(fx*fy+fz)); + #endif + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, absz = z.data_ & 0x7FFF, exp = -15; + unsigned int sign = (x.data_^y.data_) & 0x8000; + bool sub = ((sign^z.data_)&0x8000) != 0; + if(absx >= 0x7C00 || absy >= 0x7C00 || absz >= 0x7C00) + return (absx>0x7C00 || absy>0x7C00 || absz>0x7C00) ? half(detail::binary, detail::signal(x.data_, y.data_, z.data_)) : + (absx==0x7C00) ? half(detail::binary, (!absy || (sub && absz==0x7C00)) ? detail::invalid() : (sign|0x7C00)) : + (absy==0x7C00) ? half(detail::binary, (!absx || (sub && absz==0x7C00)) ? detail::invalid() : (sign|0x7C00)) : z; + if(!absx || !absy) + return absz ? z : half(detail::binary, (half::round_style==std::round_toward_neg_infinity) ? (z.data_|sign) : (z.data_&sign)); + for(; absx<0x400; absx<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + detail::uint32 m = static_cast((absx&0x3FF)|0x400) * static_cast((absy&0x3FF)|0x400); + int i = m >> 21; + exp += (absx>>10) + (absy>>10) + i; + m <<= 3 - i; + if(absz) + { + int expz = 0; + for(; absz<0x400; absz<<=1,--expz) ; + expz += absz >> 10; + detail::uint32 mz = static_cast((absz&0x3FF)|0x400) << 13; + if(expz > exp || (expz == exp && mz > m)) + { + std::swap(m, mz); + std::swap(exp, expz); + if(sub) + sign = z.data_ & 0x8000; + } + int d = exp - expz; + mz = (d<23) ? ((mz>>d)|((mz&((static_cast(1)<(half::round_style==std::round_toward_neg_infinity)<<15); + for(; m<0x800000; m<<=1,--exp) ; + } + else + { + m += mz; + i = m >> 24; + m = (m>>i) | (m&i); + exp += i; + } + } + if(exp > 30) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -10) + return half(detail::binary, detail::underflow(sign)); + return half(detail::binary, detail::fixed2half(m, exp-1, sign)); + #endif + } + + /// Maximum of half expressions. + /// **See also:** Documentation for [std::fmax](https://en.cppreference.com/w/cpp/numeric/math/fmax). + /// \param x first operand + /// \param y second operand + /// \return maximum of operands, ignoring quiet NaNs + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline HALF_CONSTEXPR_NOERR half fmax(half x, half y) + { + return half(detail::binary, (!isnan(y) && (isnan(x) || (x.data_^(0x8000|(0x8000-(x.data_>>15)))) < + (y.data_^(0x8000|(0x8000-(y.data_>>15)))))) ? detail::select(y.data_, x.data_) : detail::select(x.data_, y.data_)); + } + + /// Minimum of half expressions. + /// **See also:** Documentation for [std::fmin](https://en.cppreference.com/w/cpp/numeric/math/fmin). + /// \param x first operand + /// \param y second operand + /// \return minimum of operands, ignoring quiet NaNs + /// \exception FE_INVALID if \a x or \a y is signaling NaN + inline HALF_CONSTEXPR_NOERR half fmin(half x, half y) + { + return half(detail::binary, (!isnan(y) && (isnan(x) || (x.data_^(0x8000|(0x8000-(x.data_>>15)))) > + (y.data_^(0x8000|(0x8000-(y.data_>>15)))))) ? detail::select(y.data_, x.data_) : detail::select(x.data_, y.data_)); + } + + /// Positive difference. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::fdim](https://en.cppreference.com/w/cpp/numeric/math/fdim). + /// \param x first operand + /// \param y second operand + /// \return \a x - \a y or 0 if difference negative + /// \exception FE_... according to operator-(half,half) + inline half fdim(half x, half y) + { + if(isnan(x) || isnan(y)) + return half(detail::binary, detail::signal(x.data_, y.data_)); + return (x.data_^(0x8000|(0x8000-(x.data_>>15)))) <= (y.data_^(0x8000|(0x8000-(y.data_>>15)))) ? half(detail::binary, 0) : (x-y); + } + + /// Get NaN value. + /// **See also:** Documentation for [std::nan](https://en.cppreference.com/w/cpp/numeric/math/nan). + /// \param arg string code + /// \return quiet NaN + inline half nanh(const char *arg) + { + unsigned int value = 0x7FFF; + while(*arg) + value ^= static_cast(*arg++) & 0xFF; + return half(detail::binary, value); + } + + /// \} + /// \anchor exponential + /// \name Exponential functions + /// \{ + + /// Exponential function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::exp](https://en.cppreference.com/w/cpp/numeric/math/exp). + /// \param arg function argument + /// \return e raised to \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half exp(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::exp(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, e = (abs>>10) + (abs<=0x3FF), exp; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00&((arg.data_>>15)-1U)) : detail::signal(arg.data_)); + if(abs >= 0x4C80) + return half(detail::binary, (arg.data_&0x8000) ? detail::underflow() : detail::overflow()); + detail::uint32 m = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29); + if(e < 14) + { + exp = 0; + m >>= 14 - e; + } + else + { + exp = m >> (45-e); + m = (m<<(e-14)) & 0x7FFFFFFF; + } + return half(detail::binary, detail::exp2_post(m, exp, (arg.data_&0x8000)!=0, 0, 26)); + #endif + } + + /// Binary exponential. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::exp2](https://en.cppreference.com/w/cpp/numeric/math/exp2). + /// \param arg function argument + /// \return 2 raised to \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half exp2(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::exp2(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, e = (abs>>10) + (abs<=0x3FF), exp = (abs&0x3FF) + ((abs>0x3FF)<<10); + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00&((arg.data_>>15)-1U)) : detail::signal(arg.data_)); + if(abs >= 0x4E40) + return half(detail::binary, (arg.data_&0x8000) ? detail::underflow() : detail::overflow()); + return half(detail::binary, detail::exp2_post( + (static_cast(exp)<<(6+e))&0x7FFFFFFF, exp>>(25-e), (arg.data_&0x8000)!=0, 0, 28)); + #endif + } + + /// Exponential minus one. + /// This function may be 1 ULP off the correctly rounded exact result in <0.05% of inputs for `std::round_to_nearest` + /// and in <1% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::expm1](https://en.cppreference.com/w/cpp/numeric/math/expm1). + /// \param arg function argument + /// \return e raised to \a arg and subtracted by 1 + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half expm1(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::expm1(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000, e = (abs>>10) + (abs<=0x3FF), exp; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? (0x7C00+(sign>>1)) : detail::signal(arg.data_)); + if(abs >= 0x4A00) + return half(detail::binary, (arg.data_&0x8000) ? detail::rounded(0xBBFF, 1, 1) : detail::overflow()); + detail::uint32 m = detail::multiply64(static_cast((abs&0x3FF)+((abs>0x3FF)<<10))<<21, 0xB8AA3B29); + if(e < 14) + { + exp = 0; + m >>= 14 - e; + } + else + { + exp = m >> (45-e); + m = (m<<(e-14)) & 0x7FFFFFFF; + } + m = detail::exp2(m); + if(sign) + { + int s = 0; + if(m > 0x80000000) + { + ++exp; + m = detail::divide64(0x80000000, m, s); + } + m = 0x80000000 - ((m>>exp)|((m&((static_cast(1)<>exp) : 1; + for(exp+=14; m<0x80000000 && exp; m<<=1,--exp) ; + if(exp > 29) + return half(detail::binary, detail::overflow()); + return half(detail::binary, detail::rounded(sign+(exp<<10)+(m>>21), (m>>20)&1, (m&0xFFFFF)!=0)); + #endif + } + + /// Natural logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log](https://en.cppreference.com/w/cpp/numeric/math/log). + /// \param arg function argument + /// \return logarithm of \a arg to base e + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::log(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + return half(detail::binary, detail::log2_post( + detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 27)+8, exp, 17)); + #endif + } + + /// Common logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log10](https://en.cppreference.com/w/cpp/numeric/math/log10). + /// \param arg function argument + /// \return logarithm of \a arg to base 10 + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log10(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::log10(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + switch(abs) + { + case 0x4900: return half(detail::binary, 0x3C00); + case 0x5640: return half(detail::binary, 0x4000); + case 0x63D0: return half(detail::binary, 0x4200); + case 0x70E2: return half(detail::binary, 0x4400); + } + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + return half(detail::binary, detail::log2_post( + detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 27)+8, exp, 16)); + #endif + } + + /// Binary logarithm. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::log2](https://en.cppreference.com/w/cpp/numeric/math/log2). + /// \param arg function argument + /// \return logarithm of \a arg to base 2 + /// \exception FE_INVALID for signaling NaN or negative argument + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log2(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::log2(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15, s = 0; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(arg.data_ & 0x8000) + return half(detail::binary, (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs >= 0x7C00) + return (abs==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + if(abs == 0x3C00) + return half(detail::binary, 0); + for(; abs<0x400; abs<<=1,--exp) ; + exp += (abs>>10); + if(!(abs&0x3FF)) + { + unsigned int value = static_cast(exp<0) << 15, m = std::abs(exp) << 6; + for(exp=18; m<0x400; m<<=1,--exp) ; + return half(detail::binary, value+(exp<<10)+m); + } + detail::uint32 ilog = exp, sign = detail::sign_mask(ilog), m = + (((ilog<<27)+(detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 28)>>4))^sign) - sign; + if(!m) + return half(detail::binary, 0); + for(exp=14; m<0x8000000 && exp; m<<=1,--exp) ; + for(; m>0xFFFFFFF; m>>=1,++exp) + s |= m & 1; + return half(detail::binary, detail::fixed2half(m, exp, sign&0x8000, s)); + #endif + } + + /// Natural logarithm plus one. + /// This function may be 1 ULP off the correctly rounded exact result in <0.05% of inputs for `std::round_to_nearest` + /// and in ~1% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::log1p](https://en.cppreference.com/w/cpp/numeric/math/log1p). + /// \param arg function argument + /// \return logarithm of \a arg plus 1 to base e + /// \exception FE_INVALID for signaling NaN or argument <-1 + /// \exception FE_DIVBYZERO for -1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half log1p(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::log1p(detail::half2float(arg.data_)))); + #else + if(arg.data_ >= 0xBC00) + return half(detail::binary, (arg.data_==0xBC00) ? detail::pole(0x8000) : (arg.data_<=0xFC00) ? detail::invalid() : detail::signal(arg.data_)); + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + detail::uint32 m = static_cast((abs&0x3FF)|0x400) << 20; + if(arg.data_ & 0x8000) + { + m = 0x40000000 - (m>>-exp); + for(exp=0; m<0x40000000; m<<=1,--exp) ; + } + else + { + if(exp < 0) + { + m = 0x40000000 + (m>>-exp); + exp = 0; + } + else + { + m += 0x40000000 >> exp; + int i = m >> 31; + m >>= i; + exp += i; + } + } + return half(detail::binary, detail::log2_post(detail::log2(m), exp, 17)); + #endif + } + + /// \} + /// \anchor power + /// \name Power functions + /// \{ + + /// Square root. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sqrt](https://en.cppreference.com/w/cpp/numeric/math/sqrt). + /// \param arg function argument + /// \return square root of \a arg + /// \exception FE_INVALID for signaling NaN and negative arguments + /// \exception FE_INEXACT according to rounding + inline half sqrt(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sqrt(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 15; + if(!abs || arg.data_ >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (arg.data_>0x8000) ? detail::invalid() : arg.data_); + for(; abs<0x400; abs<<=1,--exp) ; + detail::uint32 r = static_cast((abs&0x3FF)|0x400) << 10, m = detail::sqrt<20>(r, exp+=abs>>10); + return half(detail::binary, detail::rounded((exp<<10)+(m&0x3FF), r>m, r!=0)); + #endif + } + + /// Inverse square root. + /// This function is exact to rounding for all rounding modes and thus generally more accurate than directly computing + /// 1 / sqrt(\a arg) in half-precision, in addition to also being faster. + /// \param arg function argument + /// \return reciprocal of square root of \a arg + /// \exception FE_INVALID for signaling NaN and negative arguments + /// \exception FE_INEXACT according to rounding + inline half rsqrt(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(detail::internal_t(1)/std::sqrt(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, bias = 0x4000; + if(!abs || arg.data_ >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (arg.data_>0x8000) ? + detail::invalid() : !abs ? detail::pole(arg.data_&0x8000) : 0); + for(; abs<0x400; abs<<=1,bias-=0x400) ; + unsigned int frac = (abs+=bias) & 0x7FF; + if(frac == 0x400) + return half(detail::binary, 0x7A00-(abs>>1)); + if((half::round_style == std::round_to_nearest && (frac == 0x3FE || frac == 0x76C)) || + (half::round_style != std::round_to_nearest && (frac == 0x15A || frac == 0x3FC || frac == 0x401 || frac == 0x402 || frac == 0x67B))) + return pow(arg, half(detail::binary, 0xB800)); + detail::uint32 f = 0x17376 - abs, mx = (abs&0x3FF) | 0x400, my = ((f>>1)&0x3FF) | 0x400, mz = my * my; + int expy = (f>>11) - 31, expx = 32 - (abs>>10), i = mz >> 21; + for(mz=0x60000000-(((mz>>i)*mx)>>(expx-2*expy-i)); mz<0x40000000; mz<<=1,--expy) ; + i = (my*=mz>>10) >> 31; + expy += i; + my = (my>>(20+i)) + 1; + i = (mz=my*my) >> 21; + for(mz=0x60000000-(((mz>>i)*mx)>>(expx-2*expy-i)); mz<0x40000000; mz<<=1,--expy) ; + i = (my*=(mz>>10)+1) >> 31; + return half(detail::binary, detail::fixed2half(my>>i, expy+i+14)); + #endif + } + + /// Cubic root. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cbrt](https://en.cppreference.com/w/cpp/numeric/math/cbrt). + /// \param arg function argument + /// \return cubic root of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT according to rounding + inline half cbrt(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::cbrt(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = -15; + if(!abs || abs == 0x3C00 || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1, --exp); + detail::uint32 ilog = exp + (abs>>10), sign = detail::sign_mask(ilog), f, m = + (((ilog<<27)+(detail::log2(static_cast((abs&0x3FF)|0x400)<<20, 24)>>4))^sign) - sign; + for(exp=2; m<0x80000000; m<<=1,--exp) ; + m = detail::multiply64(m, 0xAAAAAAAB); + int i = m >> 31, s; + exp += i; + m <<= 1 - i; + if(exp < 0) + { + f = m >> -exp; + exp = 0; + } + else + { + f = (m<> (31-exp); + } + m = detail::exp2(f, (half::round_style==std::round_to_nearest) ? 29 : 26); + if(sign) + { + if(m > 0x80000000) + { + m = detail::divide64(0x80000000, m, s); + ++exp; + } + exp = -exp; + } + return half(detail::binary, (half::round_style==std::round_to_nearest) ? + detail::fixed2half(m, exp+14, arg.data_&0x8000) : + detail::fixed2half((m+0x80)>>8, exp+14, arg.data_&0x8000)); + #endif + } + + /// Hypotenuse function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::hypot](https://en.cppreference.com/w/cpp/numeric/math/hypot). + /// \param x first argument + /// \param y second argument + /// \return square root of sum of squares without internal over- or underflows + /// \exception FE_INVALID if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding of the final square root + inline half hypot(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_); + #if HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::hypot(fx, fy))); + #else + return half(detail::binary, detail::float2half(std::sqrt(fx*fx+fy*fy))); + #endif + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, expx = 0, expy = 0; + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx==0x7C00) ? detail::select(0x7C00, y.data_) : + (absy==0x7C00) ? detail::select(0x7C00, x.data_) : detail::signal(x.data_, y.data_)); + if(!absx) + return half(detail::binary, absy ? detail::check_underflow(absy) : 0); + if(!absy) + return half(detail::binary, detail::check_underflow(absx)); + if(absy > absx) + std::swap(absx, absy); + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400; + mx *= mx; + my *= my; + int ix = mx >> 21, iy = my >> 21; + expx = 2*(expx+(absx>>10)) - 15 + ix; + expy = 2*(expy+(absy>>10)) - 15 + iy; + mx <<= 10 - ix; + my <<= 10 - iy; + int d = expx - expy; + my = (d<30) ? ((my>>d)|((my&((static_cast(1)<(mx+my, expx)); + #endif + } + + /// Hypotenuse function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::hypot](https://en.cppreference.com/w/cpp/numeric/math/hypot). + /// \param x first argument + /// \param y second argument + /// \param z third argument + /// \return square root of sum of squares without internal over- or underflows + /// \exception FE_INVALID if \a x, \a y or \a z is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding of the final square root + inline half hypot(half x, half y, half z) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t fx = detail::half2float(x.data_), fy = detail::half2float(y.data_), fz = detail::half2float(z.data_); + return half(detail::binary, detail::float2half(std::sqrt(fx*fx+fy*fy+fz*fz))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, absz = z.data_ & 0x7FFF, expx = 0, expy = 0, expz = 0; + if(!absx) + return hypot(y, z); + if(!absy) + return hypot(x, z); + if(!absz) + return hypot(x, y); + if(absx >= 0x7C00 || absy >= 0x7C00 || absz >= 0x7C00) + return half(detail::binary, (absx==0x7C00) ? detail::select(0x7C00, detail::select(y.data_, z.data_)) : + (absy==0x7C00) ? detail::select(0x7C00, detail::select(x.data_, z.data_)) : + (absz==0x7C00) ? detail::select(0x7C00, detail::select(x.data_, y.data_)) : + detail::signal(x.data_, y.data_, z.data_)); + if(absz > absy) + std::swap(absy, absz); + if(absy > absx) + std::swap(absx, absy); + if(absz > absy) + std::swap(absy, absz); + for(; absx<0x400; absx<<=1,--expx) ; + for(; absy<0x400; absy<<=1,--expy) ; + for(; absz<0x400; absz<<=1,--expz) ; + detail::uint32 mx = (absx&0x3FF) | 0x400, my = (absy&0x3FF) | 0x400, mz = (absz&0x3FF) | 0x400; + mx *= mx; + my *= my; + mz *= mz; + int ix = mx >> 21, iy = my >> 21, iz = mz >> 21; + expx = 2*(expx+(absx>>10)) - 15 + ix; + expy = 2*(expy+(absy>>10)) - 15 + iy; + expz = 2*(expz+(absz>>10)) - 15 + iz; + mx <<= 10 - ix; + my <<= 10 - iy; + mz <<= 10 - iz; + int d = expy - expz; + mz = (d<30) ? ((mz>>d)|((mz&((static_cast(1)<>1) | (my&1); + if(++expy > expx) + { + std::swap(mx, my); + std::swap(expx, expy); + } + } + d = expx - expy; + my = (d<30) ? ((my>>d)|((my&((static_cast(1)<(mx+my, expx)); + #endif + } + + /// Power function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in ~0.00025% of inputs. + /// + /// **See also:** Documentation for [std::pow](https://en.cppreference.com/w/cpp/numeric/math/pow). + /// \param x base + /// \param y exponent + /// \return \a x raised to \a y + /// \exception FE_INVALID if \a x or \a y is signaling NaN or if \a x is finite an negative and \a y is finite and not integral + /// \exception FE_DIVBYZERO if \a x is 0 and \a y is negative + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half pow(half x, half y) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::pow(detail::half2float(x.data_), detail::half2float(y.data_)))); + #else + int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, exp = -15; + if(!absy || x.data_ == 0x3C00) + return half(detail::binary, detail::select(0x3C00, (x.data_==0x3C00) ? y.data_ : x.data_)); + bool is_int = absy >= 0x6400 || (absy>=0x3C00 && !(absy&((1<<(25-(absy>>10)))-1))); + unsigned int sign = x.data_ & (static_cast((absy<0x6800)&&is_int&&((absy>>(25-(absy>>10)))&1))<<15); + if(absx >= 0x7C00 || absy >= 0x7C00) + return half(detail::binary, (absx>0x7C00 || absy>0x7C00) ? detail::signal(x.data_, y.data_) : + (absy==0x7C00) ? ((absx==0x3C00) ? 0x3C00 : (!absx && y.data_==0xFC00) ? detail::pole() : + (0x7C00&-((y.data_>>15)^(absx>0x3C00)))) : (sign|(0x7C00&((y.data_>>15)-1U)))); + if(!absx) + return half(detail::binary, (y.data_&0x8000) ? detail::pole(sign) : sign); + if((x.data_&0x8000) && !is_int) + return half(detail::binary, detail::invalid()); + if(x.data_ == 0xBC00) + return half(detail::binary, sign|0x3C00); + switch(y.data_) + { + case 0x3800: return sqrt(x); + case 0x3C00: return half(detail::binary, detail::check_underflow(x.data_)); + case 0x4000: return x * x; + case 0xBC00: return half(detail::binary, 0x3C00) / x; + } + for(; absx<0x400; absx<<=1,--exp) ; + detail::uint32 ilog = exp + (absx>>10), msign = detail::sign_mask(ilog), f, m = + (((ilog<<27)+((detail::log2(static_cast((absx&0x3FF)|0x400)<<20)+8)>>4))^msign) - msign; + for(exp=-11; m<0x80000000; m<<=1,--exp) ; + for(; absy<0x400; absy<<=1,--exp) ; + m = detail::multiply64(m, static_cast((absy&0x3FF)|0x400)<<21); + int i = m >> 31; + exp += (absy>>10) + i; + m <<= 1 - i; + if(exp < 0) + { + f = m >> -exp; + exp = 0; + } + else + { + f = (m<> (31-exp); + } + return half(detail::binary, detail::exp2_post(f, exp, ((msign&1)^(y.data_>>15))!=0, sign)); + #endif + } + + /// \} + /// \anchor trigonometric + /// \name Trigonometric functions + /// \{ + + /// Compute sine and cosine simultaneously. + /// This returns the same results as sin() and cos() but is faster than calling each function individually. + /// + /// This function is exact to rounding for all rounding modes. + /// \param arg function argument + /// \param sin variable to take sine of \a arg + /// \param cos variable to take cosine of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline void sincos(half arg, half *sin, half *cos) + { + #ifdef HALF_ARITHMETIC_TYPE + detail::internal_t f = detail::half2float(arg.data_); + *sin = half(detail::binary, detail::float2half(std::sin(f))); + *cos = half(detail::binary, detail::float2half(std::cos(f))); + #else + int abs = arg.data_ & 0x7FFF, sign = arg.data_ >> 15, k; + if(abs >= 0x7C00) + *sin = *cos = half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + else if(!abs) + { + *sin = arg; + *cos = half(detail::binary, 0x3C00); + } + else if(abs < 0x2500) + { + *sin = half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + } + else + { + if(half::round_style != std::round_to_nearest) + { + switch(abs) + { + case 0x48B7: + *sin = half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x1D07, 1, 1)); + *cos = half(detail::binary, detail::rounded(0xBBFF, 1, 1)); + return; + case 0x598C: + *sin = half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x80FC, 1, 1)); + return; + case 0x6A64: + *sin = half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x3BFE, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x27FF, 1, 1)); + return; + case 0x6D8C: + *sin = half(detail::binary, detail::rounded((arg.data_&0x8000)|0x0FE6, 1, 1)); + *cos = half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + return; + } + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + switch(k & 3) + { + case 1: sc = std::make_pair(sc.second, -sc.first); break; + case 2: sc = std::make_pair(-sc.first, -sc.second); break; + case 3: sc = std::make_pair(-sc.second, sc.first); break; + } + *sin = half(detail::binary, detail::fixed2half((sc.first^-static_cast(sign))+sign)); + *cos = half(detail::binary, detail::fixed2half(sc.second)); + } + #endif + } + + /// Sine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sin](https://en.cppreference.com/w/cpp/numeric/math/sin). + /// \param arg function argument + /// \return sine value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half sin(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sin(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, k; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2900) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) + { + case 0x48B7: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x1D07, 1, 1)); + case 0x6A64: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x3BFE, 1, 1)); + case 0x6D8C: return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x0FE6, 1, 1)); + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + detail::uint32 sign = -static_cast(((k>>1)&1)^(arg.data_>>15)); + return half(detail::binary, detail::fixed2half((((k&1) ? sc.second : sc.first)^sign) - sign)); + #endif + } + + /// Cosine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cos](https://en.cppreference.com/w/cpp/numeric/math/cos). + /// \param arg function argument + /// \return cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half cos(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::cos(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, k; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2500) + return half(detail::binary, detail::rounded(0x3BFF, 1, 1)); + if(half::round_style != std::round_to_nearest && abs == 0x598C) + return half(detail::binary, detail::rounded(0x80FC, 1, 1)); + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 28); + detail::uint32 sign = -static_cast(((k>>1)^k)&1); + return half(detail::binary, detail::fixed2half((((k&1) ? sc.first : sc.second)^sign) - sign)); + #endif + } + + /// Tangent function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::tan](https://en.cppreference.com/w/cpp/numeric/math/tan). + /// \param arg function argument + /// \return tangent value of \a arg + /// \exception FE_INVALID for signaling NaN or infinity + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tan(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::tan(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 13, k; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) + { + case 0x658C: return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x07E6, 1, 1)); + case 0x7330: return half(detail::binary, detail::rounded((~arg.data_&0x8000)|0x4B62, 1, 1)); + } + std::pair sc = detail::sincos(detail::angle_arg(abs, k), 30); + if(k & 1) + sc = std::make_pair(-sc.second, sc.first); + detail::uint32 signy = detail::sign_mask(sc.first), signx = detail::sign_mask(sc.second); + detail::uint32 my = (sc.first^signy) - signy, mx = (sc.second^signx) - signx; + for(; my<0x80000000; my<<=1,--exp) ; + for(; mx<0x80000000; mx<<=1,++exp) ; + return half(detail::binary, detail::tangent_post(my, mx, exp, (signy^signx^arg.data_)&0x8000)); + #endif + } + + /// Arc sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::asin](https://en.cppreference.com/w/cpp/numeric/math/asin). + /// \param arg function argument + /// \return arc sine value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half asin(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::asin(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(!abs) + return arg; + if(abs >= 0x3C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (abs>0x3C00) ? detail::invalid() : + detail::rounded(sign|0x3E48, 0, 1)); + if(abs < 0x2900) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + if(half::round_style != std::round_to_nearest && (abs == 0x2B44 || abs == 0x2DC3)) + return half(detail::binary, detail::rounded(arg.data_+1, 1, 1)); + std::pair sc = detail::atan2_args(abs); + detail::uint32 m = detail::atan2(sc.first, sc.second, (half::round_style==std::round_to_nearest) ? 27 : 26); + return half(detail::binary, detail::fixed2half(m, 14, sign)); + #endif + } + + /// Arc cosine function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::acos](https://en.cppreference.com/w/cpp/numeric/math/acos). + /// \param arg function argument + /// \return arc cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half acos(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::acos(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ >> 15; + if(!abs) + return half(detail::binary, detail::rounded(0x3E48, 0, 1)); + if(abs >= 0x3C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (abs>0x3C00) ? detail::invalid() : + sign ? detail::rounded(0x4248, 0, 1) : 0); + std::pair cs = detail::atan2_args(abs); + detail::uint32 m = detail::atan2(cs.second, cs.first, 28); + return half(detail::binary, detail::fixed2half(sign ? (0xC90FDAA2-m) : m, 15, 0, sign)); + #endif + } + + /// Arc tangent function. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::atan](https://en.cppreference.com/w/cpp/numeric/math/atan). + /// \param arg function argument + /// \return arc tangent value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atan(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::atan(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? detail::rounded(sign|0x3E48, 0, 1) : detail::signal(arg.data_)); + if(abs <= 0x2700) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + int exp = (abs>>10) + (abs<=0x3FF); + detail::uint32 my = (abs&0x3FF) | ((abs>0x3FF)<<10); + detail::uint32 m = (exp>15) ? detail::atan2(my<<19, 0x20000000>>(exp-15), (half::round_style==std::round_to_nearest) ? 26 : 24) : + detail::atan2(my<<(exp+4), 0x20000000, (half::round_style==std::round_to_nearest) ? 30 : 28); + return half(detail::binary, detail::fixed2half(m, 14, sign)); + #endif + } + + /// Arc tangent function. + /// This function may be 1 ULP off the correctly rounded exact result in ~0.005% of inputs for `std::round_to_nearest`, + /// in ~0.1% of inputs for `std::round_toward_zero` and in ~0.02% of inputs for any other rounding mode. + /// + /// **See also:** Documentation for [std::atan2](https://en.cppreference.com/w/cpp/numeric/math/atan2). + /// \param y numerator + /// \param x denominator + /// \return arc tangent value + /// \exception FE_INVALID if \a x or \a y is signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atan2(half y, half x) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::atan2(detail::half2float(y.data_), detail::half2float(x.data_)))); + #else + unsigned int absx = x.data_ & 0x7FFF, absy = y.data_ & 0x7FFF, signx = x.data_ >> 15, signy = y.data_ & 0x8000; + if(absx >= 0x7C00 || absy >= 0x7C00) + { + if(absx > 0x7C00 || absy > 0x7C00) + return half(detail::binary, detail::signal(x.data_, y.data_)); + if(absy == 0x7C00) + return half(detail::binary, (absx<0x7C00) ? detail::rounded(signy|0x3E48, 0, 1) : + signx ? detail::rounded(signy|0x40B6, 0, 1) : + detail::rounded(signy|0x3A48, 0, 1)); + return (x.data_==0x7C00) ? half(detail::binary, signy) : half(detail::binary, detail::rounded(signy|0x4248, 0, 1)); + } + if(!absy) + return signx ? half(detail::binary, detail::rounded(signy|0x4248, 0, 1)) : y; + if(!absx) + return half(detail::binary, detail::rounded(signy|0x3E48, 0, 1)); + int d = (absy>>10) + (absy<=0x3FF) - (absx>>10) - (absx<=0x3FF); + if(d > (signx ? 18 : 12)) + return half(detail::binary, detail::rounded(signy|0x3E48, 0, 1)); + if(signx && d < -11) + return half(detail::binary, detail::rounded(signy|0x4248, 0, 1)); + if(!signx && d < ((half::round_style==std::round_toward_zero) ? -15 : -9)) + { + for(; absy<0x400; absy<<=1,--d) ; + detail::uint32 mx = ((absx<<1)&0x7FF) | 0x800, my = ((absy<<1)&0x7FF) | 0x800; + int i = my < mx; + d -= i; + if(d < -25) + return half(detail::binary, detail::underflow(signy)); + my <<= 11 + i; + return half(detail::binary, detail::fixed2half(my/mx, d+14, signy, my%mx!=0)); + } + detail::uint32 m = detail::atan2( ((absy&0x3FF)|((absy>0x3FF)<<10))<<(19+((d<0) ? d : (d>0) ? 0 : -1)), + ((absx&0x3FF)|((absx>0x3FF)<<10))<<(19-((d>0) ? d : (d<0) ? 0 : 1))); + return half(detail::binary, detail::fixed2half(signx ? (0xC90FDAA2-m) : m, 15, signy, signx)); + #endif + } + + /// \} + /// \anchor hyperbolic + /// \name Hyperbolic functions + /// \{ + + /// Hyperbolic sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::sinh](https://en.cppreference.com/w/cpp/numeric/math/sinh). + /// \param arg function argument + /// \return hyperbolic sine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half sinh(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::sinh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + if(abs <= 0x2900) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + std::pair mm = detail::hyperbolic_args(abs, exp, (half::round_style==std::round_to_nearest) ? 29 : 27); + detail::uint32 m = mm.first - mm.second; + for(exp+=13; m<0x80000000 && exp; m<<=1,--exp) ; + unsigned int sign = arg.data_ & 0x8000; + if(exp > 29) + return half(detail::binary, detail::overflow(sign)); + return half(detail::binary, detail::fixed2half(m, exp, sign)); + #endif + } + + /// Hyperbolic cosine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::cosh](https://en.cppreference.com/w/cpp/numeric/math/cosh). + /// \param arg function argument + /// \return hyperbolic cosine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half cosh(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::cosh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : 0x7C00); + std::pair mm = detail::hyperbolic_args(abs, exp, (half::round_style==std::round_to_nearest) ? 23 : 26); + detail::uint32 m = mm.first + mm.second, i = (~m&0xFFFFFFFF) >> 31; + m = (m>>i) | (m&i) | 0x80000000; + if((exp+=13+i) > 29) + return half(detail::binary, detail::overflow()); + return half(detail::binary, detail::fixed2half(m, exp)); + #endif + } + + /// Hyperbolic tangent. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::tanh](https://en.cppreference.com/w/cpp/numeric/math/tanh). + /// \param arg function argument + /// \return hyperbolic tangent value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tanh(half arg) + { + #ifdef HALF_ARITHMETIC_TYPE + return half(detail::binary, detail::float2half(std::tanh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return arg; + if(abs >= 0x7C00) + return half(detail::binary, (abs>0x7C00) ? detail::signal(arg.data_) : (arg.data_-0x4000)); + if(abs >= 0x4500) + return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest && abs == 0x2D3F) + return half(detail::binary, detail::rounded(arg.data_-3, 0, 1)); + std::pair mm = detail::hyperbolic_args(abs, exp, 27); + detail::uint32 my = mm.first - mm.second - (half::round_style!=std::round_to_nearest), mx = mm.first + mm.second, i = (~mx&0xFFFFFFFF) >> 31; + for(exp=13; my<0x80000000; my<<=1,--exp) ; + mx = (mx>>i) | 0x80000000; + return half(detail::binary, detail::tangent_post(my, mx, exp-i, arg.data_&0x8000)); + #endif + } + + /// Hyperbolic area sine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::asinh](https://en.cppreference.com/w/cpp/numeric/math/asinh). + /// \param arg function argument + /// \return area sine value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half asinh(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::asinh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if(!abs || abs >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + if(abs <= 0x2900) + return half(detail::binary, detail::rounded(arg.data_-1, 1, 1)); + if(half::round_style != std::round_to_nearest) + switch(abs) + { + case 0x32D4: return half(detail::binary, detail::rounded(arg.data_-13, 1, 1)); + case 0x3B5B: return half(detail::binary, detail::rounded(arg.data_-197, 1, 1)); + } + return half(detail::binary, detail::area(arg.data_)); + #endif + } + + /// Hyperbolic area cosine. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::acosh](https://en.cppreference.com/w/cpp/numeric/math/acosh). + /// \param arg function argument + /// \return area cosine value of \a arg + /// \exception FE_INVALID for signaling NaN or arguments <1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half acosh(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::acosh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if((arg.data_&0x8000) || abs < 0x3C00) + return half(detail::binary, (abs<=0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs == 0x3C00) + return half(detail::binary, 0); + if(arg.data_ >= 0x7C00) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + return half(detail::binary, detail::area(arg.data_)); + #endif + } + + /// Hyperbolic area tangent. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::atanh](https://en.cppreference.com/w/cpp/numeric/math/atanh). + /// \param arg function argument + /// \return area tangent value of \a arg + /// \exception FE_INVALID for signaling NaN or if abs(\a arg) > 1 + /// \exception FE_DIVBYZERO for +/-1 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half atanh(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::atanh(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF, exp = 0; + if(!abs) + return arg; + if(abs >= 0x3C00) + return half(detail::binary, (abs==0x3C00) ? detail::pole(arg.data_&0x8000) : (abs<=0x7C00) ? detail::invalid() : detail::signal(arg.data_)); + if(abs < 0x2700) + return half(detail::binary, detail::rounded(arg.data_, 0, 1)); + detail::uint32 m = static_cast((abs&0x3FF)|((abs>0x3FF)<<10)) << ((abs>>10)+(abs<=0x3FF)+6), my = 0x80000000 + m, mx = 0x80000000 - m; + for(; mx<0x80000000; mx<<=1,++exp) ; + int i = my >= mx, s; + return half(detail::binary, detail::log2_post(detail::log2( + (detail::divide64(my>>i, mx, s)+1)>>1, 27)+0x10, exp+i-1, 16, arg.data_&0x8000)); + #endif + } + + /// \} + /// \anchor special + /// \name Error and gamma functions + /// \{ + + /// Error function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.5% of inputs. + /// + /// **See also:** Documentation for [std::erf](https://en.cppreference.com/w/cpp/numeric/math/erf). + /// \param arg function argument + /// \return error function value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half erf(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::erf(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF; + if(!abs || abs >= 0x7C00) + return (abs>=0x7C00) ? half(detail::binary, (abs==0x7C00) ? (arg.data_-0x4000) : detail::signal(arg.data_)) : arg; + if(abs >= 0x4200) + return half(detail::binary, detail::rounded((arg.data_&0x8000)|0x3BFF, 1, 1)); + return half(detail::binary, detail::erf(arg.data_)); + #endif + } + + /// Complementary error function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.5% of inputs. + /// + /// **See also:** Documentation for [std::erfc](https://en.cppreference.com/w/cpp/numeric/math/erfc). + /// \param arg function argument + /// \return 1 minus error function value of \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half erfc(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::erfc(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(abs >= 0x7C00) + return (abs>=0x7C00) ? half(detail::binary, (abs==0x7C00) ? (sign>>1) : detail::signal(arg.data_)) : arg; + if(!abs) + return half(detail::binary, 0x3C00); + if(abs >= 0x4400) + return half(detail::binary, detail::rounded((sign>>1)-(sign>>15), sign>>15, 1)); + return half(detail::binary, detail::erf(arg.data_)); + #endif + } + + /// Natural logarithm of gamma function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in ~0.025% of inputs. + /// + /// **See also:** Documentation for [std::lgamma](https://en.cppreference.com/w/cpp/numeric/math/lgamma). + /// \param arg function argument + /// \return natural logarith of gamma function for \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_DIVBYZERO for 0 or negative integer arguments + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half lgamma(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::lgamma(detail::half2float(arg.data_)))); + #else + int abs = arg.data_ & 0x7FFF; + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? 0x7C00 : detail::signal(arg.data_)); + if(!abs || arg.data_ >= 0xE400 || (arg.data_ >= 0xBC00 && !(abs&((1<<(25-(abs>>10)))-1)))) + return half(detail::binary, detail::pole()); + if(arg.data_ == 0x3C00 || arg.data_ == 0x4000) + return half(detail::binary, 0); + return half(detail::binary, detail::gamma(arg.data_)); + #endif + } + + /// Gamma function. + /// This function may be 1 ULP off the correctly rounded exact result for any rounding mode in <0.25% of inputs. + /// + /// **See also:** Documentation for [std::tgamma](https://en.cppreference.com/w/cpp/numeric/math/tgamma). + /// \param arg function argument + /// \return gamma function value of \a arg + /// \exception FE_INVALID for signaling NaN, negative infinity or negative integer arguments + /// \exception FE_DIVBYZERO for 0 + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half tgamma(half arg) + { + #if defined(HALF_ARITHMETIC_TYPE) && HALF_ENABLE_CPP11_CMATH + return half(detail::binary, detail::float2half(std::tgamma(detail::half2float(arg.data_)))); + #else + unsigned int abs = arg.data_ & 0x7FFF; + if(!abs) + return half(detail::binary, detail::pole(arg.data_)); + if(abs >= 0x7C00) + return (arg.data_==0x7C00) ? arg : half(detail::binary, detail::signal(arg.data_)); + if(arg.data_ >= 0xE400 || (arg.data_ >= 0xBC00 && !(abs&((1<<(25-(abs>>10)))-1)))) + return half(detail::binary, detail::invalid()); + if(arg.data_ >= 0xCA80) + return half(detail::binary, detail::underflow((1-((abs>>(25-(abs>>10)))&1))<<15)); + if(arg.data_ <= 0x100 || (arg.data_ >= 0x4900 && arg.data_ < 0x8000)) + return half(detail::binary, detail::overflow()); + if(arg.data_ == 0x3C00) + return arg; + return half(detail::binary, detail::gamma(arg.data_)); + #endif + } + + /// \} + /// \anchor rounding + /// \name Rounding + /// \{ + + /// Nearest integer not less than half value. + /// **See also:** Documentation for [std::ceil](https://en.cppreference.com/w/cpp/numeric/math/ceil). + /// \param arg half to round + /// \return nearest integer not less than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half ceil(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer not greater than half value. + /// **See also:** Documentation for [std::floor](https://en.cppreference.com/w/cpp/numeric/math/floor). + /// \param arg half to round + /// \return nearest integer not greater than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half floor(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer not greater in magnitude than half value. + /// **See also:** Documentation for [std::trunc](https://en.cppreference.com/w/cpp/numeric/math/trunc). + /// \param arg half to round + /// \return nearest integer not greater in magnitude than \a arg + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half trunc(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer. + /// **See also:** Documentation for [std::round](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half round(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer. + /// **See also:** Documentation for [std::lround](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID if value is not representable as `long` + inline long lround(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::rint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID for signaling NaN + /// \exception FE_INEXACT if value had to be rounded + inline half rint(half arg) { return half(detail::binary, detail::integral(arg.data_)); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::lrint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID if value is not representable as `long` + /// \exception FE_INEXACT if value had to be rounded + inline long lrint(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::nearbyint](https://en.cppreference.com/w/cpp/numeric/math/nearbyint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID for signaling NaN + inline half nearbyint(half arg) { return half(detail::binary, detail::integral(arg.data_)); } +#if HALF_ENABLE_CPP11_LONG_LONG + /// Nearest integer. + /// **See also:** Documentation for [std::llround](https://en.cppreference.com/w/cpp/numeric/math/round). + /// \param arg half to round + /// \return nearest integer, rounded away from zero in half-way cases + /// \exception FE_INVALID if value is not representable as `long long` + inline long long llround(half arg) { return detail::half2int(arg.data_); } + + /// Nearest integer using half's internal rounding mode. + /// **See also:** Documentation for [std::llrint](https://en.cppreference.com/w/cpp/numeric/math/rint). + /// \param arg half expression to round + /// \return nearest integer using default rounding mode + /// \exception FE_INVALID if value is not representable as `long long` + /// \exception FE_INEXACT if value had to be rounded + inline long long llrint(half arg) { return detail::half2int(arg.data_); } +#endif + + /// \} + /// \anchor float + /// \name Floating point manipulation + /// \{ + + /// Decompress floating-point number. + /// **See also:** Documentation for [std::frexp](https://en.cppreference.com/w/cpp/numeric/math/frexp). + /// \param arg number to decompress + /// \param exp address to store exponent at + /// \return significant in range [0.5, 1) + /// \exception FE_INVALID for signaling NaN + inline half frexp(half arg, int *exp) + { + *exp = 0; + unsigned int abs = arg.data_ & 0x7FFF; + if(abs >= 0x7C00 || !abs) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--*exp) ; + *exp += (abs>>10) - 14; + return half(detail::binary, (arg.data_&0x8000)|0x3800|(abs&0x3FF)); + } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::scalbln](https://en.cppreference.com/w/cpp/numeric/math/scalbn). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half scalbln(half arg, long exp) + { + unsigned int abs = arg.data_ & 0x7FFF, sign = arg.data_ & 0x8000; + if(abs >= 0x7C00 || !abs) + return (abs>0x7C00) ? half(detail::binary, detail::signal(arg.data_)) : arg; + for(; abs<0x400; abs<<=1,--exp) ; + exp += abs >> 10; + if(exp > 30) + return half(detail::binary, detail::overflow(sign)); + else if(exp < -10) + return half(detail::binary, detail::underflow(sign)); + else if(exp > 0) + return half(detail::binary, sign|(exp<<10)|(abs&0x3FF)); + unsigned int m = (abs&0x3FF) | 0x400; + return half(detail::binary, detail::rounded(sign|(m>>(1-exp)), (m>>-exp)&1, (m&((1<<-exp)-1))!=0)); + } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::scalbn](https://en.cppreference.com/w/cpp/numeric/math/scalbn). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half scalbn(half arg, int exp) { return scalbln(arg, exp); } + + /// Multiply by power of two. + /// This function is exact to rounding for all rounding modes. + /// + /// **See also:** Documentation for [std::ldexp](https://en.cppreference.com/w/cpp/numeric/math/ldexp). + /// \param arg number to modify + /// \param exp power of two to multiply with + /// \return \a arg multplied by 2 raised to \a exp + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + inline half ldexp(half arg, int exp) { return scalbln(arg, exp); } + + /// Extract integer and fractional parts. + /// **See also:** Documentation for [std::modf](https://en.cppreference.com/w/cpp/numeric/math/modf). + /// \param arg number to decompress + /// \param iptr address to store integer part at + /// \return fractional part + /// \exception FE_INVALID for signaling NaN + inline half modf(half arg, half *iptr) + { + unsigned int abs = arg.data_ & 0x7FFF; + if(abs > 0x7C00) + { + arg = half(detail::binary, detail::signal(arg.data_)); + return *iptr = arg, arg; + } + if(abs >= 0x6400) + return *iptr = arg, half(detail::binary, arg.data_&0x8000); + if(abs < 0x3C00) + return iptr->data_ = arg.data_ & 0x8000, arg; + unsigned int exp = abs >> 10, mask = (1<<(25-exp)) - 1, m = arg.data_ & mask; + iptr->data_ = arg.data_ & ~mask; + if(!m) + return half(detail::binary, arg.data_&0x8000); + for(; m<0x400; m<<=1,--exp) ; + return half(detail::binary, (arg.data_&0x8000)|(exp<<10)|(m&0x3FF)); + } + + /// Extract exponent. + /// **See also:** Documentation for [std::ilogb](https://en.cppreference.com/w/cpp/numeric/math/ilogb). + /// \param arg number to query + /// \return floating-point exponent + /// \retval FP_ILOGB0 for zero + /// \retval FP_ILOGBNAN for NaN + /// \retval INT_MAX for infinity + /// \exception FE_INVALID for 0 or infinite values + inline int ilogb(half arg) + { + int abs = arg.data_ & 0x7FFF, exp; + if(!abs || abs >= 0x7C00) + { + detail::raise(FE_INVALID); + return !abs ? FP_ILOGB0 : (abs==0x7C00) ? INT_MAX : FP_ILOGBNAN; + } + for(exp=(abs>>10)-15; abs<0x200; abs<<=1,--exp) ; + return exp; + } + + /// Extract exponent. + /// **See also:** Documentation for [std::logb](https://en.cppreference.com/w/cpp/numeric/math/logb). + /// \param arg number to query + /// \return floating-point exponent + /// \exception FE_INVALID for signaling NaN + /// \exception FE_DIVBYZERO for 0 + inline half logb(half arg) + { + int abs = arg.data_ & 0x7FFF, exp; + if(!abs) + return half(detail::binary, detail::pole(0x8000)); + if(abs >= 0x7C00) + return half(detail::binary, (abs==0x7C00) ? 0x7C00 : detail::signal(arg.data_)); + for(exp=(abs>>10)-15; abs<0x200; abs<<=1,--exp) ; + unsigned int value = static_cast(exp<0) << 15; + if(exp) + { + unsigned int m = std::abs(exp) << 6; + for(exp=18; m<0x400; m<<=1,--exp) ; + value |= (exp<<10) + m; + } + return half(detail::binary, value); + } + + /// Next representable value. + /// **See also:** Documentation for [std::nextafter](https://en.cppreference.com/w/cpp/numeric/math/nextafter). + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW for infinite result from finite argument + /// \exception FE_UNDERFLOW for subnormal result + inline half nextafter(half from, half to) + { + int fabs = from.data_ & 0x7FFF, tabs = to.data_ & 0x7FFF; + if(fabs > 0x7C00 || tabs > 0x7C00) + return half(detail::binary, detail::signal(from.data_, to.data_)); + if(from.data_ == to.data_ || !(fabs|tabs)) + return to; + if(!fabs) + { + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT); + return half(detail::binary, (to.data_&0x8000)+1); + } + unsigned int out = from.data_ + (((from.data_>>15)^static_cast( + (from.data_^(0x8000|(0x8000-(from.data_>>15))))<(to.data_^(0x8000|(0x8000-(to.data_>>15))))))<<1) - 1; + detail::raise(FE_OVERFLOW, fabs<0x7C00 && (out&0x7C00)==0x7C00); + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT && (out&0x7C00)<0x400); + return half(detail::binary, out); + } + + /// Next representable value. + /// **See also:** Documentation for [std::nexttoward](https://en.cppreference.com/w/cpp/numeric/math/nexttoward). + /// \param from value to compute next representable value for + /// \param to direction towards which to compute next value + /// \return next representable value after \a from in direction towards \a to + /// \exception FE_INVALID for signaling NaN + /// \exception FE_OVERFLOW for infinite result from finite argument + /// \exception FE_UNDERFLOW for subnormal result + inline half nexttoward(half from, long double to) + { + int fabs = from.data_ & 0x7FFF; + if(fabs > 0x7C00) + return half(detail::binary, detail::signal(from.data_)); + long double lfrom = static_cast(from); + if(detail::builtin_isnan(to) || lfrom == to) + return half(static_cast(to)); + if(!fabs) + { + detail::raise(FE_UNDERFLOW, !HALF_ERRHANDLING_UNDERFLOW_TO_INEXACT); + return half(detail::binary, (static_cast(detail::builtin_signbit(to))<<15)+1); + } + unsigned int out = from.data_ + (((from.data_>>15)^static_cast(lfrom 0x7C00; } + + /// Check if normal number. + /// **See also:** Documentation for [std::isnormal](https://en.cppreference.com/w/cpp/numeric/math/isnormal). + /// \param arg number to check + /// \retval true if normal number + /// \retval false if either subnormal, zero, infinity or NaN + inline HALF_CONSTEXPR bool isnormal(half arg) { return ((arg.data_&0x7C00)!=0) & ((arg.data_&0x7C00)!=0x7C00); } + + /// Check sign. + /// **See also:** Documentation for [std::signbit](https://en.cppreference.com/w/cpp/numeric/math/signbit). + /// \param arg number to check + /// \retval true for negative number + /// \retval false for positive number + inline HALF_CONSTEXPR bool signbit(half arg) { return (arg.data_&0x8000) != 0; } + + /// \} + /// \anchor compfunc + /// \name Comparison + /// \{ + + /// Quiet comparison for greater than. + /// **See also:** Documentation for [std::isgreater](https://en.cppreference.com/w/cpp/numeric/math/isgreater). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater than \a y + /// \retval false else + inline HALF_CONSTEXPR bool isgreater(half x, half y) + { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) > ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for greater equal. + /// **See also:** Documentation for [std::isgreaterequal](https://en.cppreference.com/w/cpp/numeric/math/isgreaterequal). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x greater equal \a y + /// \retval false else + inline HALF_CONSTEXPR bool isgreaterequal(half x, half y) + { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) >= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for less than. + /// **See also:** Documentation for [std::isless](https://en.cppreference.com/w/cpp/numeric/math/isless). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less than \a y + /// \retval false else + inline HALF_CONSTEXPR bool isless(half x, half y) + { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) < ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comparison for less equal. + /// **See also:** Documentation for [std::islessequal](https://en.cppreference.com/w/cpp/numeric/math/islessequal). + /// \param x first operand + /// \param y second operand + /// \retval true if \a x less equal \a y + /// \retval false else + inline HALF_CONSTEXPR bool islessequal(half x, half y) + { + return ((x.data_^(0x8000|(0x8000-(x.data_>>15))))+(x.data_>>15)) <= ((y.data_^(0x8000|(0x8000-(y.data_>>15))))+(y.data_>>15)) && !isnan(x) && !isnan(y); + } + + /// Quiet comarison for less or greater. + /// **See also:** Documentation for [std::islessgreater](https://en.cppreference.com/w/cpp/numeric/math/islessgreater). + /// \param x first operand + /// \param y second operand + /// \retval true if either less or greater + /// \retval false else + inline HALF_CONSTEXPR bool islessgreater(half x, half y) + { + return x.data_!=y.data_ && ((x.data_|y.data_)&0x7FFF) && !isnan(x) && !isnan(y); + } + + /// Quiet check if unordered. + /// **See also:** Documentation for [std::isunordered](https://en.cppreference.com/w/cpp/numeric/math/isunordered). + /// \param x first operand + /// \param y second operand + /// \retval true if unordered (one or two NaN operands) + /// \retval false else + inline HALF_CONSTEXPR bool isunordered(half x, half y) { return isnan(x) || isnan(y); } + + /// \} + /// \anchor casting + /// \name Casting + /// \{ + + /// Cast to or from half-precision floating-point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the default rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s returns the argument unmodified. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + /// \exception FE_INVALID if \a T is integer type and result is not representable as \a T + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template T half_cast(U arg) { return detail::half_caster::cast(arg); } + + /// Cast to or from half-precision floating-point number. + /// This casts between [half](\ref half_float::half) and any built-in arithmetic type. The values are converted + /// directly using the specified rounding mode, without any roundtrip over `float` that a `static_cast` would otherwise do. + /// + /// Using this cast with neither of the two types being a [half](\ref half_float::half) or with any of the two types + /// not being a built-in arithmetic type (apart from [half](\ref half_float::half), of course) results in a compiler + /// error and casting between [half](\ref half_float::half)s returns the argument unmodified. + /// \tparam T destination type (half or built-in arithmetic type) + /// \tparam R rounding mode to use. + /// \tparam U source type (half or built-in arithmetic type) + /// \param arg value to cast + /// \return \a arg converted to destination type + /// \exception FE_INVALID if \a T is integer type and result is not representable as \a T + /// \exception FE_OVERFLOW, ...UNDERFLOW, ...INEXACT according to rounding + template T half_cast(U arg) { return detail::half_caster::cast(arg); } + /// \} + + /// \} + /// \anchor errors + /// \name Error handling + /// \{ + + /// Clear exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::feclearexcept](https://en.cppreference.com/w/cpp/numeric/fenv/feclearexcept). + /// \param excepts OR of exceptions to clear + /// \retval 0 all selected flags cleared successfully + inline int feclearexcept(int excepts) { detail::errflags() &= ~excepts; return 0; } + + /// Test exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fetestexcept](https://en.cppreference.com/w/cpp/numeric/fenv/fetestexcept). + /// \param excepts OR of exceptions to test + /// \return OR of selected exceptions if raised + inline int fetestexcept(int excepts) { return detail::errflags() & excepts; } + + /// Raise exception flags. + /// This raises the specified floating point exceptions and also invokes any additional automatic exception handling as + /// configured with the [HALF_ERRHANDLIG_...](\ref HALF_ERRHANDLING_ERRNO) preprocessor symbols. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::feraiseexcept](https://en.cppreference.com/w/cpp/numeric/fenv/feraiseexcept). + /// \param excepts OR of exceptions to raise + /// \retval 0 all selected exceptions raised successfully + inline int feraiseexcept(int excepts) { detail::errflags() |= excepts; detail::raise(excepts); return 0; } + + /// Save exception flags. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fegetexceptflag](https://en.cppreference.com/w/cpp/numeric/fenv/feexceptflag). + /// \param flagp adress to store flag state at + /// \param excepts OR of flags to save + /// \retval 0 for success + inline int fegetexceptflag(int *flagp, int excepts) { *flagp = detail::errflags() & excepts; return 0; } + + /// Restore exception flags. + /// This only copies the specified exception state (including unset flags) without incurring any additional exception handling. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// + /// **See also:** Documentation for [std::fesetexceptflag](https://en.cppreference.com/w/cpp/numeric/fenv/feexceptflag). + /// \param flagp adress to take flag state from + /// \param excepts OR of flags to restore + /// \retval 0 for success + inline int fesetexceptflag(const int *flagp, int excepts) { detail::errflags() = (detail::errflags()|(*flagp&excepts)) & (*flagp|~excepts); return 0; } + + /// Throw C++ exceptions based on set exception flags. + /// This function manually throws a corresponding C++ exception if one of the specified flags is set, + /// no matter if automatic throwing (via [HALF_ERRHANDLING_THROW_...](\ref HALF_ERRHANDLING_THROW_INVALID)) is enabled or not. + /// This function works even if [automatic exception flag handling](\ref HALF_ERRHANDLING_FLAGS) is disabled, + /// but in that case manual flag management is the only way to raise flags. + /// \param excepts OR of exceptions to test + /// \param msg error message to use for exception description + /// \throw std::domain_error if `FE_INVALID` or `FE_DIVBYZERO` is selected and set + /// \throw std::overflow_error if `FE_OVERFLOW` is selected and set + /// \throw std::underflow_error if `FE_UNDERFLOW` is selected and set + /// \throw std::range_error if `FE_INEXACT` is selected and set + inline void fethrowexcept(int excepts, const char *msg = "") + { + excepts &= detail::errflags(); + if(excepts & (FE_INVALID|FE_DIVBYZERO)) + throw std::domain_error(msg); + if(excepts & FE_OVERFLOW) + throw std::overflow_error(msg); + if(excepts & FE_UNDERFLOW) + throw std::underflow_error(msg); + if(excepts & FE_INEXACT) + throw std::range_error(msg); + } + /// \} +} + + +#undef HALF_UNUSED_NOERR +#undef HALF_CONSTEXPR +#undef HALF_CONSTEXPR_CONST +#undef HALF_CONSTEXPR_NOERR +#undef HALF_NOEXCEPT +#undef HALF_NOTHROW +#undef HALF_THREAD_LOCAL +#undef HALF_TWOS_COMPLEMENT_INT +#ifdef HALF_POP_WARNINGS + #pragma warning(pop) + #undef HALF_POP_WARNINGS +#endif + +#endif diff --git a/mexFiles/logFile/emClarity.logfile b/mexFiles/logFile/emClarity.logfile new file mode 100644 index 00000000..df6719f0 --- /dev/null +++ b/mexFiles/logFile/emClarity.logfile @@ -0,0 +1,16 @@ + + *************************************** + +emClarity version 725839e +run starting on 15-Feb-2024 06:49:26 +cmd int matlab -nodisplay -nosplash + + *************************************** + +Error using textscan +Invalid file identifier. Use fopen to generate a valid file identifier. +Error in BH_parseParameterFile (line 7) + +Error in emClarity>emC_testParse (line 691) + +Error in emClarity (line 165) diff --git a/mexFiles/mexFP16.cu b/mexFiles/mexFP16.cu new file mode 100644 index 00000000..07f6762e --- /dev/null +++ b/mexFiles/mexFP16.cu @@ -0,0 +1,167 @@ + +#include "include/core_headers.cuh" + +#define mexFP16_DEBUG_PRINT(args) mexPrintf("%s\n", args) +// #define mexFP16_DEBUG_PRINT() +__global__ void convert_fp16_to_fp32(const uint16_t* __restrict__ input_half, float* __restrict__ output_single, const int N) { + // Could be improved with a simple vector load. + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < N; idx += gridDim.x * blockDim.x) + output_single[idx] = __half2float(__ushort_as_half(input_half[idx])); + +} + +__global__ void convert_fp32_to_fp16(const float* __restrict__ input_single, uint16_t* __restrict__ output_half, const int N) { + // Could be improved with a simple vector store. + for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < N; idx += gridDim.x * blockDim.x) + output_half[idx] = __half_as_ushort(__float2half_rn(input_single[idx])); + +} + + + +void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { + + if (nrhs != 3) { + mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + "This function requires 2 input matrices and a boolean."); + } + + bool single_array_is_on_gpu = false; + bool half_array_is_on_gpu = false; + // First check to see if we have a gpu arra + if (mxIsGPUArray(prhs[0])) { + // Now let's see if it isvalid data + if (!mxGPUIsValidGPUData(prhs[0])) { + mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + "This function requires the first input to be a valid gpuArray."); + } + // And if it is single precision + if (mxGPUGetClassID((const mxGPUArray *)prhs[0]) != mxSINGLE_CLASS) { + mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + "This function requires the first input to be of type single mxGPUGetClassID."); + } + single_array_is_on_gpu = true; + } + // Same thing for the half array + if (mxIsGPUArray(prhs[1])) { + if (!mxGPUIsValidGPUData(prhs[1])) { + mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + "This function requires the second input to be a valid gpuArray."); + } + if (mxGPUGetClassID((const mxGPUArray *)prhs[1]) != mxUINT16_CLASS) { + mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + "This function requires the second input to be of type uint16 mxGPUGetClassID."); + } + half_array_is_on_gpu = true; + } + + if (!mxIsLogical(prhs[2])) { + mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + "This function requires the third input to be of type logical."); + } + if (mxGetNumberOfElements(prhs[0]) != mxGetNumberOfElements(prhs[1])) { + mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + "The number of elements in the input arrays must be the same."); + } + + size_t n_elements = mxGetNumberOfElements(prhs[0]); + + float* input_single = (float *) mxGetData(prhs[0]); + uint16_t* input_uint16 = (uint16_t *) mxGetData(prhs[1]); + bool* cast_to_fp16 = (bool *) mxGetData(prhs[2]); + + half_float::half* half_ptr = (half_float::half *)(mxGetData(prhs[1])); + + // First, we need to take care of any data conversion needed. + float* temporary_single; + uint16_t* temporary_uint16; + + // Output half on gpu so we need to use the conversion kernel + if (cast_to_fp16[0]) { + mexFP16_DEBUG_PRINT("Casting to half\n"); + if (half_array_is_on_gpu) { + mexFP16_DEBUG_PRINT("Casting to half that is on device already\n"); + temporary_uint16 = input_uint16; + if (single_array_is_on_gpu) { + temporary_single = input_single; + } + else { + mexFP16_DEBUG_PRINT("Copying single to device\n"); + checkCudaErrors(cudaMallocAsync(&temporary_single, n_elements * sizeof(float), cudaStreamPerThread)); + checkCudaErrors(cudaMemcpyAsync(temporary_single, input_single, n_elements * sizeof(float), cudaMemcpyHostToDevice, cudaStreamPerThread)); + } + convert_fp32_to_fp16<<<1024, 32, 0, cudaStreamPerThread>>>(temporary_single, temporary_uint16, n_elements); + + if (!single_array_is_on_gpu) + checkCudaErrors(cudaFreeAsync(temporary_single, cudaStreamPerThread)); + + checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); + } + else { + mexFP16_DEBUG_PRINT("Casting to half that is on host already\n"); + // Even though we could do this in place, then only copy over the half precision data to the host, + // we want to keep the input data un altered. + if (single_array_is_on_gpu) { + mexFP16_DEBUG_PRINT("Copying single to host\n"); + checkCudaErrors(cudaMallocHost(&temporary_single, n_elements * sizeof(float))); + checkCudaErrors(cudaMemcpyAsync(temporary_single, input_single, n_elements * sizeof(float), cudaMemcpyDeviceToHost, cudaStreamPerThread)); + checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); + } + else + temporary_single = input_single; + + mexFP16_DEBUG_PRINT(std::to_string(n_elements)); + for (int i = 0; i < n_elements ; i++) { + half_ptr[i] = half_float::half(temporary_single[i]); + } + + if (single_array_is_on_gpu) + checkCudaErrors(cudaFreeHost(temporary_single)); + + } + } + else { + mexFP16_DEBUG_PRINT("Casting to single\n"); + // Casting from half to single + if (single_array_is_on_gpu) { + temporary_single = input_single; + if (half_array_is_on_gpu) { + temporary_uint16 = input_uint16; + } + else { + checkCudaErrors(cudaMallocAsync(&temporary_uint16, n_elements * sizeof(uint16_t), cudaStreamPerThread)); + checkCudaErrors(cudaMemcpyAsync(temporary_uint16, input_uint16, n_elements * sizeof(uint16_t), cudaMemcpyHostToDevice, cudaStreamPerThread)); + } + convert_fp16_to_fp32<<<1024, 32, 0, cudaStreamPerThread>>>(temporary_uint16, temporary_single, n_elements); + + if (!half_array_is_on_gpu) + checkCudaErrors(cudaFreeAsync(temporary_uint16, cudaStreamPerThread)); + + checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); + } + else { + mexFP16_DEBUG_PRINT("Casting to single that is on host already\n"); + // the output single is on the host + if (half_array_is_on_gpu) { + mexFP16_DEBUG_PRINT("Copying half to host\n"); + checkCudaErrors(cudaMallocHost(&temporary_uint16, n_elements * sizeof(uint16_t))); + checkCudaErrors(cudaMemcpyAsync(temporary_uint16, input_uint16, n_elements * sizeof(uint16_t), cudaMemcpyDeviceToHost, cudaStreamPerThread)); + checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); + half_ptr = (half_float::half *)(temporary_uint16); + } + + mexFP16_DEBUG_PRINT("n_elements " + std::to_string(n_elements)); + for (int i = 0; i < n_elements ; i++) { + input_single[i] = half_float::half_cast(half_ptr[i]); + } + + if (half_array_is_on_gpu) + checkCudaErrors(cudaFreeHost(temporary_uint16)); + } + } // end if if cast to fp16 else to single + + + return; + + +} \ No newline at end of file From 83b3a2396cb55f5e75e885a16d4e82e7b67fe560 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 22 Mar 2024 10:24:48 -0400 Subject: [PATCH 072/151] other things --- .gitignore | 1 + mexFiles/logFile/emClarity.logfile | 16 ---------------- 2 files changed, 1 insertion(+), 16 deletions(-) delete mode 100644 mexFiles/logFile/emClarity.logfile diff --git a/.gitignore b/.gitignore index 64ed841b..dcd8dc51 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ mexFiles/compiled +mexFiles/logFile/ !mexFiles/compiled/emC_autoAlign.sh bin/ lib/ diff --git a/mexFiles/logFile/emClarity.logfile b/mexFiles/logFile/emClarity.logfile deleted file mode 100644 index df6719f0..00000000 --- a/mexFiles/logFile/emClarity.logfile +++ /dev/null @@ -1,16 +0,0 @@ - - *************************************** - -emClarity version 725839e -run starting on 15-Feb-2024 06:49:26 -cmd int matlab -nodisplay -nosplash - - *************************************** - -Error using textscan -Invalid file identifier. Use fopen to generate a valid file identifier. -Error in BH_parseParameterFile (line 7) - -Error in emClarity>emC_testParse (line 691) - -Error in emClarity (line 165) From 04906e25dbb474315a43b890ba3fe140d737c1de Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 22 Mar 2024 11:50:33 -0400 Subject: [PATCH 073/151] CPU to GPU swap during half cast seems to be working --- mexFiles/mexFP16.cu | 95 ++++++++++++++++++++++++-------------- testScripts/emc_halfcast.m | 20 ++++++-- 2 files changed, 75 insertions(+), 40 deletions(-) diff --git a/mexFiles/mexFP16.cu b/mexFiles/mexFP16.cu index 07f6762e..202b84c9 100644 --- a/mexFiles/mexFP16.cu +++ b/mexFiles/mexFP16.cu @@ -1,8 +1,9 @@ #include "include/core_headers.cuh" -#define mexFP16_DEBUG_PRINT(args) mexPrintf("%s\n", args) -// #define mexFP16_DEBUG_PRINT() +// #define mexFP16_DEBUG_PRINT(args) mexPrintf("%s\n", args) +#define mexFP16_DEBUG_PRINT(...) + __global__ void convert_fp16_to_fp32(const uint16_t* __restrict__ input_half, float* __restrict__ output_single, const int N) { // Could be improved with a simple vector load. for (int idx = blockIdx.x * blockDim.x + threadIdx.x; idx < N; idx += gridDim.x * blockDim.x) @@ -21,11 +22,10 @@ __global__ void convert_fp32_to_fp16(const float* __restrict__ input_single, uin void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { - if (nrhs != 3) { + if (nrhs != 4) { mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", - "This function requires 2 input matrices and a boolean."); + "This function requires 2 input matrices, a boolean (to_half), and an int (*n_elements)."); } - bool single_array_is_on_gpu = false; bool half_array_is_on_gpu = false; // First check to see if we have a gpu arra @@ -35,23 +35,26 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", "This function requires the first input to be a valid gpuArray."); } - // And if it is single precision - if (mxGPUGetClassID((const mxGPUArray *)prhs[0]) != mxSINGLE_CLASS) { - mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", - "This function requires the first input to be of type single mxGPUGetClassID."); - } + // And if it is single precision ( written this way causes a segfault, trust that emc_halfcast has done the checking.) + // if (mxGPUGetClassID((const mxGPUArray *)prhs[0]) != mxSINGLE_CLASS) { + // mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + // "This function requires the first input to be of type single mxGPUGetClassID."); + // } single_array_is_on_gpu = true; } + // Same thing for the half array if (mxIsGPUArray(prhs[1])) { if (!mxGPUIsValidGPUData(prhs[1])) { mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", "This function requires the second input to be a valid gpuArray."); } - if (mxGPUGetClassID((const mxGPUArray *)prhs[1]) != mxUINT16_CLASS) { - mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", - "This function requires the second input to be of type uint16 mxGPUGetClassID."); - } + //( written this way causes a segfault, trust that emc_halfcast has done the checking.) + // if (mxGPUGetClassID((const mxGPUArray *)prhs[1]) != mxUINT16_CLASS) { + // mexPrintf("Here %d\n", __LINE__); + // mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", + // "This function requires the second input to be of type uint16 mxGPUGetClassID."); + // } half_array_is_on_gpu = true; } @@ -59,18 +62,37 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", "This function requires the third input to be of type logical."); } - if (mxGetNumberOfElements(prhs[0]) != mxGetNumberOfElements(prhs[1])) { + + if (!mxIsInt64(prhs[3])) { mexErrMsgIdAndTxt("MATLAB:mexFP16:rhs", - "The number of elements in the input arrays must be the same."); + "This function requires the fourth input to be of type int."); } - size_t n_elements = mxGetNumberOfElements(prhs[0]); - float* input_single = (float *) mxGetData(prhs[0]); - uint16_t* input_uint16 = (uint16_t *) mxGetData(prhs[1]); + float* input_single; + uint16_t* input_uint16; + + // mxGPUCreateFromMxArray will return a read only pointer if the underlying is a matlab gpuArray + // To avoid a copy but still get a pointer we can cast the const away. + // This seems dodgy as fuck + if (single_array_is_on_gpu) { + mxGPUArray const * inputArray = mxGPUCreateFromMxArray(prhs[0]); + input_single = (float *) mxGPUGetData((mxGPUArray *)inputArray); + } + else + input_single = (float *) mxGetData(prhs[0]); + + if (half_array_is_on_gpu) { + mxGPUArray const * inputArray = mxGPUCreateFromMxArray(prhs[1]); + input_uint16 = (uint16_t *) mxGPUGetData((mxGPUArray *)inputArray); + } + else + input_uint16 = (uint16_t *) mxGetData(prhs[1]); + bool* cast_to_fp16 = (bool *) mxGetData(prhs[2]); + size_t* n_elements = (size_t *) mxGetData(prhs[3]); + - half_float::half* half_ptr = (half_float::half *)(mxGetData(prhs[1])); // First, we need to take care of any data conversion needed. float* temporary_single; @@ -87,10 +109,10 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { } else { mexFP16_DEBUG_PRINT("Copying single to device\n"); - checkCudaErrors(cudaMallocAsync(&temporary_single, n_elements * sizeof(float), cudaStreamPerThread)); - checkCudaErrors(cudaMemcpyAsync(temporary_single, input_single, n_elements * sizeof(float), cudaMemcpyHostToDevice, cudaStreamPerThread)); + checkCudaErrors(cudaMallocAsync(&temporary_single, *n_elements * sizeof(float), cudaStreamPerThread)); + checkCudaErrors(cudaMemcpyAsync(temporary_single, input_single, *n_elements * sizeof(float), cudaMemcpyHostToDevice, cudaStreamPerThread)); } - convert_fp32_to_fp16<<<1024, 32, 0, cudaStreamPerThread>>>(temporary_single, temporary_uint16, n_elements); + convert_fp32_to_fp16<<<1, 1, 0, cudaStreamPerThread>>>(temporary_single, temporary_uint16, *n_elements); if (!single_array_is_on_gpu) checkCudaErrors(cudaFreeAsync(temporary_single, cudaStreamPerThread)); @@ -103,15 +125,17 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { // we want to keep the input data un altered. if (single_array_is_on_gpu) { mexFP16_DEBUG_PRINT("Copying single to host\n"); - checkCudaErrors(cudaMallocHost(&temporary_single, n_elements * sizeof(float))); - checkCudaErrors(cudaMemcpyAsync(temporary_single, input_single, n_elements * sizeof(float), cudaMemcpyDeviceToHost, cudaStreamPerThread)); + checkCudaErrors(cudaMallocHost(&temporary_single, *n_elements * sizeof(float))); + checkCudaErrors(cudaMemcpyAsync(temporary_single, input_single, *n_elements * sizeof(float), cudaMemcpyDeviceToHost, cudaStreamPerThread)); checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); } else temporary_single = input_single; - mexFP16_DEBUG_PRINT(std::to_string(n_elements)); - for (int i = 0; i < n_elements ; i++) { + mexFP16_DEBUG_PRINT(std::to_string(*n_elements)); + half_float::half* half_ptr = (half_float::half *)(input_uint16); + + for (int i = 0; i < *n_elements ; i++) { half_ptr[i] = half_float::half(temporary_single[i]); } @@ -129,10 +153,10 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { temporary_uint16 = input_uint16; } else { - checkCudaErrors(cudaMallocAsync(&temporary_uint16, n_elements * sizeof(uint16_t), cudaStreamPerThread)); - checkCudaErrors(cudaMemcpyAsync(temporary_uint16, input_uint16, n_elements * sizeof(uint16_t), cudaMemcpyHostToDevice, cudaStreamPerThread)); + checkCudaErrors(cudaMallocAsync(&temporary_uint16, *n_elements * sizeof(uint16_t), cudaStreamPerThread)); + checkCudaErrors(cudaMemcpyAsync(temporary_uint16, input_uint16, *n_elements * sizeof(uint16_t), cudaMemcpyHostToDevice, cudaStreamPerThread)); } - convert_fp16_to_fp32<<<1024, 32, 0, cudaStreamPerThread>>>(temporary_uint16, temporary_single, n_elements); + convert_fp16_to_fp32<<<1024, 32, 0, cudaStreamPerThread>>>(temporary_uint16, temporary_single, *n_elements); if (!half_array_is_on_gpu) checkCudaErrors(cudaFreeAsync(temporary_uint16, cudaStreamPerThread)); @@ -142,16 +166,17 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { else { mexFP16_DEBUG_PRINT("Casting to single that is on host already\n"); // the output single is on the host + half_float::half* half_ptr = (half_float::half *)(input_uint16); if (half_array_is_on_gpu) { mexFP16_DEBUG_PRINT("Copying half to host\n"); - checkCudaErrors(cudaMallocHost(&temporary_uint16, n_elements * sizeof(uint16_t))); - checkCudaErrors(cudaMemcpyAsync(temporary_uint16, input_uint16, n_elements * sizeof(uint16_t), cudaMemcpyDeviceToHost, cudaStreamPerThread)); + checkCudaErrors(cudaMallocHost(&temporary_uint16, *n_elements * sizeof(uint16_t))); + checkCudaErrors(cudaMemcpyAsync(temporary_uint16, input_uint16, *n_elements * sizeof(uint16_t), cudaMemcpyDeviceToHost, cudaStreamPerThread)); checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); half_ptr = (half_float::half *)(temporary_uint16); } - - mexFP16_DEBUG_PRINT("n_elements " + std::to_string(n_elements)); - for (int i = 0; i < n_elements ; i++) { + + for (int i = 0; i < *n_elements ; i++) { + float tmp = half_float::half_cast(half_ptr[i]); input_single[i] = half_float::half_cast(half_ptr[i]); } diff --git a/testScripts/emc_halfcast.m b/testScripts/emc_halfcast.m index f4e2f2fc..551d5d49 100644 --- a/testScripts/emc_halfcast.m +++ b/testScripts/emc_halfcast.m @@ -7,7 +7,7 @@ to_gpu = false; to_cpu = false; to_half = false; - + % Determin if we are going to or from half based on the input precision. switch underlyingType(input_vol) case 'uint16' @@ -26,12 +26,22 @@ else to_gpu = true; end + else + if isa(input_vol, 'gpuArray') + to_gpu = true; + else + to_cpu = true; + end end if (to_cpu && to_gpu) error('Cannot convert to and from GPU at the same time'); end + % fprintf("To half: %d\n", to_half); + % fprintf("To GPU: %d\n", to_gpu); + % fprintf("To CPU: %d\n", to_cpu); + % fprintf("Swap: %d\n", swap_host_device); if (to_half) if (to_gpu) @@ -39,14 +49,14 @@ else output_vol = zeros(size(input_vol), 'uint16'); end - mexFP16(input_vol, output_vol, true); + mexFP16(input_vol, output_vol, to_half, int64(numel(input_vol))); else if (to_gpu) - output_vol = zeros(size(input_vol), 'single', 'gpuArray'); + output_vol = zeros(size(input_vol), 'single', 'gpuArray')+2; else - output_vol = zeros(size(input_vol), 'single'); + output_vol = zeros(size(input_vol), 'single') +1; end - mexFP16(output_vol, input_vol, false); + mexFP16(output_vol, input_vol, to_half, int64(numel(input_vol))); end end \ No newline at end of file From bcb9c96c0b05772ddaf14de52aee15ed2c420e2a Mon Sep 17 00:00:00 2001 From: himesb Date: Sun, 24 Mar 2024 09:39:46 -0400 Subject: [PATCH 074/151] Fixes bug in ctf update where new xform was used but not saved, turn off epxerimental flag in template search, more work on halfcast with gpu/cpu transfer embedded. --- alignment/BH_templateSearch3d_2.m | 103 +++++++++--------------------- ctf/BH_ctf_Correct3d.m | 2 +- ctf/BH_ctf_Updatefft.m | 9 ++- mexFiles/mexFP16.cu | 17 +++-- synthetic/BH_synthetic_mapBack.m | 51 ++++++++------- testScripts/emc_halfcast.m | 8 ++- 6 files changed, 83 insertions(+), 107 deletions(-) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index eb2f6a6f..ca3c0303 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -405,12 +405,12 @@ wanted_storage_precision = 'single'; if (test_half) - wanted_storage_precision = 'half'; + wanted_storage_precision = 'uint16'; end -tomoStack = zeros([sizeChunk,nTomograms], 'single'); +tomoStack = zeros([sizeChunk,nTomograms], wanted_storage_precision); if test_local - localStack = zeros([sizeChunk,nTomograms], 'single'); + localStack = zeros([sizeChunk,nTomograms], wanted_storage_precision); end % tomoNonZero = zeros(nTomograms,6,'uint64'); @@ -489,42 +489,13 @@ rmsMask = BH_movingAverage_2(tomoChunk.^2, statsRadius(1)); rmsMask = sqrt(rmsMask - averageMask.^2); - % SAVE_IMG(BH_padZeros3d(real(single(... - % (rmsMask))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'),'rmsMask.mrc'); - % SAVE_IMG(templateMask,'templateMask.mrc'); - % tempPADMask = zeros(size(tomoChunk),'single','gpuArray'); - % tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... - % padBIN(1,2)+1: end - padBIN(2,2), ... - % padBIN(1,3)+1: end - padBIN(2,3)) = templateMask; - % tempPADMask = tempPADMask ./ sum(tempPADMask(:)); - % SAVE_IMG(tempPADMask,'tempPADMaskPre.mrc'); - % tempPADMask = (conj(bhF.fwdFFT(bhF.normalization_factor.^-1 .* tempPADMask )));% ./ (sum(tempPADMask > 0.01)./sum(tempPADMask(:))))); - % SAVE_IMG(bhF.invFFT(conj(tempPADMask)),'tempPADMask.mrc'); - % ms = real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor.^3 .*tomoChunk.^2.*validCalcMask),'fwd').*tempPADMask)); - % ma = real(bhF.invFFT(bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor.^3 .*tomoChunk.*validCalcMask),'fwd').*tempPADMask)).^2; - % md = BH_padZeros3d(real(single(... - % (ms-ma))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'); - % SAVE_IMG(md,'md.mrc'); - % SAVE_IMG(sqrt(md), 'smd.mrc') - % SAVE_IMG(BH_padZeros3d(real(single(... - % ms)),... - % trimValid(1,:),trimValid(2,:),'GPU','single'),'ms.mrc'); - % SAVE_IMG(BH_padZeros3d(real(single(... - % (ma))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'),'ma.mrc'); - % error('asdf') - % correctedRMS = (ms - ma); - % correctedRMS = sqrt(BH_padZeros3d(real(single(... - % correctedRMS)),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single')); - % mean(correctedRMS(:)) - % 1/mean(correctedRMS(:)) - % SAVE_IMG(correctedRMS,'correctedRMS.mrc'); - % error('asdf') + if (test_local) - localStack(:,:,:,tomoIDX) = gather(rmsMask); + if (test_half) + localStack(:,:,:,tomoIDX) = emc_halfcast(rmsMask, true); + else + localStack(:,:,:,tomoIDX) = gather(rmsMask); + end else if ~(scale_mip) tomoChunk = (tomoChunk - averageMask) ./ rmsMask; @@ -532,12 +503,8 @@ end clear rmsMask averageMask - - tomoChunk = gather(tomoChunk .*validCalcMask); - - tmp_sum = sum(tomoChunk(validCalcMask > 0.1)); % REVERT % tmp_sum = sum(tomoChunk(:)); @@ -545,8 +512,12 @@ fullX2 = fullX2 + gather(tmp_sum.^2); fullnX = fullnX + gather(prod(sizeChunk)); - - tomoStack(:,:,:,tomoIDX) = tomoChunk; + if (test_half) + % The default is to return uint16 on the same device (host in this case) + tomoStack(:,:,:,tomoIDX) = emc_halfcast(tomoChunk); + else + tomoStack(:,:,:,tomoIDX) = tomoChunk; + end tomoCoords(tomoIDX,:) = [cutX,cutY,cutZ]; tomoIDX = tomoIDX + 1; @@ -558,11 +529,7 @@ % Normalize the global variance globalVariance = (fullX2/fullnX) - (fullX/fullnX)^2; -%fprintf('After local normalization, scaling also the global variance %3.3e\n',globalVariance); -% for iChunk = 1:tomoIDX-1 -% tomoStack(:,:,:,iChunk) = tomoStack(:,:,:,iChunk) ./sqrt(globalVariance); -% end clear tomoWedgeMask validCalcMask bandpassFilter statBinary tomoChunk @@ -580,19 +547,7 @@ totalTime = 0; firstLoopOverTomo = true; -% Center the spectrum by multiplication not swapping (this should just -% be in the fourierTransformer class if it isn't already) -% % swapPhase(obj, inputVol, direction) with fwd should do it -% [dU,dV,dW] = BH_multi_gridCoordinates(size(tomoStack(:,:,:,1)),... -% 'Cartesian','GPU', ... -% {'none'},1,1,0); -% swapQuadrants = exp((-2i*pi).*(dU.*(floor(size(dU,1)/2)+1) + ... -% (dV.*(floor(size(dV,2)/2)+1) + ... -% (dW.*(floor(size(dW,3)/2)+1))))); -% clear dU dV dW - -% swapQuadrants = swapQuadrants(1:floor(size(swapQuadrants,1)/2)+1,:,:); if (emc.use_new_grid_search) theta_search = 1:gridSearch.number_of_out_of_plane_angles; @@ -659,10 +614,20 @@ ,iAngle,size(angleStep,1), tomoIDX,nTomograms); end - - tomoFou = gpuArray(tomoStack(:,:,:,tomoIDX)); + if (test_half) + % Convert and return on GPU + tomoFou = emc_halfcast(tomoStack(:,:,:,tomoIDX), true); + else + tomoFou = gpuArray(tomoStack(:,:,:,tomoIDX)); + end + if test_local - localFou = BH_padZeros3d(localStack(:,:,:,tomoIDX),trimValid(1,:),trimValid(2,:),'GPU','single'); + if (test_half) + localFou = BH_padZeros3d(emc_halfcast(localStack(:,:,:,tomoIDX), true),trimValid(1,:),trimValid(2,:),'GPU','single'); + else + localFou = BH_padZeros3d(localStack(:,:,:,tomoIDX),trimValid(1,:),trimValid(2,:),'GPU','single'); + end + % localStack(:,:,:,tomoIDX)); end % % profile on @@ -672,15 +637,7 @@ tomoFou = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)), 'fwd'); - - - % profile on - % if (scale_mip) - % tomoFou_2 = swapQuadrants.*bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou.^2)); - % end - - - % tomoFou = swapQuadrants.*bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)); + @@ -728,8 +685,6 @@ % I should probaly switch to using the SF3D masked reference, but that also changes the baseline implementation % so I'll leave it for now. - - tempPADMask = tempPADMask .* 0; tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index a175d4c8..649f2b08 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -746,7 +746,7 @@ recombineCMD = fopen(file_of_outputs,'w'); fprintf(recombineCMD,'%d\n', n_total_sections); - cleanup3 = sprintf('rm %s',file_of_outputs); + cleanup3 = sprintf('rm -f %s',file_of_outputs); % if (use_inverted_newstack) % slab_order = n_slabs_to_reconstruct:-1:1; % else diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 6c9df6e7..ba9c3a05 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -182,6 +182,7 @@ INPUT_CELL{1,7} = ctfPath; else if ~exist(tlt{1}, 'file') + error('The file %s is not found.', tlt{1}); fprintf('\nignoring %s, because the file is not found.\n', tlt{1}); end if ~exist(PRJ_STACK{1}, 'file') @@ -319,6 +320,9 @@ error('xfprod failed'); else mbEST = load(tmp_combined_xf, '-ascii'); + for i = 1:d3 + tlt_tmp{i}([7,8,9,10,2,3]) = mbEST(i,:); + end end else error('Why would we get here?') @@ -360,14 +364,13 @@ % % In case some implicit expansion were to happen for whatever reason. % error('dXYZ is a matrix and should be a vector'); % end + % This is now updated above dXYZ(1:2) = tlt_tmp{i}(2:3); combinedXF = tlt_tmp{i}(7:10); - tlt_tmp{i}(2:3) = dXYZ(1:2); - % combinedXF = reshape((newXF*origXF)',1,4); - tlt_tmp{i}(7:10) = combinedXF; + % tlt_tmp{i}(7:10) = combinedXF; else combinedXF = tlt_tmp{i}(7:10); dXYZ = [tlt_tmp{i}(2:3),0]; diff --git a/mexFiles/mexFP16.cu b/mexFiles/mexFP16.cu index 202b84c9..741420a8 100644 --- a/mexFiles/mexFP16.cu +++ b/mexFiles/mexFP16.cu @@ -1,5 +1,6 @@ #include "include/core_headers.cuh" +#include // #define mexFP16_DEBUG_PRINT(args) mexPrintf("%s\n", args) #define mexFP16_DEBUG_PRINT(...) @@ -76,8 +77,12 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { // To avoid a copy but still get a pointer we can cast the const away. // This seems dodgy as fuck if (single_array_is_on_gpu) { + // mexEvalString("pause(3)"); mxGPUArray const * inputArray = mxGPUCreateFromMxArray(prhs[0]); input_single = (float *) mxGPUGetData((mxGPUArray *)inputArray); + // if we don't destroy this array we have a bad memory leak, yet it also doesn't seem like more memory is allocated + // inside this block based on pausing and watching. + mxGPUDestroyGPUArray(inputArray); } else input_single = (float *) mxGetData(prhs[0]); @@ -85,6 +90,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { if (half_array_is_on_gpu) { mxGPUArray const * inputArray = mxGPUCreateFromMxArray(prhs[1]); input_uint16 = (uint16_t *) mxGPUGetData((mxGPUArray *)inputArray); + mxGPUDestroyGPUArray(inputArray); } else input_uint16 = (uint16_t *) mxGetData(prhs[1]); @@ -92,6 +98,8 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { bool* cast_to_fp16 = (bool *) mxGetData(prhs[2]); size_t* n_elements = (size_t *) mxGetData(prhs[3]); + const size_t threads = 1024; + const size_t blocks = (threads / *n_elements + 1024 - 1) /threads; // First, we need to take care of any data conversion needed. @@ -112,7 +120,8 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { checkCudaErrors(cudaMallocAsync(&temporary_single, *n_elements * sizeof(float), cudaStreamPerThread)); checkCudaErrors(cudaMemcpyAsync(temporary_single, input_single, *n_elements * sizeof(float), cudaMemcpyHostToDevice, cudaStreamPerThread)); } - convert_fp32_to_fp16<<<1, 1, 0, cudaStreamPerThread>>>(temporary_single, temporary_uint16, *n_elements); + + convert_fp32_to_fp16<<>>(temporary_single, temporary_uint16, *n_elements); if (!single_array_is_on_gpu) checkCudaErrors(cudaFreeAsync(temporary_single, cudaStreamPerThread)); @@ -126,7 +135,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { if (single_array_is_on_gpu) { mexFP16_DEBUG_PRINT("Copying single to host\n"); checkCudaErrors(cudaMallocHost(&temporary_single, *n_elements * sizeof(float))); - checkCudaErrors(cudaMemcpyAsync(temporary_single, input_single, *n_elements * sizeof(float), cudaMemcpyDeviceToHost, cudaStreamPerThread)); + checkCudaErrors(cudaMemcpy(temporary_single, input_single, *n_elements * sizeof(float), cudaMemcpyDeviceToHost)); checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); } else @@ -156,7 +165,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { checkCudaErrors(cudaMallocAsync(&temporary_uint16, *n_elements * sizeof(uint16_t), cudaStreamPerThread)); checkCudaErrors(cudaMemcpyAsync(temporary_uint16, input_uint16, *n_elements * sizeof(uint16_t), cudaMemcpyHostToDevice, cudaStreamPerThread)); } - convert_fp16_to_fp32<<<1024, 32, 0, cudaStreamPerThread>>>(temporary_uint16, temporary_single, *n_elements); + convert_fp16_to_fp32<<>>(temporary_uint16, temporary_single, *n_elements); if (!half_array_is_on_gpu) checkCudaErrors(cudaFreeAsync(temporary_uint16, cudaStreamPerThread)); @@ -170,7 +179,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { if (half_array_is_on_gpu) { mexFP16_DEBUG_PRINT("Copying half to host\n"); checkCudaErrors(cudaMallocHost(&temporary_uint16, *n_elements * sizeof(uint16_t))); - checkCudaErrors(cudaMemcpyAsync(temporary_uint16, input_uint16, *n_elements * sizeof(uint16_t), cudaMemcpyDeviceToHost, cudaStreamPerThread)); + checkCudaErrors(cudaMemcpy(temporary_uint16, input_uint16, *n_elements * sizeof(uint16_t), cudaMemcpyDeviceToHost)); checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); half_ptr = (half_float::half *)(temporary_uint16); } diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 6759f891..24dcb541 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -242,7 +242,8 @@ - mapBackRePrjSize = min(256,subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize')); + mapBackRePrjSize = min(64,subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize')) + % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos if nTomograms == 0 @@ -350,6 +351,7 @@ end system(sprintf('mkdir -p %smapBack%d',tmpCache,mapBackIter+1)); + system(sprintf('mkdir -p cache/mapBack%d',mapBackIter+1)); % re-initialize the parpool for each tilt series to free up mem. if ~isempty(gcp('nocreate')) @@ -530,7 +532,7 @@ if any(emc.tomoCPR_target_n_patches_x_y) % This will be re-calculated once the tilt-series size is known. - targetPatchSize = floor([tiltHeader.nX,tiltHeader.nY] ./ emc.tomoCPR_target_n_patches_x_y); + targetPatchSize = floor([tiltHeader.nX,tiltHeader.nY].*samplingRate ./ emc.tomoCPR_target_n_patches_x_y); fprintf('\nUsing targetPatchSize of [%d,%d] for %s\n',targetPatchSize, tomoList{iTomo}); end @@ -796,10 +798,10 @@ % % % % It may be faster to work with a rotated vol since the reading in may cause % % % % problems, but the projection is so slow, that this isn't worth dealing with % % % % now. - if (nWorkers > 1) - chunkSize = ceil(sTY./nWorkers); - chunkInc = zeros(nWorkers,3); - for iWorker = 1:nWorkers-1 + if (emc.n_tilt_workers > 1) + chunkSize = ceil(sTY./emc.n_tilt_workers); + chunkInc = zeros(emc.n_tilt_workers,3); + for iWorker = 1:emc.n_tilt_workers-1 chunkInc(iWorker,:) = [iWorker,(iWorker-1)*chunkSize+1,iWorker*chunkSize]; end iWorker = iWorker +1 ; @@ -809,7 +811,7 @@ chunkInc = [1,1,sTY]; end - + % % % % % % % if (buildTomo) @@ -1000,7 +1002,6 @@ end % loop over error and masked tomo - fprintf(reModFile,['#!/bin/bash\n\n',... 'tilt -StandardInput << EOF\n',... 'input %s\n', ... @@ -1683,8 +1684,6 @@ % write the com script for running tiltalign RotDef = 5; TltDef = 4; - aliCom_name = sprintf('%smapBack%d/%s.align',mbOUT{1:3}); - aliCom = fopen(aliCom_name,'w'); if (emc.shift_z_to_to_centroid) final_line1 = 'ShiftZFromOriginal'; @@ -1706,6 +1705,9 @@ tilt_script_name = sprintf('cache/mapBack%d/%s%s',mbOUT{2},tn2,tn3); end + aliCom_name = sprintf('%smapBack%d/%s.align',mbOutAlt{1:3}); + aliCom = fopen(aliCom_name,'w'); + % Testing local alignment with optimized parameters using the new imod options for leave out fprintf(aliCom,['%smapBack%d/%s_fit-full.fid\n',... %1 '%smapBack%d/%s%s.3dmod\n',... %2 @@ -1732,8 +1734,8 @@ tilt_script_name,... emc.k_factor_scaling, ... mbOutAlt{1:3},outCTF, ... - targetPatchSize, ... - targetPatchSize,... + targetPatchSize(1), ... + targetPatchSize(2),... nFiducialsPerPatch, ... floor(nFiducialsPerPatch/3),... emc.min_overlap, ... @@ -1849,7 +1851,8 @@ fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}),'w'); fprintf(fOUT,'#!/bin/bash\n\n'); end - fprintf(fOUT,'cat %s | /scratch/etna/master_align.sh `xargs` &\n',aliCom_name); + % fprintf(fOUT,'cat %s | /scratch/etna/master_align.sh `xargs` &\n',aliCom_name); + fprintf(fOUT,'%s\n',aliCom_name); % Since we send to the background in a shell, makes sure the % function waits on children. @@ -1867,7 +1870,7 @@ % fprintf(fOUT,['%smapBack%d/%s.align > ',... % '%smapBack%d/%s.align_ta.log &\n'], ... % mbOutAlt{1:3},mbOutAlt{1:3}); - fprintf(fOUT,'cat %s | /scratch/etna/master_align.sh `xargs` &\n',aliCom_name); + fprintf(fOUT,'%s\n',aliCom_name); % Since we send to the background in a shell, makes sure the % function waits on children. @@ -1887,6 +1890,7 @@ %%%system(sprintf('grep -A %d " At minimum tilt" ./mapBack/%s_ta.log > tmp.log',nPrjs+2,TN)); %%%system(sprintf('awk ''{if(NR >3) print $5}'' tmp.log > mapBack/%s.mag',TN)); %%%end %uf cibdutuib + end % loop over tilts @@ -1896,20 +1900,21 @@ altFiles = sprintf('%smapBack%d/runAlignments_*.sh',mbOUT{1:2}); if (multi_node_run) - fOUT = fopen(mainFile,'w'); - fprintf(fOUT,'#!/bin/bash\n\n'); - fclose(fOUT); - + % fOUT = fopen(mainFile,'w'); + % fprintf(fOUT,'#!/bin/bash\n\n'); + % fclose(fOUT); + system(sprintf('rm %s && touch %s',mainFile,mainFile)); fprintf('Combining Results from alt and main\n'); system(sprintf('cat %s >> %s',altFiles,mainFile)); end - fOUT = fopen(mainFile,'a'); - fprintf(fOUT,'\nwait\n'); - fclose(fOUT); + % fOUT = fopen(mainFile,'a'); + % fprintf(fOUT,'\nwait\n'); + % fclose(fOUT); - system(sprintf('chmod a=wrx %smapBack%d/runAlignments.sh', mbOUT{1:2})); - system(sprintf('%smapBack%d/runAlignments.sh', mbOUT{1:2})); + % system(sprintf('chmod a=wrx %smapBack%d/runAlignments.sh', mbOUT{1:2})); + % system(sprintf('%smapBack%d/runAlignments.sh', mbOUT{1:2})); + system(sprintf('cat %smapBack%d/runAlignments.sh | parallel -j%d "cat {} | /scratch/etna/master_align.sh `xargs`"', mbOUT{1:2}, emc.nCpuCores)) end if ( conserveDiskSpace ) diff --git a/testScripts/emc_halfcast.m b/testScripts/emc_halfcast.m index 551d5d49..35b032ed 100644 --- a/testScripts/emc_halfcast.m +++ b/testScripts/emc_halfcast.m @@ -4,6 +4,10 @@ swap_host_device = false; end + % I'm not sure why, but this breaks when numel == 1 + if ~(numel(input_vol) > 1) + error('Input volume to emc_halfcast must have more than one element\n'); + end to_gpu = false; to_cpu = false; to_half = false; @@ -52,9 +56,9 @@ mexFP16(input_vol, output_vol, to_half, int64(numel(input_vol))); else if (to_gpu) - output_vol = zeros(size(input_vol), 'single', 'gpuArray')+2; + output_vol = zeros(size(input_vol), 'single', 'gpuArray'); else - output_vol = zeros(size(input_vol), 'single') +1; + output_vol = zeros(size(input_vol), 'single'); end mexFP16(output_vol, input_vol, to_half, int64(numel(input_vol))); end From 991d3a73e9f72ca5623c4399993ad3720caf0853 Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 25 Mar 2024 12:48:10 -0400 Subject: [PATCH 075/151] broken, but almost re-ordered alignRawv2 --- alignment/BH_alignRaw3d_v2.m | 1180 ++++++++---------- masking/BH_bandLimitCenterNormalize.m | 6 +- transformations/BH_multi_combineLowResInfo.m | 4 +- 3 files changed, 547 insertions(+), 643 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 355b3890..971547f0 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -159,7 +159,9 @@ nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]; - +if (nReferences(1) ~= length(refVector{1})) + error('Number of references does not match the number of unique groups in the classVector') +end nRefOut(1:2) = [length(unique(refGroup{1})) + sum(( refSym{1} < 0 )),... length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; @@ -182,7 +184,8 @@ BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms); [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) + if (flgStartThird) @@ -415,11 +418,11 @@ refOUT = cell(2.*nReferences(iGold),2); for iRef = 1:nReferences(iGold) - - refTMP_2 = refIMG{iGold}{iRef}; refIMG{iGold}{iRef} = []; - refTMP = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); + + + refTMP = refIMG{iGold}{iRef}(padWindow(1,1) + 1: end - padWindow(2,1), ... + padWindow(1,2) + 1: end - padWindow(2,2), ... + padWindow(1,3) + 1: end - padWindow(2,3)); % if not using a weighted average (adapted SPW filter), apply an @@ -432,34 +435,34 @@ - ref_FT2{iGold}{iRef} = gather(refTMP_2); + % ref_FT2{iGold}{iRef} = gather(refTMP_2); % Trim for output reference - refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); + % refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... + % padWindow(1,2) + 1: end - padWindow(2,2), ... + % padWindow(1,3) + 1: end - padWindow(2,3)); % Overwrite a copy of the filtered, bandpassed ref for output - refOUT{nOut} = real(ifftn(conj(ref_FT1{iGold}{iRef}))); - refOUT{nOut} = gather(refOUT{nOut}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); + % refOUT{nOut} = real(ifftn(conj(ref_FT1{iGold}{iRef}))); + % refOUT{nOut} = gather(refOUT{nOut}(padCalc(1,1) + 1: end - padCalc(2,1), ... + % padCalc(1,2) + 1: end - padCalc(2,2), ... + % padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); - refOUT{nOut} = refOUT{nOut}.*volMask; + % refOUT{nOut} = refOUT{nOut}.*volMask; - refOUT{nOut+1} = real(ifftn(BH_bandLimitCenterNormalize(... - refTMP_2, '', '', padCalc, 'single'))); - refOUT{nOut+1} = gather(refOUT{nOut+1}(padCalc(1,1) + 1: end - padCalc(2,1), ... - padCalc(1,2) + 1: end - padCalc(2,2), ... - padCalc(1,3) + 1: end - padCalc(2,3)) ); - nOut = nOut + 2; + % refOUT{nOut+1} = real(ifftn(BH_bandLimitCenterNormalize(... + % refTMP_2, '', '', padCalc, 'single'))); + % refOUT{nOut+1} = gather(refOUT{nOut+1}(padCalc(1,1) + 1: end - padCalc(2,1), ... + % padCalc(1,2) + 1: end - padCalc(2,2), ... + % padCalc(1,3) + 1: end - padCalc(2,3)) ); + % nOut = nOut + 2; - refOUT{nOut} = refOUT{nOut} - mean(refOUT{nOut}(:)); - refOUT{nOut} = refOUT{nOut} ./ rms(refOUT{nOut}(:)); + % refOUT{nOut} = refOUT{nOut} - mean(refOUT{nOut}(:)); + % refOUT{nOut} = refOUT{nOut} ./ rms(refOUT{nOut}(:)); - refOUT{nOut+1} = refOUT{nOut+1} - mean(refOUT{nOut+1}(:)); - refOUT{nOut+1} = refOUT{nOut+1} ./ rms(refOUT{nOut+1}(:)); + % refOUT{nOut+1} = refOUT{nOut+1} - mean(refOUT{nOut+1}(:)); + % refOUT{nOut+1} = refOUT{nOut+1} ./ rms(refOUT{nOut+1}(:)); end @@ -545,14 +548,13 @@ -try - EMC_parpool(nParProcesses+1) -catch - delete(gcp('nocreate')) - EMC_parpool(nParProcesses+1) -end +% try +% EMC_parpool(nParProcesses+1) +% catch +% delete(gcp('nocreate')) +% EMC_parpool(nParProcesses+1) +% end -size(ref_FT2) system('mkdir -p alignResume'); @@ -566,7 +568,9 @@ if (emc.force_no_symmetry) particle_symmetry = 'C1'; end -parfor iParProc = parVect +for iParProc = parVect + +% parfor iParProc = parVect symmetry = emc.symmetry; bestAngles_tmp = struct(); @@ -589,6 +593,7 @@ bestAngles_tmp.(tomoList{iTomo}) = load(previousAlignment); fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); else + fprintf('In the else\n'); % There is some memory leak somewhere that I haven't been able to figure % out. I am clearing all vars but output in the children functions ... this % isn't ideal, but for now is an acceptable stop gap. @@ -597,26 +602,11 @@ % shake up the random number generator for phi and theta rng('shuffle'); - bandpassFilt_tmp = cell(nReferences(1),1); - bandpassFiltREF_tmp = cell(nReferences(1),1); - for iRef = 1:nReferences(1) - if emc.multi_reference_alignment <= 2 - bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); - else - bandpassFilt_tmp{iRef} = (bandpassFilt{iRef}); - bandpassFiltREF_tmp{iRef} = (bandpassFiltREF{iRef}); - end - end - - - - ref_FT1_tmp = cell(2,1); - ref_FT2_tmp = cell(2,1); - ref_WGT_tmp = cell(2,1); - ref_WGT_rot = cell(2,1); - + bandpassFilt_tmp = cell(1,1); + bandpassFiltREF_tmp = cell(1,1); + + volMask_tmp = gpuArray(volMask); volBinary_tmp = single(find( volMask_tmp > 0.01 )); peakMaskInterpolator = ''; @@ -627,11 +617,8 @@ mip.('mask') = gpuArray(stat_mask); end - wCCC_tmp = cell(length(wCCC)); - - for iRef = 1:nReferences(1) for iWccc = 1:length(wCCC{iRef}) if (flgWeightCCC) @@ -643,26 +630,6 @@ end end - - - for iGold = 1:2 - for iRef = 1:nReferences(iGold) - if emc.multi_reference_alignment <= 2 - ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = gpuArray(refWgtROT{iGold}{iRef}); - else - % Temp workaround, six big ribo refs crashing - ref_FT1_tmp{iGold}{iRef} = (ref_FT1{iGold}{iRef}); - ref_FT2_tmp{iGold}{iRef} = (ref_FT2{iGold}{iRef}); - ref_WGT_tmp{iGold}{iRef} = (refWGT{iGold}{iRef}); - ref_WGT_rot{iGold}{iRef} = (refWgtROT{iGold}{iRef}); - end - end - end - - % sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) tic; @@ -672,17 +639,14 @@ tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); - + fprintf("line %d\n", 641); binShift = [0,0,0]; nSubTomos = size(positionList,1); - iTiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoName).tiltName; - - % Can't clear inside the parfor, but make sure we don't have two tomograms % in memory at once. @@ -704,7 +668,6 @@ volHeader = getHeader(volumeData); end - % For now, set up for full grid-search only, as I intend to just do % translational and in-plane searches for now anyhow. @@ -715,61 +678,77 @@ iv1 = []; iv2 = []; iv3 = []; cccStorageBest = cell(emc.nPeaks,1); cccStorageRefine = cell(emc.nPeaks,1); + cccStorage2 = cell(emc.nPeaks,1); + cccStorageTrans= zeros(1,10, 'single', 'gpuArray'); + cccInitial_arr = zeros(nSubTomos,10, 'single', 'gpuArray'); + cccInitial_arr(:,6) = -9999; + for iPeak = 1:emc.nPeaks - cccStorageBest{iPeak} = zeros(nSubTomos,10); - cccStorageRefine{iPeak}= zeros(nSubTomos,10); + cccStorageBest{iPeak} = zeros(nSubTomos,10, 'single'); + cccStorageRefine{iPeak}= zeros(nSubTomos,10, 'single'); + cccStorage2{iPeak} = zeros(nSubTomos,10, 'single', 'gpuArray'); end % reset for each tomogram wdgIDX = 0; - - for iSubTomo = 1:nSubTomos - - - make_SF3D = true; - breakPeak = 0; % for try catch on cut out vols - - for iPeak = 1:emc.nPeaks - - if (emc.track_stats) - measure_noise = true; - mip.('x') = {}; - mip.('x2') = {}; - mip.('N') = 0; - end - if (breakPeak) - continue; - end - getInitialCCC = 1; - cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); - cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); - - % Used in refinment loop - angCount = 1; - - % Check that the given subTomo is not to be ignored - classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); - particleIDX = positionList(iSubTomo, 4); - half_set = positionList(iSubTomo, 7); - - - % if classVector{half_set}(1,:) == 0 - % classPosition = 1; - % flgAllClasses = true; - % else - % classPosition = find(classVector{half_set}(1,:) == classIDX); - % flgAllClasses = false; - % end - % Align all valid subtomos, even if the do not belong to the classes we've selected as references. - % To ignore particles, remove them with geometry RemoveClases.m - % FIXME: what was this for? - flgAllClasses = true; - - + fprintf('nRefs %d\n',nReferences(1)); + for iRefIdx = 1:nReferences(1); + fprintf('iRefIdx %d\n',iRefIdx); + % Just use C1 to initialize, whether or not this is the final + refInterpolator = {2,1}; + refWdgInterpolator= {2,1}; + for iHalfSet = 1:2 + ref_FT1{iHalfSet}{iRefIdx} = gpuArray(ref_FT1{iHalfSet}{iRefIdx}); + refWgtROT{iHalfSet}{iRefIdx} = gpuArray(refWgtROT{iHalfSet}{iRefIdx}); + refInterpolator{iHalfSet} = interpolator(ifftn(ref_FT1{iHalfSet}{iRefIdx}),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator{iHalfSet} = interpolator(refWgtROT{iHalfSet}{iRefIdx},[0,0,0],[0,0,0],'Bah','forward','C1',false); + end + bandpassFilt_tmp{1} = gpuArray(bandpassFilt{iRefIdx}); + bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{iRefIdx}); + + for iSubTomo = 1:nSubTomos + fprintf('iSubTomo %d\n',iSubTomo); + make_SF3D = true; + breakPeak = 0; % for try catch on cut out vols - if (classIDX ~= -9999) && ... % All previously ignored particles - ( flgAllClasses || ismember(classIDX, classVector{half_set}(1,:)) ) + for iPeak = 1:emc.nPeaks + + if (emc.track_stats) + measure_noise = true; + mip.('x') = {}; + mip.('x2') = {}; + mip.('N') = 0; + end + if (breakPeak) + continue; + end + getInitialCCC = 1; + % Check that the given subTomo is not to be ignored + classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); + particleIDX = positionList(iSubTomo, 4); + half_set = positionList(iSubTomo, 7); + + if (classIDX == -9999) + continue; + end + + % In the case we are only aligning refs to those classes they came from, we + % need to check to see if we duck out here. + if( emc.multi_reference_alignment == 2 ) + refToAlign = find(classIDX == classVector{half_set}(1,:)); + if isempty(refToAlign) + fprintf('WARNING: wanted class %d not found in classVector\n',classIDX); + for i = 1:size(classVector{half_set},2) + fprintf('%d ',classVector{half_set}(1,i)); + end + error('classIDX not found in classVector'); + end + if (refToAlign ~= iRefIdx) + continue; + end + end + center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); @@ -786,16 +765,11 @@ sizeWindow,maskRadius, center); end - - - if ischar(indVAL) fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) nIgnored = nIgnored + 1; geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; else - - if (emc.flgCutOutVolumes) % Test with some generic padding , only to be used on bin 1 at % first!!! TODO add a flag to check this. @@ -819,7 +793,6 @@ [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... padVAL(2,1:3), 'GPU', 'singleTaper'); - if (make_SF3D) make_SF3D = false; % For now excluding the soften weight. @@ -829,14 +802,9 @@ % needed [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); - - % Just use C1 to initialize, whether or not this is the final - refInterpolator = ''; - refWdgInterpolator= ''; + particleInterpolator= ''; - [refInterpolator, ~] = interpolator(gpuArray(ref_FT2_tmp{1}{1}),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator = interpolator(gpuArray(ref_WGT_rot{half_set}{iRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); end @@ -862,7 +830,6 @@ phiInc = angleStep(iAngle,3); thetaInc = angleStep(iAngle,4); numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - end % To prevent only searching the same increments each time in a limited @@ -871,12 +838,9 @@ azimuthalRandomizer = (rand(1)-0.5)*phiInc; - % Calculate the increment in phi so that the azimuthal sampling is % consistent and equal to the out of plane increment. - - if (emc.use_new_grid_search) % FIXME randomizer passed as bool to eulerSearch phi_search = gridSearch.parameter_map.phi{iAngle}; @@ -884,7 +848,6 @@ phi_search = 0:angleStep(iAngle,2); end - for iAzimuth = phi_search if (emc.use_new_grid_search) @@ -893,25 +856,16 @@ else phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); psiInc = angleStep(iAngle,5); - end - for iInPlane = inPlaneSearch psi = iInPlane; - %[phi,theta,psi-phi]; - RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); RotMat = reshape(angles,3,3) * RotMat; - - cccStorageTrans= zeros(1.*nReferences(1),10,'gpuArray'); - + for alignLoop = 1:2 - - switch alignLoop - case 1 % This takes care of non-inter shift in the origin that is % ignored during the windowing of the particle. @@ -921,14 +875,10 @@ iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... padWindow(1,2) + 1:end - padWindow(2,2) , ... padWindow(1,3) + 1:end - padWindow(2,3) ); - case 2 - bestOfRefs = sortrows(gather(cccStorageTrans), -6); - %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); - - estPeakCoord = bestOfRefs(1,8:10); - + estPeakCoord = gather(cccStorageTrans(1,8:10)); + [ iTrimParticle ] = particleInterpolator.interp3d(... RotMat,... estPeakCoord,rotConvention ,... @@ -952,508 +902,458 @@ 'inv',particle_symmetry); end % switch on align loop - - switch emc.multi_reference_alignment - case 0 - refToAlign = 1; + switch alignLoop case 1 - refToAlign = 1:max(nReferences(:)); - case 2 - refToAlign = classIDX; - otherwise - error('emc.multi_reference_alignment is not 0,1,2') - end - - for iRef = refToAlign - - switch alignLoop + % use transpose of RotMat - case 1 - % use transpose of RotMat - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - estPeakCoord,rotConvention ,... - 'forward','C1'); - - - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - - - - % maybe I should be rotating peak mask here in case it has - % an odd shape, since we are leaving the proper frame - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{iRef} ,'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{iRef} ,'',padCalc,flgPrecision); - - if (emc.track_stats && measure_noise) - - - [ ~, mip ] = BH_multi_xcf_Translational_2( ... - rotPart_FT, ... - conj(iRotRef),... - ifftshift(iRotWdg),... - iMaxWedgeIfft,... - iRotMask, peakCOM,... - mip); - - - end - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - - cccStorageTrans(iRef,:) = [iRef, particleIDX, ... - phi, theta, psi - phi, ... - 0, 0, ... - peakCoord + estPeakCoord]; - case 2 - - % get starting point - if (getInitialCCC) - - initialRotPart_FT = BH_bandLimitCenterNormalize(... - iTrimInitial.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - - - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( initialRotPart_FT, ... - ref_FT1_tmp{half_set}{iRef}, ... - ifftshift(iWedgeInitial),... - ref_WGT_tmp{half_set}{iRef}, ... - wCCC_tmp{iRef}); - - - - - cccInitial(iRef,:) = [iRef, particleIDX, ... - 0,0,0, ... - iCCC, 1, ... - shiftVAL]; - - - initialRotPart_FT = []; - - - end - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); - + [ iRotRef ] = refInterpolator{iHalfSet}.interp3d(... + RotMat',... + estPeakCoord,rotConvention ,... + 'forward','C1'); + + [ iRotWdg ] = refWdgInterpolator{iHalfSet}.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + % maybe I should be rotating peak mask here in case it has + % an odd shape, since we are leaving the proper frame + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{1} ,'',... + [0,0,0;0,0,0],flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{1} ,'',padCalc,flgPrecision); + + if (emc.track_stats && measure_noise) + [ ~, mip ] = BH_multi_xcf_Translational_2( ... + rotPart_FT, ... + conj(iRotRef),... + ifftshift(iRotWdg),... + iMaxWedgeIfft,... + iRotMask, peakCOM,... + mip); + end + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + cccStorageTrans(1,:) = [iRefIdx, particleIDX, ... + phi, theta, psi - phi, ... + 0, 0, ... + peakCoord + estPeakCoord]; + case 2 + % get starting point + if (getInitialCCC) + initialRotPart_FT = BH_bandLimitCenterNormalize(... + iTrimInitial.*volMask_tmp,... + bandpassFilt_tmp{1} ,volBinary_tmp,padCalc,flgPrecision); [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{half_set}{iRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{half_set}{iRef}, ... - wCCC_tmp{iRef}); - - - - + BH_multi_xcf_Rotational( initialRotPart_FT, ... + ref_FT1{half_set}{iRefIdx}, ... + ifftshift(iWedgeInitial),... + refWGT{half_set}{iRefIdx}, ... + wCCC_tmp{iRefIdx}); - - % Note that no new translational estimate is made, so no - % need to multiply by RotMat - cccStorage2(angCount,:) = ... - [iRef, particleIDX, ... - phi, theta, psi , ... - iCCC, 1, ... - estPeakCoord]; - - - angCount = angCount + 1; - end - - - end % loop over references. - - - - end - % This volume won't be needed until the next subTomo is considered, - % which is also where getInitialCCC Boolean is set to True again. - iTrimInitial = []; - getInitialCCC = 0; - - end % in plane angles - end % azimuth - end % polar - - % % % fprintf('Power ratio is %3.3f\n',powerOut./powerInitial); - - cccPreRefineSort = sortrows(gather(cccStorage2),-6); - - if (length(refToAlign) > 1) - cccInitial = sortrows(gather(cccInitial), -6); - cccInitial = cccInitial(1,:); - else - cccInitial = gather(cccInitial(refToAlign,:)); + if ( iCCC > cccInitial_arr(iSubTomo,6)) + cccInitial_arr(iSubTomo,:) = [iRefIdx, particleIDX, ... + 0,0,0, ... + iCCC, 1, ... + shiftVAL]; + initialRotPart_FT = []; + end + end + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{1} ,volBinary_tmp,padCalc,flgPrecision); + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1{half_set}{iRefIdx},... + ifftshift(iWedgeMask),... + refWGT{half_set}{iRefIdx}, ... + wCCC_tmp{iRefIdx}); + + % Note that no new translational estimate is made, so no + % need to multiply by RotMat + if (iCCC > cccStorage2{iPeak}(iSubTomo,6) ) + + cccStorage2{iPeak}(iSubTomo,:) = ... + [iRefIdx, particleIDX, ... + phi, theta, psi , ... + iCCC, 1, ... + estPeakCoord]; + end + end %case + end % switch + % This volume won't be needed until the next subTomo is considered, + % which is also where getInitialCCC Boolean is set to True again. + iTrimInitial = []; + getInitialCCC = 0; % FIXME is this still right with the loop inversion + end % alignLoop (trans then rotation) + end % in plane angles (psi) + end % azimuth (phi) + end % polar (theta) + end % peak loop + end % subTomo loop + % Gather the ref back to the CPU as we'll need in in the subsequent alignment blocks + for iHalfSet = 1:2 + ref_FT1{iHalfSet}{iRefIdx} = gather(ref_FT1{iHalfSet}{iRefIdx}); + refWgtROT{iHalfSet}{iRefIdx} = gather(refWgtROT{iHalfSet}{iRefIdx}); + end + end % loop over references + + % Now we'll loop over the subtomos and peaks just grabbing the + nSubTomos + for iSubTomo = 1:nSubTomos + cccInitial = gather( cccInitial_arr(iSubTomo,:) ); + for iPeak = 1:emc.nPeaks + cccPreRefineSort = gather(cccStorage2{iPeak}(iSubTomo,:)); + if cccInitial(1,6 ) > cccPreRefineSort(1,6) + cccPreRefineSort(1,:) = cccInitial(1,:); + end + + % This only seems to be a problem with cut out volumes. + % Normalization maybe? + if ~any(cccPreRefineSort(1,:)) + cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); + continue + end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + if (flgRefine) + % Get the results from just this subTomo and sort on CCC + rRef = cccPreRefineSort(1,1); + rPart = cccPreRefineSort(1,2); + rPhi = cccPreRefineSort(1,3); + rPhiInc = phiInc / 4; + rTheta= cccPreRefineSort(1,4); + rTheInc = thetaInc /2; + rPsi = cccPreRefineSort(1,5); + rPsiInc = psiInc /2; + % Confirm shiftVAL is doing what it should be + rXYZest = cccPreRefineSort(1,8:10); + + bandpassFilt_tmp{1} = gpuArray(bandpassFilt{rRef}); + bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{rRef}); + refInterpolator = {2,1}; + refWdgInterpolator= {2,1}; + for iHalfSet = 1:2 + refInterpolator{iHalfSet} = interpolator(ifftn(gpuArray(ref_FT1{iHalfSet}{rRef})),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator{iHalfSet} = interpolator(gpuArray(refWgtROT{iHalfSet}{rRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); + end - end - - if cccInitial(1,6 ) > cccPreRefineSort(1,6) - cccPreRefineSort(1,:) = cccInitial(1,:); - end - - + if (rTheInc) + % For a larger out of plane step, search a larger range in plane + psiRefineStep = floor(sqrt(rTheInc)); + else + psiRefineStep = 1; + end + + thetaRefineStep =1; + phiRefineStep=2; + totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; + totalRefineStep = prod((2.*totalRefineStep)+1); - % This only seems to be a problem with cut out volumes. - % Normalization maybe? - if ~any(cccPreRefineSort(1,:)) - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); - continue + cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); + + if (rPsiInc == 0) + inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; + else + inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; + end + polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; + azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; + + searchList = zeros(totalRefineStep,3); + nSearch = 1; + for iPhi = azimuthalRefine + for iTheta = polarRefine + for iPsi = inPlaneRefine + % best iPsi is origin Psi - Phi, no need to subtract here. + + searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; + + nSearch = nSearch + 1; + end end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + end % end of building angle list - if (flgRefine) - - - - % Get the results from just this subTomo and sort on CCC - - rRef = cccPreRefineSort(1,1); - rPart = cccPreRefineSort(1,2); - rPhi = cccPreRefineSort(1,3); - rPhiInc = phiInc / 4; - rTheta= cccPreRefineSort(1,4); - rTheInc = thetaInc /2; - rPsi = cccPreRefineSort(1,5); - rPsiInc = psiInc /2; - % Confirm shiftVAL is doing what it should be - rXYZest = cccPreRefineSort(1,8:10); - - if (rTheInc) - % For a larger out of plane step, search a larger range in plane - psiRefineStep = floor(sqrt(rTheInc)); - else - psiRefineStep = 1; + for iRefine = 1:nSearch-1 + for alignLoop = 1:2 + if alignLoop == 1 + rXYZ = rXYZest; + elseif alignLoop == 2 + rXYZ = cccStorage3(iRefine,8:10); end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - thetaRefineStep =1; - phiRefineStep=2; - totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; - totalRefineStep = prod((2.*totalRefineStep)+1); - - cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); - - if (rPsiInc == 0) - inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; - else - inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; - end - polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; - azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; - - searchList = zeros(totalRefineStep,3); - nSearch = 1; - for iPhi = azimuthalRefine - for iTheta = polarRefine - for iPsi = inPlaneRefine - % best iPsi is origin Psi - Phi, no need to subtract here. - - searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; - - nSearch = nSearch + 1; - end - end - end % end of building angle list - - for iRefine = 1:nSearch-1 - for alignLoop = 1:2 - if alignLoop == 1 - rXYZ = rXYZest; - elseif alignLoop == 2 - rXYZ = cccStorage3(iRefine,8:10); - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - - RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - - - - switch alignLoop - % This keeps seperate shifts due to windowing and binning from - % shifts found in CCC - case 1 - - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - case 2 - - [ iTrimParticle ] = particleInterpolator.interp3d(... - RotMat,... - rXYZ,rotConvention ,... - 'inv',particle_symmetry); - - - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... - RotMat,... - [0,0,0],rotConvention ,... - 'inv',particle_symmetry); - - end - - - - if alignLoop == 1 - - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... + RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; + + switch alignLoop + % This keeps seperate shifts due to windowing and binning from + % shifts found in CCC + case 1 + % Estimate the peakshift by rotating the ref not the particle. + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + case 2 + [ iTrimParticle ] = particleInterpolator.interp3d(... + RotMat,... rXYZ,rotConvention ,... - 'forward','C1'); - - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); + 'inv',particle_symmetry); - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... + [ iWedgeMask ] = imgWdgInterpolator.interp3d(... + RotMat,... [0,0,0],rotConvention ,... - 'forward','C1'); - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{rRef},'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{rRef} ,'',padCalc,flgPrecision); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - - % 2016-11-11 also took out (+ rXYZ) - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - 1, 1, ... - peakCoord+rXYZ]; - else - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{rRef},volBinary_tmp,... - padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_tmp{half_set}{rRef},... - ifftshift(iWedgeMask),... - ref_WGT_tmp{half_set}{rRef}, ... - wCCC_tmp{iRef}); - - - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - iCCC, 1, ... - rXYZ] ; - end - end - + 'inv',particle_symmetry); end - - sortRef = sortrows(gather(cccStorage3),-6); - cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); - - end % end of refinement loop - - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % Get the final translational shift for the best scoring angular - % match. - try - if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) - bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); + + if ( alignLoop == 1 ) + + [ iRotRef ] = refInterpolator{iHalfSet}.interp3d(... + RotMat',... + rXYZ,rotConvention ,... + 'forward','C1'); + + + [ iRotWdg ] = refWdgInterpolator{iHalfSet}.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{1},'',... + [0,0,0;0,0,0],flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{1} ,'',padCalc,flgPrecision); + + size(rotPart_FT) + size(iRotWdg) + size(iRotMask) + size(iRotRef) + size(iMaxWedgeIfft) + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + % 2016-11-11 also took out (+ rXYZ) + cccStorage3(iRefine,:) = [rRef, rPart, ... + searchList(iRefine,:), ... + 1, 1, ... + peakCoord+rXYZ]; else - bestRotPeak = cccPreRefineSort(1,:); - bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); - end - catch - fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); - cccStorageRefine{iPeak}(iSubTomo,:) - cccPreRefineSort(1,:) - end - - finalRef = bestRotPeak(1,1); - finalPart = bestRotPeak(1,2); - finalPhi = bestRotPeak(1,3); - finalTheta= bestRotPeak(1,4); - finalPsi = bestRotPeak(1,5); - % Confirm shiftVAL is doing what it should be - finalrXYZest = bestRotPeak(1,8:10); - - RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - - - - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - - % use transpose of RotMat - %%% 2016-11-11 estPeakCoord should have been finalrXYZest in - %%% the last writing, but now switching to zeros - try - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - finalrXYZest,rotConvention ,... - 'forward','C1'); - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - catch - fprintf('\n\nFinal ref,part,phi,theta,psi %f %f %f %f %f\n\n',... - bestRotPeak(:,1:5)); - bestRotPeak(1,1:5) - fprintf('BreakPeak %d\n',breakPeak); - error('errrorsoedfsdf') - end - - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{finalRef} ,'',... - padCalc,flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{finalRef} ,'',padCalc,flgPrecision ); - - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - - % Subtract shiftVAL since this is due to windowing, not the actual - % position. - cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... - peakCoord + finalrXYZest - shiftVAL]) ; - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - % It is probably more useful see the shifts in the particle - % reference frame vs. the avg which was the original - if (emc.printShiftsInParticleBasis) - printShifts = zeros(3,3); - printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); - printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); - printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); - else - printShifts = [cccInitial(1,end-2:end); ... - cccPreRefineSort(1,end-2:end);... - cccStorageBest{iPeak}(iSubTomo,end-2:end)]; - end - - % Print out in Angstrom - printShifts = printShifts .* emc.pixel_size_angstroms; - - - deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); - if (emc.print_alignment_stats && deltaCCC < 0 && abs(deltaCCC) > 0.15*cccInitial(1,6)) - fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); - fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{1},volBinary_tmp,... + padCalc,flgPrecision); + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1{half_set}{rRef},... + ifftshift(iWedgeMask),... + refWGT{half_set}{rRef}, ... + wCCC_tmp{iRef}); + + + cccStorage3(iRefine,:) = [rRef, rPart, ... + searchList(iRefine,:), ... + iCCC, 1, ... + rXYZ] ; + end % if alignLoope == 1 + end % end of alignLoop - end + sortRef = sortrows(gather(cccStorage3),-6); + cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); - if (emc.track_stats) - - if thetaInc > 0 - cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); - else - cccStorageBest{iPeak}(iSubTomo,end-3) = 0; - end - - % % I'm not sold on what do do with this. The distribution over the - % % shift parameters doesn't really seem to make sense to me. There - % % are too many factors that can lead to large shifts (e.g. - % % tomoCPR) If we were searching the full angular space each - % % iteration, then this would make sense. - % mip_mean = mip.X./mip.N; - % mip_covar = mip.X2./mip.N - transpose(mip_mean)*(mip_mean); - % mip_covar_inv = mip_covar\eye(3); - % gauss_norm = ((2.*pi).^(3/2).*abs(mip_covar)).^-1; - % gauss_exp = exp(-0.5.*(printShifts(2,:)-mip_mean)*mip_covar_inv*transpose(printShifts(2,:)-mip_mean)); - - end + end % end of if flgRegine + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Get the final translational shift for the best scoring angular + % match. + try + if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) + bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); + else + bestRotPeak = cccPreRefineSort(1,:); + bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); + end + catch + fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); + cccStorageRefine{iPeak}(iSubTomo,:) + cccPreRefineSort(1,:) + end - cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); - cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); - if (emc.print_alignment_stats && flgRefine) - cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:), ... - 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... - cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... - 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - - else - if (emc.print_alignment_stats) - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - end - end + finalRef = bestRotPeak(1,1); + finalPart = bestRotPeak(1,2); + finalPhi = bestRotPeak(1,3); + finalTheta= bestRotPeak(1,4); + finalPsi = bestRotPeak(1,5); + % Confirm shiftVAL is doing what it should be + finalrXYZest = bestRotPeak(1,8:10); + + bandpassFilt_tmp{1} = gpuArray(bandpassFilt{finalRef}); + bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{finalRef}); + refInterpolator = {2,1}; + refWdgInterpolator= {2,1}; + for iHalfSet = 1:2 + refInterpolator{iHalfSet} = interpolator(ifftn(gpuArray(ref_FT1{iHalfSet}{finalRef})),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator{iHalfSet} = interpolator(gpuArray(refWgtROT{iHalfSet}{finalRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); + end + + RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; + + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + % use transpose of RotMat + %%% 2016-11-11 estPeakCoord should have been finalrXYZest in + %%% the last writing, but now switching to zeros + try + [ iRotRef ] = refInterpolator{iHalfSet}.interp3d(... + RotMat',... + finalrXYZest,rotConvention ,... + 'forward','C1'); + [ iRotWdg ] = refWdgInterpolator{iHalfSet}.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + catch + fprintf('\n\nFinal ref,part,phi,theta,psi %f %f %f %f %f\n\n',... + bestRotPeak(:,1:5)); + bestRotPeak(1,1:5) + fprintf('BreakPeak %d\n',breakPeak); + error('errrorsoedfsdf') + end + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{1} ,'',... + [0,0,0;0,0,0],flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{1} ,'',padCalc,flgPrecision ); + + + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + % Subtract shiftVAL since this is due to windowing, not the actual + % position. + cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... + peakCoord + finalrXYZest - shiftVAL]) ; + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + % It is probably more useful see the shifts in the particle + % reference frame vs. the avg which was the original + if (emc.printShiftsInParticleBasis) + printShifts = zeros(3,3); + printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); + printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); + printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); + else + printShifts = [cccInitial(1,end-2:end); ... + cccPreRefineSort(1,end-2:end);... + cccStorageBest{iPeak}(iSubTomo,end-2:end)]; + end + % Print out in Angstrom + printShifts = printShifts .* emc.pixel_size_angstroms; + + + deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); + if (emc.print_alignment_stats && deltaCCC < 0 && abs(deltaCCC) > 0.15*cccInitial(1,6)) + fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); + fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + + end + + if (emc.track_stats) + + if thetaInc > 0 + cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); + else + cccStorageBest{iPeak}(iSubTomo,end-3) = 0; + end + + % % I'm not sold on what do do with this. The distribution over the + % % shift parameters doesn't really seem to make sense to me. There + % % are too many factors that can lead to large shifts (e.g. + % % tomoCPR) If we were searching the full angular space each + % % iteration, then this would make sense. + % mip_mean = mip.X./mip.N; + % mip_covar = mip.X2./mip.N - transpose(mip_mean)*(mip_mean); + % mip_covar_inv = mip_covar\eye(3); + % gauss_norm = ((2.*pi).^(3/2).*abs(mip_covar)).^-1; + % gauss_exp = exp(-0.5.*(printShifts(2,:)-mip_mean)*mip_covar_inv*transpose(printShifts(2,:)-mip_mean)); + + end - end % if condition on newly ignored particles + cccInitial + cccStorageBest{iPeak}(iSubTomo,:) + cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); + cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); + if (emc.print_alignment_stats && flgRefine) + cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:), ... + 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... + cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... + 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + else + if (emc.print_alignment_stats) + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + end end + if ~(rem(iSubTomo,100)) timeClass = toc; @@ -1471,10 +1371,10 @@ rotPart_FT = []; rotParticle = []; end % end loop over possible peaks - - iMaxWedgeIfft = []; + end % loop over subTomos + iMaxWedgeIfft = []; for iPeak = 1:emc.nPeaks diff --git a/masking/BH_bandLimitCenterNormalize.m b/masking/BH_bandLimitCenterNormalize.m index aad7661d..fab524fd 100755 --- a/masking/BH_bandLimitCenterNormalize.m +++ b/masking/BH_bandLimitCenterNormalize.m @@ -45,7 +45,11 @@ IMG = BH_padZeros3d(IMG, PAD(1,:), PAD(2,:),'GPU', PRECISION,real(mean(IMG(:)))); IMG = fftn(IMG(MASK)); else - IMG = fftn(BH_padZeros3d(IMG, PAD(1,:), PAD(2,:),'GPU', PRECISION,real(mean(IMG(:))))); + if any(PAD(:)) + IMG = fftn(BH_padZeros3d(IMG, PAD(1,:), PAD(2,:),'GPU', PRECISION,real(mean(IMG(:))))); + else + IMG = fftn(IMG); + end end if (flgMeanWOMask) IMG(1) = 0; diff --git a/transformations/BH_multi_combineLowResInfo.m b/transformations/BH_multi_combineLowResInfo.m index fee84179..ea0b6dc3 100755 --- a/transformations/BH_multi_combineLowResInfo.m +++ b/transformations/BH_multi_combineLowResInfo.m @@ -2,8 +2,8 @@ %UNTITLED Summary of this function goes here % Detailed explanation goes here -refIDX = BH_multi_isCell( inputRefs{1} ) -nRefs = length(refIDX) +refIDX = BH_multi_isCell( inputRefs{1} ); +nRefs = length(refIDX); [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates([512,512,512],... 'Cartesian','cpu',{'none'},1,0,1); From 364d9b0f90c6bdf400a965b1bc32c822ed668059 Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 26 Mar 2024 09:33:54 -0400 Subject: [PATCH 076/151] multi-ref 2 seems to be working weith refactor --- alignment/BH_alignRaw3d_v2.m | 812 +++++++++++++++++------------------ 1 file changed, 402 insertions(+), 410 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 971547f0..5494823d 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -548,12 +548,12 @@ -% try -% EMC_parpool(nParProcesses+1) -% catch -% delete(gcp('nocreate')) -% EMC_parpool(nParProcesses+1) -% end +try + EMC_parpool(nParProcesses+1) +catch + delete(gcp('nocreate')) + EMC_parpool(nParProcesses+1) +end system('mkdir -p alignResume'); @@ -568,9 +568,9 @@ if (emc.force_no_symmetry) particle_symmetry = 'C1'; end -for iParProc = parVect +% for iParProc = parVect -% parfor iParProc = parVect +parfor iParProc = parVect symmetry = emc.symmetry; bestAngles_tmp = struct(); @@ -582,7 +582,6 @@ fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); for iTomo = iterList{iParProc} - % Check for interupted alignment. previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); if exist(previousAlignment,'file') @@ -593,7 +592,6 @@ bestAngles_tmp.(tomoList{iTomo}) = load(previousAlignment); fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); else - fprintf('In the else\n'); % There is some memory leak somewhere that I haven't been able to figure % out. I am clearing all vars but output in the children functions ... this % isn't ideal, but for now is an acceptable stop gap. @@ -627,8 +625,8 @@ % The check in xcf_rotational looks for a cell wCCC_tmp{iRef} = 0; end - end - end + end % iWccc + end % iRef % sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) tic; @@ -639,7 +637,9 @@ tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); - fprintf("line %d\n", 641); + includeList = positionList(:,26:26:26*emc.nPeaks ) ~= -9999; + includeList = any(includeList,2); + position_list = positionList(includeList,:); binShift = [0,0,0]; nSubTomos = size(positionList,1); @@ -679,62 +679,127 @@ cccStorageBest = cell(emc.nPeaks,1); cccStorageRefine = cell(emc.nPeaks,1); cccStorage2 = cell(emc.nPeaks,1); + cccInitial_arr = cell(emc.nPeaks,1); cccStorageTrans= zeros(1,10, 'single', 'gpuArray'); - cccInitial_arr = zeros(nSubTomos,10, 'single', 'gpuArray'); - cccInitial_arr(:,6) = -9999; for iPeak = 1:emc.nPeaks cccStorageBest{iPeak} = zeros(nSubTomos,10, 'single'); cccStorageRefine{iPeak}= zeros(nSubTomos,10, 'single'); cccStorage2{iPeak} = zeros(nSubTomos,10, 'single', 'gpuArray'); + cccInitial_arr{iPeak} = zeros(nSubTomos,10, 'single', 'gpuArray'); + cccInitial_arr{iPeak}(:,6) = -9999; + cccStorageRefine{iPeak}(:,6) = -9999; + end % reset for each tomogram wdgIDX = 0; - fprintf('nRefs %d\n',nReferences(1)); - for iRefIdx = 1:nReferences(1); - fprintf('iRefIdx %d\n',iRefIdx); - % Just use C1 to initialize, whether or not this is the final - refInterpolator = {2,1}; - refWdgInterpolator= {2,1}; - for iHalfSet = 1:2 - ref_FT1{iHalfSet}{iRefIdx} = gpuArray(ref_FT1{iHalfSet}{iRefIdx}); - refWgtROT{iHalfSet}{iRefIdx} = gpuArray(refWgtROT{iHalfSet}{iRefIdx}); - refInterpolator{iHalfSet} = interpolator(ifftn(ref_FT1{iHalfSet}{iRefIdx}),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator{iHalfSet} = interpolator(refWgtROT{iHalfSet}{iRefIdx},[0,0,0],[0,0,0],'Bah','forward','C1',false); - end - bandpassFilt_tmp{1} = gpuArray(bandpassFilt{iRefIdx}); - bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{iRefIdx}); - - for iSubTomo = 1:nSubTomos - fprintf('iSubTomo %d\n',iSubTomo); - make_SF3D = true; - breakPeak = 0; % for try catch on cut out vols + for iSubTomo = 1:nSubTomos + make_SF3D = true; + breakPeak = 0; % for try catch on cut out vols + + + for iPeak = 1:emc.nPeaks + if (emc.track_stats) + measure_noise = true; + mip.('x') = {}; + mip.('x2') = {}; + mip.('N') = 0; + end + if (breakPeak) + continue; + end + getInitialCCC = 1; + + % Check that the given subTomo is not to be ignored + classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); + particleIDX = positionList(iSubTomo, 4); + half_set = positionList(iSubTomo, 7); + + if (classIDX == -9999) + continue; + end + + center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; + angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); - for iPeak = 1:emc.nPeaks + % Find range to extract, and check for domain error. + if (emc.flgCutOutVolumes) + % Need some check that the windowsize has not changed! TODO TODO - if (emc.track_stats) - measure_noise = true; - mip.('x') = {}; - mip.('x2') = {}; - mip.('N') = 0; - end - if (breakPeak) - continue; + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid(2*CUTPADDING+sizeWindow, ... + sizeWindow,maskRadius, center); + else + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... + sizeWindow,maskRadius, center); + end + + if ischar(indVAL) + fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) + nIgnored = nIgnored + 1; + geometry_tmp.(tomoList{iTomo})(geometry_tmp.(tomoList{iTomo})(:,4) == particleIDX, 26) = -9999; + continue; + else + if (emc.flgCutOutVolumes) + % Test with some generic padding , only to be used on bin 1 at + % first!!! TODO add a flag to check this. + try + particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); + iparticle = gpuArray(OPEN_IMG('single',particleOUT_name,[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + catch + fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); + geometry_tmp.(tomoList{iTomo})(geometry_tmp.(tomoList{iTomo})(:,4) == particleIDX, 26) = -9999; + breakPeak = 1; + continue; + end + else + + iparticle = gpuArray(OPEN_IMG('single', volumeData, [indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); end - getInitialCCC = 1; + [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... + padVAL(2,1:3), 'GPU', 'singleTaper'); - % Check that the given subTomo is not to be ignored - classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); - particleIDX = positionList(iSubTomo, 4); - half_set = positionList(iSubTomo, 7); - - if (classIDX == -9999) - continue; + if (make_SF3D) + make_SF3D = false; + % For now excluding the soften weight. + [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center, reconCoords, emc.wiener_constant); + imgWdgInterpolator = ''; + % The unshifted mask is kept in texture mem until no longer + % needed + [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); + + particleInterpolator= ''; + + particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); end + end % Else clause on windowing (if is a continue) + + for iRefIdx = 1:nReferences(1); + % Just use C1 to initialize, whether or not this is the final + refInterpolator = ''; + refWdgInterpolator= ''; + ref_FT1_thread_local = gpuArray(ref_FT1{half_set}{iRefIdx}); + refWgtROT_thread_local = gpuArray(refWgtROT{half_set}{iRefIdx}); + refInterpolator = interpolator(ifftn(ref_FT1_thread_local),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator = interpolator(refWgtROT_thread_local,[0,0,0],[0,0,0],'Bah','forward','C1',false); + bandpassFilt_tmp{1} = gpuArray(bandpassFilt{iRefIdx}); + bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{iRefIdx}); + % In the case we are only aligning refs to those classes they came from, we % need to check to see if we duck out here. - if( emc.multi_reference_alignment == 2 ) + switch emc.multi_reference_alignment + case 0 + refToAlign = 1; + case 1 + refToAlign = classVector{half_set}(1,iRefIdx); + case 2 refToAlign = find(classIDX == classVector{half_set}(1,:)); if isempty(refToAlign) fprintf('WARNING: wanted class %d not found in classVector\n',classIDX); @@ -749,294 +814,232 @@ end - center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; - angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); - - % Find range to extract, and check for domain error. - if (emc.flgCutOutVolumes) - % Need some check that the windowsize has not changed! TODO TODO - - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(2*CUTPADDING+sizeWindow, ... - sizeWindow,maskRadius, center); + if (emc.use_new_grid_search) + theta_search = 1:gridSearch.number_of_out_of_plane_angles; else - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... - sizeWindow,maskRadius, center); + theta_search = 1:size(angleStep,1); end - if ischar(indVAL) - fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) - nIgnored = nIgnored + 1; - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - else - if (emc.flgCutOutVolumes) - % Test with some generic padding , only to be used on bin 1 at - % first!!! TODO add a flag to check this. - try - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iparticle = gpuArray(OPEN_IMG('single',particleOUT_name,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - catch - fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); - geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; - breakPeak = 1; - continue; + for iAngle = theta_search + + if (emc.use_new_grid_search) + theta = gridSearch.parameter_map.theta(iAngle); + if length(gridSearch.parameter_map.phi{iAngle}) > 1 + phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); + else + phiInc = 0; end + thetaInc = gridSearch.theta_step; + numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); else - - iparticle = gpuArray(OPEN_IMG('single', volumeData, [indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); + theta = angleStep(iAngle,1); + phiInc = angleStep(iAngle,3); + thetaInc = angleStep(iAngle,4); + numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; end - [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... - padVAL(2,1:3), 'GPU', 'singleTaper'); - if (make_SF3D) - make_SF3D = false; - % For now excluding the soften weight. - [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center, reconCoords, emc.wiener_constant); - imgWdgInterpolator = ''; - % The unshifted mask is kept in texture mem until no longer - % needed - [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); - - particleInterpolator= ''; - - particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); - end + % To prevent only searching the same increments each time in a limited + % grid search, radomly offset the azimuthal angle by a random number + % between 0 and 1/2 the azimuthal increment. + + azimuthalRandomizer = (rand(1)-0.5)*phiInc; + + % Calculate the increment in phi so that the azimuthal sampling is + % consistent and equal to the out of plane increment. if (emc.use_new_grid_search) - theta_search = 1:gridSearch.number_of_out_of_plane_angles; + % FIXME randomizer passed as bool to eulerSearch + phi_search = gridSearch.parameter_map.phi{iAngle}; else - theta_search = 1:size(angleStep,1); + phi_search = 0:angleStep(iAngle,2); end - for iAngle = theta_search - - if (emc.use_new_grid_search) - theta = gridSearch.parameter_map.theta(iAngle); - if length(gridSearch.parameter_map.phi{iAngle}) > 1 - phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); - else - phiInc = 0; - end - thetaInc = gridSearch.theta_step; - numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); - else - theta = angleStep(iAngle,1); - phiInc = angleStep(iAngle,3); - thetaInc = angleStep(iAngle,4); - numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - end - - % To prevent only searching the same increments each time in a limited - % grid search, radomly offset the azimuthal angle by a random number - % between 0 and 1/2 the azimuthal increment. - - azimuthalRandomizer = (rand(1)-0.5)*phiInc; - - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. + for iAzimuth = phi_search if (emc.use_new_grid_search) - % FIXME randomizer passed as bool to eulerSearch - phi_search = gridSearch.parameter_map.phi{iAngle}; + phi = rem(iAzimuth + azimuthalRandomizer,360); + psiInc = gridSearch.psi_step; else - phi_search = 0:angleStep(iAngle,2); + phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); + psiInc = angleStep(iAngle,5); end - for iAzimuth = phi_search + for iInPlane = inPlaneSearch + psi = iInPlane; - if (emc.use_new_grid_search) - phi = rem(iAzimuth + azimuthalRandomizer,360); - psiInc = gridSearch.psi_step; - else - phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); - psiInc = angleStep(iAngle,5); - end - - for iInPlane = inPlaneSearch - psi = iInPlane; - - RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - for alignLoop = 1:2 - switch alignLoop - case 1 - % This takes care of non-inter shift in the origin that is - % ignored during the windowing of the particle. - estPeakCoord = shiftVAL; - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - case 2 - - estPeakCoord = gather(cccStorageTrans(1,8:10)); + RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; + + for alignLoop = 1:2 + switch alignLoop + case 1 + % This takes care of non-inter shift in the origin that is + % ignored during the windowing of the particle. + estPeakCoord = shiftVAL; + % Estimate the peakshift by rotating the ref not the particle. + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + case 2 - [ iTrimParticle ] = particleInterpolator.interp3d(... - RotMat,... - estPeakCoord,rotConvention ,... + estPeakCoord = gather(cccStorageTrans(1,8:10)); + + [ iTrimParticle ] = particleInterpolator.interp3d(... + RotMat,... + estPeakCoord,rotConvention ,... + 'inv',particle_symmetry); + + if (getInitialCCC) + [ iTrimInitial ] = particleInterpolator.interp3d(... + reshape(angles,3,3),... + shiftVAL,rotConvention ,... 'inv',particle_symmetry); - if (getInitialCCC) - [ iTrimInitial ] = particleInterpolator.interp3d(... - reshape(angles,3,3),... - shiftVAL,rotConvention ,... - 'inv',particle_symmetry); - - [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... - reshape(angles,3,3),... - [0,0,0],rotConvention ,... - 'inv',particle_symmetry); - end - - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... - RotMat,... + [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... + reshape(angles,3,3),... [0,0,0],rotConvention ,... 'inv',particle_symmetry); - end % switch on align loop - - switch alignLoop - case 1 - % use transpose of RotMat - - [ iRotRef ] = refInterpolator{iHalfSet}.interp3d(... - RotMat',... - estPeakCoord,rotConvention ,... - 'forward','C1'); - - [ iRotWdg ] = refWdgInterpolator{iHalfSet}.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - % maybe I should be rotating peak mask here in case it has - % an odd shape, since we are leaving the proper frame - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{1} ,'',... - [0,0,0;0,0,0],flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{1} ,'',padCalc,flgPrecision); - - if (emc.track_stats && measure_noise) - - [ ~, mip ] = BH_multi_xcf_Translational_2( ... - rotPart_FT, ... - conj(iRotRef),... - ifftshift(iRotWdg),... - iMaxWedgeIfft,... - iRotMask, peakCOM,... - mip); - - end - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); + end + + [ iWedgeMask ] = imgWdgInterpolator.interp3d(... + RotMat,... + [0,0,0],rotConvention ,... + 'inv',particle_symmetry); + end % switch on align loop + + switch alignLoop + case 1 + % use transpose of RotMat + + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + estPeakCoord,rotConvention ,... + 'forward','C1'); + + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + % maybe I should be rotating peak mask here in case it has + % an odd shape, since we are leaving the proper frame + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{1} ,'',... + [0,0,0;0,0,0],flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{1} ,'',padCalc,flgPrecision); + + if (emc.track_stats && measure_noise) - cccStorageTrans(1,:) = [iRefIdx, particleIDX, ... - phi, theta, psi - phi, ... - 0, 0, ... - peakCoord + estPeakCoord]; - case 2 - % get starting point - if (getInitialCCC) - initialRotPart_FT = BH_bandLimitCenterNormalize(... - iTrimInitial.*volMask_tmp,... - bandpassFilt_tmp{1} ,volBinary_tmp,padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( initialRotPart_FT, ... - ref_FT1{half_set}{iRefIdx}, ... - ifftshift(iWedgeInitial),... - refWGT{half_set}{iRefIdx}, ... - wCCC_tmp{iRefIdx}); - - if ( iCCC > cccInitial_arr(iSubTomo,6)) - cccInitial_arr(iSubTomo,:) = [iRefIdx, particleIDX, ... - 0,0,0, ... - iCCC, 1, ... - shiftVAL]; - initialRotPart_FT = []; - end - end + [ ~, mip ] = BH_multi_xcf_Translational_2( ... + rotPart_FT, ... + conj(iRotRef),... + ifftshift(iRotWdg),... + iMaxWedgeIfft,... + iRotMask, peakCOM,... + mip); - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... + end + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + cccStorageTrans(1,:) = [iRefIdx, particleIDX, ... + phi, theta, psi - phi, ... + 0, 0, ... + peakCoord + estPeakCoord]; + case 2 + % get starting point + if (getInitialCCC) + initialRotPart_FT = BH_bandLimitCenterNormalize(... + iTrimInitial.*volMask_tmp,... bandpassFilt_tmp{1} ,volBinary_tmp,padCalc,flgPrecision); [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1{half_set}{iRefIdx},... - ifftshift(iWedgeMask),... + BH_multi_xcf_Rotational( initialRotPart_FT, ... + ref_FT1_thread_local, ... + ifftshift(iWedgeInitial),... refWGT{half_set}{iRefIdx}, ... wCCC_tmp{iRefIdx}); - % Note that no new translational estimate is made, so no - % need to multiply by RotMat - if (iCCC > cccStorage2{iPeak}(iSubTomo,6) ) - - cccStorage2{iPeak}(iSubTomo,:) = ... - [iRefIdx, particleIDX, ... - phi, theta, psi , ... - iCCC, 1, ... - estPeakCoord]; - end - end %case - end % switch - % This volume won't be needed until the next subTomo is considered, - % which is also where getInitialCCC Boolean is set to True again. - iTrimInitial = []; - getInitialCCC = 0; % FIXME is this still right with the loop inversion - end % alignLoop (trans then rotation) - end % in plane angles (psi) - end % azimuth (phi) - end % polar (theta) - end % peak loop - end % subTomo loop - % Gather the ref back to the CPU as we'll need in in the subsequent alignment blocks - for iHalfSet = 1:2 - ref_FT1{iHalfSet}{iRefIdx} = gather(ref_FT1{iHalfSet}{iRefIdx}); - refWgtROT{iHalfSet}{iRefIdx} = gather(refWgtROT{iHalfSet}{iRefIdx}); - end - end % loop over references + if ( iCCC > cccInitial_arr{iPeak}(iSubTomo,6)) + cccInitial_arr{iPeak}(iSubTomo,:) = [iRefIdx, particleIDX, ... + 0,0,0, ... + iCCC, 1, ... + shiftVAL]; + initialRotPart_FT = []; + end + end + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{1} ,volBinary_tmp,padCalc,flgPrecision); + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1_thread_local,... + ifftshift(iWedgeMask),... + refWGT{half_set}{iRefIdx}, ... + wCCC_tmp{iRefIdx}); + + % Note that no new translational estimate is made, so no + % need to multiply by RotMat + if (iCCC > cccStorage2{iPeak}(iSubTomo,6) ) + + cccStorage2{iPeak}(iSubTomo,:) = ... + [iRefIdx, particleIDX, ... + phi, theta, psi , ... + iCCC, 1, ... + estPeakCoord]; + end + + end % switch + % This volume won't be needed until the next subTomo is considered, + % which is also where getInitialCCC Boolean is set to True again. + iTrimInitial = []; + end % alignLoop (trans then rotation) + getInitialCCC = 0; % needs to be calculated on the first pass through on the second iter of align loop + end % in plane angles (psi) + end % azimuth (phi) + end % polar (theta) + end % loop over references - % Now we'll loop over the subtomos and peaks just grabbing the - nSubTomos - for iSubTomo = 1:nSubTomos - cccInitial = gather( cccInitial_arr(iSubTomo,:) ); - for iPeak = 1:emc.nPeaks - cccPreRefineSort = gather(cccStorage2{iPeak}(iSubTomo,:)); - if cccInitial(1,6 ) > cccPreRefineSort(1,6) - cccPreRefineSort(1,:) = cccInitial(1,:); - end + cccInitial = gather( cccInitial_arr{iPeak}(iSubTomo,:) ); + if (cccInitial(1,6) == -9999) + continue; + end + cccPreRefineSort = gather(cccStorage2{iPeak}(iSubTomo,:)); + % This only seems to be a problem with cut out volumes. % Normalization maybe? if ~any(cccPreRefineSort(1,:)) cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); - continue + continue; end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - + %% + + if cccInitial(1,6) > cccPreRefineSort(1,6) + cccPreRefineSort(1,:) = cccInitial(1,:); + end + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + if (flgRefine) % Get the results from just this subTomo and sort on CCC + rRef = cccPreRefineSort(1,1); rPart = cccPreRefineSort(1,2); rPhi = cccPreRefineSort(1,3); @@ -1048,14 +1051,15 @@ % Confirm shiftVAL is doing what it should be rXYZest = cccPreRefineSort(1,8:10); + % Host to Device for best reference bandpassFilt_tmp{1} = gpuArray(bandpassFilt{rRef}); bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{rRef}); - refInterpolator = {2,1}; - refWdgInterpolator= {2,1}; - for iHalfSet = 1:2 - refInterpolator{iHalfSet} = interpolator(ifftn(gpuArray(ref_FT1{iHalfSet}{rRef})),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator{iHalfSet} = interpolator(gpuArray(refWgtROT{iHalfSet}{rRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); - end + refInterpolator = ''; + refWdgInterpolator= ''; + ref_FT1_thread_local = gpuArray(ref_FT1{half_set}{rRef}); + refWgtROT_thread_local = gpuArray(refWgtROT{half_set}{rRef}); + refInterpolator = interpolator(ifftn(ref_FT1_thread_local),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator = interpolator(gpuArray(refWgtROT_thread_local),[0,0,0],[0,0,0],'Bah','forward','C1',false); if (rTheInc) % For a larger out of plane step, search a larger range in plane @@ -1092,16 +1096,9 @@ end end end % end of building angle list - for iRefine = 1:nSearch-1 for alignLoop = 1:2 - if alignLoop == 1 - rXYZ = rXYZest; - elseif alignLoop == 2 - rXYZ = cccStorage3(iRefine,8:10); - end - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 - + RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); RotMat = reshape(angles,3,3) * RotMat; @@ -1109,91 +1106,89 @@ % This keeps seperate shifts due to windowing and binning from % shifts found in CCC case 1 + rXYZ = rXYZest; % Estimate the peakshift by rotating the ref not the particle. iTrimParticle = ... iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... padWindow(1,2) + 1:end - padWindow(2,2) , ... padWindow(1,3) + 1:end - padWindow(2,3) ); - case 2 - [ iTrimParticle ] = particleInterpolator.interp3d(... - RotMat,... + + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... rXYZ,rotConvention ,... - 'inv',particle_symmetry); + 'forward','C1'); - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... - RotMat,... + + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... [0,0,0],rotConvention ,... - 'inv',particle_symmetry); - end - - if ( alignLoop == 1 ) - - [ iRotRef ] = refInterpolator{iHalfSet}.interp3d(... - RotMat',... - rXYZ,rotConvention ,... - 'forward','C1'); - - - [ iRotWdg ] = refWdgInterpolator{iHalfSet}.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{1},'',... - [0,0,0;0,0,0],flgPrecision); + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{1},'',... + [0,0,0;0,0,0],flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{1} ,'',padCalc,flgPrecision); + + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + % 2016-11-11 also took out (+ rXYZ) + cccStorage3(iRefine,:) = [rRef, rPart, ... + searchList(iRefine,:), ... + -9999, 1, ... + peakCoord+rXYZ]; + case 2 + rXYZ = cccStorage3(iRefine,8:10); - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{1} ,'',padCalc,flgPrecision); + [ iTrimParticle ] = particleInterpolator.interp3d(... + RotMat,... + rXYZ,rotConvention ,... + 'inv',particle_symmetry); - size(rotPart_FT) - size(iRotWdg) - size(iRotMask) - size(iRotRef) - size(iMaxWedgeIfft) - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); + [ iWedgeMask ] = imgWdgInterpolator.interp3d(... + RotMat,... + [0,0,0],rotConvention ,... + 'inv',particle_symmetry); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{1},volBinary_tmp,... + padCalc,flgPrecision); - % 2016-11-11 also took out (+ rXYZ) - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - 1, 1, ... - peakCoord+rXYZ]; - else - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{1},volBinary_tmp,... - padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1{half_set}{rRef},... - ifftshift(iWedgeMask),... - refWGT{half_set}{rRef}, ... - wCCC_tmp{iRef}); - - - cccStorage3(iRefine,:) = [rRef, rPart, ... - searchList(iRefine,:), ... - iCCC, 1, ... - rXYZ] ; - end % if alignLoope == 1 + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1_thread_local,... + ifftshift(iWedgeMask),... + refWGT{half_set}{rRef}, ... + wCCC_tmp{rRef}); + + + cccStorage3(iRefine,:) = [rRef, rPart, ... + searchList(iRefine,:), ... + iCCC, 1, ... + rXYZ] ; + end % switch align loop + end % end of alignLoop - - sortRef = sortrows(gather(cccStorage3),-6); + end % end of iRefine loop + sortRef = sortrows(gather(cccStorage3),-6); + + if (sortRef(1,6) > cccStorageRefine{iPeak}(iSubTomo,6)) cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); - + end + end % end of if flgRegine - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Get the final translational shift for the best scoring angular % match. @@ -1209,7 +1204,6 @@ cccStorageRefine{iPeak}(iSubTomo,:) cccPreRefineSort(1,:) end - finalRef = bestRotPeak(1,1); finalPart = bestRotPeak(1,2); finalPhi = bestRotPeak(1,3); @@ -1218,17 +1212,22 @@ % Confirm shiftVAL is doing what it should be finalrXYZest = bestRotPeak(1,8:10); - bandpassFilt_tmp{1} = gpuArray(bandpassFilt{finalRef}); - bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{finalRef}); - refInterpolator = {2,1}; - refWdgInterpolator= {2,1}; - for iHalfSet = 1:2 - refInterpolator{iHalfSet} = interpolator(ifftn(gpuArray(ref_FT1{iHalfSet}{finalRef})),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator{iHalfSet} = interpolator(gpuArray(refWgtROT{iHalfSet}{finalRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); - end RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); RotMat = reshape(angles,3,3) * RotMat; + + if (~flgRefine) + % We can use the same reference and interpolators if we've reloaded them for the refine step, otherwise + % we need to transfer them host to device here to get the final shifts. + bandpassFilt_tmp{1} = gpuArray(bandpassFilt{finalRef}); + bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{finalRef}); + refInterpolator = ''; + refWdgInterpolator= ''; + ref_FT1_thread_local = gpuArray(ref_FT1{half_set}{finalRef}); + refWgtROT_thread_local = gpuArray(refWgtROT{half_set}{finalRef}); + refInterpolator = interpolator(ifftn(ref_FT1_thread_local),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator = interpolator(gpuArray(refWgtROT_thread_local),[0,0,0],[0,0,0],'Bah','forward','C1',false); + end iTrimParticle = ... iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... @@ -1239,11 +1238,11 @@ %%% 2016-11-11 estPeakCoord should have been finalrXYZest in %%% the last writing, but now switching to zeros try - [ iRotRef ] = refInterpolator{iHalfSet}.interp3d(... + [ iRotRef ] = refInterpolator.interp3d(... RotMat',... finalrXYZest,rotConvention ,... 'forward','C1'); - [ iRotWdg ] = refWdgInterpolator{iHalfSet}.interp3d(... + [ iRotWdg ] = refWdgInterpolator.interp3d(... RotMat',... [0,0,0],rotConvention ,... 'forward','C1'); @@ -1271,15 +1270,14 @@ [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); % Subtract shiftVAL since this is due to windowing, not the actual % position. - cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... - peakCoord + finalrXYZest - shiftVAL]) ; + cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), peakCoord + finalrXYZest - shiftVAL]) ; %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % It is probably more useful see the shifts in the particle @@ -1290,9 +1288,9 @@ printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); else - printShifts = [cccInitial(1,end-2:end); ... - cccPreRefineSort(1,end-2:end);... - cccStorageBest{iPeak}(iSubTomo,end-2:end)]; + printShifts = [cccInitial(1,end-2:end); ... + cccPreRefineSort(1,end-2:end);... + cccStorageBest{iPeak}(iSubTomo,end-2:end)]; end % Print out in Angstrom @@ -1331,8 +1329,6 @@ end - cccInitial - cccStorageBest{iPeak}(iSubTomo,:) cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); if (emc.print_alignment_stats && flgRefine) @@ -1371,9 +1367,7 @@ rotPart_FT = []; rotParticle = []; end % end loop over possible peaks - end % loop over subTomos - iMaxWedgeIfft = []; for iPeak = 1:emc.nPeaks @@ -1389,7 +1383,6 @@ sortCCC(:,1+10*(iPeak-1):10+10*(iPeak-1)) = cccStorageBest{iPeak}; end - % % % % I think this is redundant now, but leaving until I double check. % % % save('sortCCC.mat','sortCCC'); [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); @@ -1409,8 +1402,7 @@ fprintf(angOut,'\n'); end fclose(angOut); - - end % if clause to check for previous alignment + end % if/ -> else clause to check for previous alignment, return to loop on references end % loop over tomos bestAnglesResults{iParProc} = bestAngles_tmp; geometryResults{iParProc} = geometry_tmp; From 45317dd8ef06a8fa76a27112742f641bbed67c68 Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 26 Mar 2024 15:30:15 -0400 Subject: [PATCH 077/151] Fix HP output when using colormap --- synthetic/BH_synthetic_mapBack.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 24dcb541..01ff6285 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -767,7 +767,7 @@ clear avgTomo if (emc.save_mapback_classes || flgClassAvg) - SAVE_IMG(avgColor,{sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),'half'},pixel_size); + SAVE_IMG(single(avgColor),{sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),'half'},pixel_size); clear avgColor end From b68e74cb2e3bdcef39232a41c16f6fc8600da68f Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 17 Jun 2024 12:08:04 -0400 Subject: [PATCH 078/151] With broken refactored alignRaw_v2, weird z-shifts, reverting that file to 44fea6575cf221d02897799b96e6eb46374a123c --- alignment/BH_alignRaw3d_v2.m | 11 +-- alignment/BH_templateSearch3d_2.m | 27 +++--- metaData/BH_parseParameterFile.m | 14 +++ synthetic/BH_synthetic_mapBack.m | 93 ++++++++++++++++---- testScripts/mCompile.sh | 2 +- transformations/BH_multi_combineLowResInfo.m | 63 ++++++++----- 6 files changed, 149 insertions(+), 61 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 5494823d..fc7e41b9 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -291,8 +291,6 @@ [ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, emc.pixel_size_angstroms, maxGoldStandard ); - - % optimize the fft for the given size. Padding to the next power of 2 is usually % slower given the dimensionalityl of the volume data. fftPlanner = rand(sizeCalc); @@ -301,10 +299,6 @@ clear fftPlanner - - - - stat_mask = []; if (eraseMask) peakMask = EMC_maskShape(eraseMaskType,sizeCalc,floor(eraseMaskRadius),'cpu',{'kernel',false}); @@ -684,11 +678,14 @@ for iPeak = 1:emc.nPeaks cccStorageBest{iPeak} = zeros(nSubTomos,10, 'single'); + cccStorageBest{iPeak}(:,6) = -9999; cccStorageRefine{iPeak}= zeros(nSubTomos,10, 'single'); + cccStorageRefine{iPeak}(:,6) = -9999; cccStorage2{iPeak} = zeros(nSubTomos,10, 'single', 'gpuArray'); + cccStorage2{iPeak}(:,6) = -9999; cccInitial_arr{iPeak} = zeros(nSubTomos,10, 'single', 'gpuArray'); cccInitial_arr{iPeak}(:,6) = -9999; - cccStorageRefine{iPeak}(:,6) = -9999; + end % reset for each tomogram diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index ca3c0303..8f3d0b9c 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -12,12 +12,20 @@ test_half = false; ctf3dNoSubTomoMeta = true; -if length(varargin) == 1 +if length(varargin) > 0 % Allow for an override of the max number, useful when only a few tomos % have a strong feature like carbon that is hard to avoid. gpuIDX = EMC_str2double(varargin{1}); -elseif length(varargin) > 1 - error('emClarity templateSearch paramN.m tiltN regionN referenceName symmetry(C1) '); +else + gpuIDX = 1; +end +if length(varargin) == 2 + mapBackIter = EMC_str2double(varargin{2}); +else + mapBackIter = 0; +end +if length(varargin) > 2 + error('emClarity templateSearch paramN.m tiltN regionN referenceName symmetry(C1) '); end tomoIdx = EMC_str2double(tomoIdx); @@ -40,18 +48,7 @@ % Currently hardcoded to always expect a tomogram constructed with ctf correction % using emClarity ctf3d paramN.m templateSearch use_ctf3d_templateSearch=true; -if ctf3dNoSubTomoMeta - mapBackIter = 0; -else - try - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - mapBackIter = subTomoMeta.currentTomoCPR - catch - % TODO: is there a better check on whether we are using ctf3d templateSearch vs ctf 3d? - mapBackIter = 0; - shouldBeCTF = false; - end -end + samplingRate = emc.('Tmp_samplingRate'); try diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 3fcd1c32..801ea26b 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -384,6 +384,11 @@ emc = EMC_assert_deprecated_substitution(emc, false, 'save_mapback_classes', 'flgColorMap'); EMC_assert_boolean(emc.save_mapback_classes); +if isfield(emc, 'only_use_reference_classes') + EMC_assert_boolean(emc.only_use_reference_classes); +else + emc.only_use_reference_classes = false; +end % These seemed to be necessary at some point to translate between IMOD and emClarity % coordinate systems, but the should probably be looked at again. TODO: @@ -598,6 +603,15 @@ emc.n_tilt_workers = 4; end +% Number of tiltalign processes to run in parallel in tomoCPR +% For now, default to zero and manually re-run while sorting out the +% optimization process +if isfield(emc, 'run_tomocpr_alignments') + EMC_assert_numeric(emc.run_tomocpr_alignments, 1); +else + emc.run_tomocpr_alignments = 0; +end + if isfield(emc, 'max_ctf3dDepth') EMC_assert_numeric(emc.max_ctf3dDepth, 1, [1 * 10^-9, 1000 * 10^-9]); else diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 01ff6285..22ea6a21 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -61,6 +61,7 @@ cycleNumber = sprintf('cycle%0.3u', CYCLE); + samplingRate = emc.('Ali_samplingRate'); % used to determine the number of fiducials/patch for local area. MOL_MASS = emc.('particleMass'); @@ -598,6 +599,7 @@ iRefIDX = 1; iClassIDX = positionList(iSubTomo,26); + use_this_class = true if (nRefs > 1) % Assuming generally there are fewer classes seleceted as references than there are total classes % For those that aren't one of the select ones, we could try to track the best matched reference from the most recent @@ -605,6 +607,11 @@ % FIXME: having a class occupancy factor would be better than just picking a random one. if ~(ismember(iClassIDX,classVector{1}) || ismember(iClassIDX,classVector{2})) + if (emc.only_use_reference_classes) + use_this_class = false; + end + % If the class is not in the list, pick a random one. This will be ignored if only_use_reference_classes is set. + % However, we want to assign it so the iRefIDX line below is okay. use_class = datasample(classVector{1},1); else use_class = iClassIDX; @@ -626,8 +633,12 @@ [ indVAL, padVAL, shiftVAL ] = BH_isWindowValid(reconstruction_size, sizeAvgVol, sizeAvgVol./5, subtomo_origin_in_sample); - if ischar(indVAL) - fprintf('ignoring subTomo %d for out of bounds conditions.\n', iSubTomo); + if ischar(indVAL) || ~use_this_class + if ischar(indVal) + fprintf('ignoring subTomo %d for out of bounds conditions.\n', iSubTomo); + else + fprintf('ignoring subTomo %d for class %d because it is not one of the references.\n', iSubTomo, iClassIDX); + end else if positionList(iSubTomo,7) == 1 iAvgResamp = BH_resample3d(refVol{1}{iRefIDX},subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'); @@ -1705,7 +1716,10 @@ tilt_script_name = sprintf('cache/mapBack%d/%s%s',mbOUT{2},tn2,tn3); end + % We'll make two copies, one that can be re-run from the local project directory + % and one that is run while mapBack is running. aliCom_name = sprintf('%smapBack%d/%s.align',mbOutAlt{1:3}); + aliCom_name_rerun = sprintf('cache/mapBack%d/%s.align',mbOutAlt{2:3}); aliCom = fopen(aliCom_name,'w'); % Testing local alignment with optimized parameters using the new imod options for leave out @@ -1742,7 +1756,46 @@ emc.min_overlap,... mbOutAlt{1:3},... mbOutAlt{1:3}); + fclose(aliCom); + aliCom_rerun = fopen(aliCom_name_rerun,'w'); + +% Testing local alignment with optimized parameters using the new imod options for leave out +fprintf(aliCom_rerun,[... + 'mapBack%d/%s_fit-full.fid\n',... %1 + 'mapBack%d/%s%s.3dmod\n',... %2 + 'mapBack%d/%s%s.resid\n',... %3 + 'mapBack%d/%s%s.xyz\n',... %4 + 'mapBack%d/%s%s.tlt\n',... %5 + 'mapBack%d/%s%s.xtilt\n',... %6 + 'mapBack%d/%s%s.tltxf\n',... %7 + '%s\n',... %8 input tilt file + '%3.3f\n',... KFactorScaling %9 + 'mapBack%d/%s%s.local\n',... OutputLocalFile %10 + '%d\n%d\n', ...TargetPatchSizeXandY %11 12 + '%d\n%d\n',... MinFidsTotalAndEachSurface %13 14 + '%f\n%f\n',... MinSizeOrOverlapXandY %15 16 + 'mapBack%d/%s.align_ta.log\n',... output log file %17 + 'mapBack%d/%s.align_ta_optimizer.log\n'],... output log file for optimizer %18 + mbOutAlt{2:3},... % for ModelFile + mbOutAlt{2:3},outCTF,... %2 + mbOutAlt{2:3},outCTF,... %3 + mbOutAlt{2:3},outCTF,... %4 + mbOutAlt{2:3},outCTF,... %5 + mbOutAlt{2:3},outCTF,... %6 + mbOutAlt{2:3},outCTF, ... %7 + tilt_script_name,... + emc.k_factor_scaling, ... + mbOutAlt{2:3},outCTF, ... + targetPatchSize(1), ... + targetPatchSize(2),... + nFiducialsPerPatch, ... + floor(nFiducialsPerPatch/3),... + emc.min_overlap, ... + emc.min_overlap,... + mbOutAlt{2:3},... + mbOutAlt{2:3}); + fclose(aliCom_rerun); %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % % TODO: It looks like the output model file (3dmod) is the solved positions, % % but is saved at a pixel size of 1. Scaling by the sampling rate in all dimensions @@ -1831,7 +1884,7 @@ % % fprintf(aliCom,'\nawk ''{if(NR >3) print $5}'' ./mapBack%d/tmp.log > mapBack%d/%s.mag',mbOUT{1:3},mbOUT{1:3}); %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - fclose(aliCom); + % system(sprintf('chmod a=wrx %smapBack%d/%s.align',mbOUT{1:3})); if (is_first_run) @@ -1846,13 +1899,14 @@ % end if ( multi_node_run ) + % These will be aggregated in the main runAlignments.sh fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'w'); else fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}),'w'); fprintf(fOUT,'#!/bin/bash\n\n'); end % fprintf(fOUT,'cat %s | /scratch/etna/master_align.sh `xargs` &\n',aliCom_name); - fprintf(fOUT,'%s\n',aliCom_name); + fprintf(fOUT,'%s\n',aliCom_name_rerun); % Since we send to the background in a shell, makes sure the % function waits on children. @@ -1870,7 +1924,7 @@ % fprintf(fOUT,['%smapBack%d/%s.align > ',... % '%smapBack%d/%s.align_ta.log &\n'], ... % mbOutAlt{1:3},mbOutAlt{1:3}); - fprintf(fOUT,'%s\n',aliCom_name); + fprintf(fOUT,'%s\n',aliCom_name_rerun); % Since we send to the background in a shell, makes sure the % function waits on children. @@ -1880,6 +1934,10 @@ fclose(fOUT); end + % if (emc.run_tomocpr_alignments) + + % end + %system(sprintf('./mapBack/%s.align > ./mapBack/%s_ta.log',TN,TN)); %%%%%%%%% There is still sometimes a shift in Z, fit slope of the X shifts @@ -1896,13 +1954,10 @@ if ( flgRunAlignments ) - mainFile = sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}); - altFiles = sprintf('%smapBack%d/runAlignments_*.sh',mbOUT{1:2}); - + mainFile = sprintf('cache/mapBack%d/runAlignments.sh',mbOUT{2}); + altFiles = sprintf('cache/mapBack%d/runAlignments_*.sh',mbOUT{2}); + % Only possible as [cycle, 0, 0] if (multi_node_run) - % fOUT = fopen(mainFile,'w'); - % fprintf(fOUT,'#!/bin/bash\n\n'); - % fclose(fOUT); system(sprintf('rm %s && touch %s',mainFile,mainFile)); fprintf('Combining Results from alt and main\n'); system(sprintf('cat %s >> %s',altFiles,mainFile)); @@ -1914,7 +1969,9 @@ % system(sprintf('chmod a=wrx %smapBack%d/runAlignments.sh', mbOUT{1:2})); % system(sprintf('%smapBack%d/runAlignments.sh', mbOUT{1:2})); - system(sprintf('cat %smapBack%d/runAlignments.sh | parallel -j%d "cat {} | /scratch/etna/master_align.sh `xargs`"', mbOUT{1:2}, emc.nCpuCores)) + + % BH_multi_parallelWorkers will return at most nthreads + system(sprintf("cat cache/mapBack%d/runAlignments.sh | parallel -j%d 'cat {} | /scratch/etna/master_align.sh `xargs`'", mbOUT{1:2}, floor(BH_multi_parallelWorkers(256)/2))) end if ( conserveDiskSpace ) @@ -1922,17 +1979,19 @@ end - -if (tmpCache) - if (flgRunAlignments) - system(sprintf('mv %smapBack%d mapBack%d', mbOUT{1:2}, mbOUT{2})); - else +if (flgRunAlignments) + system(sprintf('mv %smapBack%d mapBack%d', mbOUT{1:2}, mbOUT{2})); +else + if (multi_node_run) % This is an partial run system(sprintf('mkdir -p cache/mapBack%d', mbOUT{2})); system(sprintf('mv %smapBack%d/* cache/mapBack%d', mbOUT{1:2}, mbOUT{2})); + else + error('This should not happen'); end end + % if (flgCleanCache) % % Double check that this exists to avoid data loss. % checkDir = dir(tmpCache); diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index fac4e5d6..999ebb07 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=0 -nightly=5 +nightly=6 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" diff --git a/transformations/BH_multi_combineLowResInfo.m b/transformations/BH_multi_combineLowResInfo.m index ea0b6dc3..97190e9c 100755 --- a/transformations/BH_multi_combineLowResInfo.m +++ b/transformations/BH_multi_combineLowResInfo.m @@ -2,12 +2,27 @@ %UNTITLED Summary of this function goes here % Detailed explanation goes here +use_original = false; +input_was_resized = false; +padded_size = 512; + +if (use_original) + method = 'cpu'; + wanted_size = [1,1,1] .* padded_size; + if ~all(size(inputRefs) == wanted_size) + input_was_resized = true; + end +else + method = 'GPU'; + wanted_size = size(inputRefs{1}{1}); +end refIDX = BH_multi_isCell( inputRefs{1} ); nRefs = length(refIDX); -[radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates([512,512,512],... - 'Cartesian','cpu',{'none'},1,0,1); +[radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(wanted_size, 'Cartesian', method, {'none'}, 1, 0, 1); radialGrid = radialGrid ./ pixelSize; +shared_lowres_info_idx = (radialGrid < 1./resCutOff); +radialGrid = []; outputRefs = cell(2,1); outputRefs{1} = cell(nRefs,1); @@ -27,30 +42,36 @@ - [ combPAD ] = BH_multi_padVal( size(inputRefs{1}{iRef}), 512 ); + [ combPAD ] = BH_multi_padVal( size(inputRefs{1}{iRef}), wanted_size ); % Oversample so the cutoff is more accurate, and use double precision for the % same reason. - oddPAD = fftn(BH_padZeros3d(inputRefs{1}{iRef}, ... - combPAD(1,:),combPAD(2,:),'cpu','doubleTaper')); - evePAD = fftn(BH_padZeros3d(inputRefs{2}{iRef}, ... - combPAD(1,:),combPAD(2,:),'cpu','doubleTaper')); + outputRefs{1}{iRef} = fftn(BH_padZeros3d(inputRefs{1}{iRef}, combPAD(1,:), combPAD(2,:), method, 'singleTaper')); + outputRefs{2}{iRef} = fftn(BH_padZeros3d(inputRefs{2}{iRef}, combPAD(1,:), combPAD(2,:), method, 'singleTaper')); - sharedInfo = (radialGrid < 1/resCutOff) .* (oddWeight.*oddPAD + eveWeight.*evePAD); + sharedInfo = oddWeight.*outputRefs{1}{iRef}(shared_lowres_info_idx) + eveWeight.*outputRefs{2}{iRef}(shared_lowres_info_idx); - oddPAD = real(ifftn(sharedInfo + (radialGrid >= 1/resCutOff).*oddPAD)); - evePAD = real(ifftn(sharedInfo + (radialGrid >= 1/resCutOff).*evePAD)); + if (use_original) + outputRefs{1}{iRef} = real(ifftn(sharedInfo + (radialGrid >= 1/resCutOff).*outputRefs{1}{iRef})); + outputRefs{2}{iRef} = real(ifftn(sharedInfo + (radialGrid >= 1/resCutOff).*outputRefs{2}{iRef})); + else + outputRefs{1}{iRef}(shared_lowres_info_idx) = sharedInfo; + outputRefs{2}{iRef}(shared_lowres_info_idx) = sharedInfo; + outputRefs{1}{iRef} = real(ifftn( outputRefs{1}{iRef})); + outputRefs{2}{iRef} = real(ifftn( outputRefs{2}{iRef})); + end + clear sharedInfo oddWeight eveWeight - - outputRefs{1}{iRef} = single(oddPAD(1+combPAD(1,1):end-combPAD(2,1), ... - 1+combPAD(1,2):end-combPAD(2,2), ... - 1+combPAD(1,3):end-combPAD(2,3))); - clear oddPAD - - outputRefs{2}{iRef} = single(evePAD(1+combPAD(1,1):end-combPAD(2,1), ... - 1+combPAD(1,2):end-combPAD(2,2), ... - 1+combPAD(1,3):end-combPAD(2,3))); - clear evePAD + + if (use_original) + outputRefs{1}{iRef} = single( outputRefs{1}{iRef}(1+combPAD(1,1):end-combPAD(2,1), ... + 1+combPAD(1,2):end-combPAD(2,2), ... + 1+combPAD(1,3):end-combPAD(2,3))); + + outputRefs{2}{iRef} = single( outputRefs{2}{iRef}(1+combPAD(1,1):end-combPAD(2,1), ... + 1+combPAD(1,2):end-combPAD(2,2), ... + 1+combPAD(1,3):end-combPAD(2,3))); + end end -clear radialGrid inputRefs +clear inputRefs end From 33690d3f84cd98f494d9daf60e7656d84a59eb52 Mon Sep 17 00:00:00 2001 From: ben Date: Tue, 18 Jun 2024 12:43:02 -0400 Subject: [PATCH 079/151] Keeping the reverted alignRaw. Note tomoCPR is still not running the alignments at the end --- alignment/BH_alignRaw3d_v2.m | 1323 ++++++++++++++++-------------- synthetic/BH_synthetic_mapBack.m | 6 +- testScripts/mCompile.sh | 3 +- 3 files changed, 721 insertions(+), 611 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index fc7e41b9..355b3890 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -159,9 +159,7 @@ nReferences(1:2) = [length(unique(refGroup{1})),length(unique(refGroup{1}))]; nReferences = nReferences .* [~isempty(refGroup{1}),~isempty(refGroup{2})]; -if (nReferences(1) ~= length(refVector{1})) - error('Number of references does not match the number of unique groups in the classVector') -end + nRefOut(1:2) = [length(unique(refGroup{1})) + sum(( refSym{1} < 0 )),... length(unique(refGroup{2})) + sum(( refSym{2} < 0 ))]; @@ -184,8 +182,7 @@ BH_multi_maskCheck(emc, 'Ali', emc.pixel_size_angstroms); [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) - + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); if (flgStartThird) @@ -291,6 +288,8 @@ [ refIMG ] = BH_multi_combineLowResInfo( refIMG, imgCounts, emc.pixel_size_angstroms, maxGoldStandard ); + + % optimize the fft for the given size. Padding to the next power of 2 is usually % slower given the dimensionalityl of the volume data. fftPlanner = rand(sizeCalc); @@ -299,6 +298,10 @@ clear fftPlanner + + + + stat_mask = []; if (eraseMask) peakMask = EMC_maskShape(eraseMaskType,sizeCalc,floor(eraseMaskRadius),'cpu',{'kernel',false}); @@ -412,11 +415,11 @@ refOUT = cell(2.*nReferences(iGold),2); for iRef = 1:nReferences(iGold) - - - refTMP = refIMG{iGold}{iRef}(padWindow(1,1) + 1: end - padWindow(2,1), ... - padWindow(1,2) + 1: end - padWindow(2,2), ... - padWindow(1,3) + 1: end - padWindow(2,3)); + + refTMP_2 = refIMG{iGold}{iRef}; refIMG{iGold}{iRef} = []; + refTMP = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... + padWindow(1,2) + 1: end - padWindow(2,2), ... + padWindow(1,3) + 1: end - padWindow(2,3)); % if not using a weighted average (adapted SPW filter), apply an @@ -429,34 +432,34 @@ - % ref_FT2{iGold}{iRef} = gather(refTMP_2); + ref_FT2{iGold}{iRef} = gather(refTMP_2); % Trim for output reference - % refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... - % padWindow(1,2) + 1: end - padWindow(2,2), ... - % padWindow(1,3) + 1: end - padWindow(2,3)); + refTMP_2 = refTMP_2(padWindow(1,1) + 1: end - padWindow(2,1), ... + padWindow(1,2) + 1: end - padWindow(2,2), ... + padWindow(1,3) + 1: end - padWindow(2,3)); % Overwrite a copy of the filtered, bandpassed ref for output - % refOUT{nOut} = real(ifftn(conj(ref_FT1{iGold}{iRef}))); - % refOUT{nOut} = gather(refOUT{nOut}(padCalc(1,1) + 1: end - padCalc(2,1), ... - % padCalc(1,2) + 1: end - padCalc(2,2), ... - % padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); + refOUT{nOut} = real(ifftn(conj(ref_FT1{iGold}{iRef}))); + refOUT{nOut} = gather(refOUT{nOut}(padCalc(1,1) + 1: end - padCalc(2,1), ... + padCalc(1,2) + 1: end - padCalc(2,2), ... + padCalc(1,3) + 1: end - padCalc(2,3)) .* volMask); - % refOUT{nOut} = refOUT{nOut}.*volMask; + refOUT{nOut} = refOUT{nOut}.*volMask; - % refOUT{nOut+1} = real(ifftn(BH_bandLimitCenterNormalize(... - % refTMP_2, '', '', padCalc, 'single'))); - % refOUT{nOut+1} = gather(refOUT{nOut+1}(padCalc(1,1) + 1: end - padCalc(2,1), ... - % padCalc(1,2) + 1: end - padCalc(2,2), ... - % padCalc(1,3) + 1: end - padCalc(2,3)) ); - % nOut = nOut + 2; + refOUT{nOut+1} = real(ifftn(BH_bandLimitCenterNormalize(... + refTMP_2, '', '', padCalc, 'single'))); + refOUT{nOut+1} = gather(refOUT{nOut+1}(padCalc(1,1) + 1: end - padCalc(2,1), ... + padCalc(1,2) + 1: end - padCalc(2,2), ... + padCalc(1,3) + 1: end - padCalc(2,3)) ); + nOut = nOut + 2; - % refOUT{nOut} = refOUT{nOut} - mean(refOUT{nOut}(:)); - % refOUT{nOut} = refOUT{nOut} ./ rms(refOUT{nOut}(:)); + refOUT{nOut} = refOUT{nOut} - mean(refOUT{nOut}(:)); + refOUT{nOut} = refOUT{nOut} ./ rms(refOUT{nOut}(:)); - % refOUT{nOut+1} = refOUT{nOut+1} - mean(refOUT{nOut+1}(:)); - % refOUT{nOut+1} = refOUT{nOut+1} ./ rms(refOUT{nOut+1}(:)); + refOUT{nOut+1} = refOUT{nOut+1} - mean(refOUT{nOut+1}(:)); + refOUT{nOut+1} = refOUT{nOut+1} ./ rms(refOUT{nOut+1}(:)); end @@ -549,6 +552,7 @@ EMC_parpool(nParProcesses+1) end +size(ref_FT2) system('mkdir -p alignResume'); @@ -562,8 +566,6 @@ if (emc.force_no_symmetry) particle_symmetry = 'C1'; end -% for iParProc = parVect - parfor iParProc = parVect symmetry = emc.symmetry; @@ -576,6 +578,7 @@ fprintf('parProc %d/%d assigned to GPU %d\n',iParProc,nParProcesses,iGPUidx); for iTomo = iterList{iParProc} + % Check for interupted alignment. previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); if exist(previousAlignment,'file') @@ -594,11 +597,26 @@ % shake up the random number generator for phi and theta rng('shuffle'); - bandpassFilt_tmp = cell(1,1); - bandpassFiltREF_tmp = cell(1,1); - + bandpassFilt_tmp = cell(nReferences(1),1); + bandpassFiltREF_tmp = cell(nReferences(1),1); + for iRef = 1:nReferences(1) + if emc.multi_reference_alignment <= 2 + bandpassFilt_tmp{iRef} = gpuArray(bandpassFilt{iRef}); + bandpassFiltREF_tmp{iRef} = gpuArray(bandpassFiltREF{iRef}); + else + bandpassFilt_tmp{iRef} = (bandpassFilt{iRef}); + bandpassFiltREF_tmp{iRef} = (bandpassFiltREF{iRef}); + end + end + + + + ref_FT1_tmp = cell(2,1); + ref_FT2_tmp = cell(2,1); + ref_WGT_tmp = cell(2,1); + ref_WGT_rot = cell(2,1); + - volMask_tmp = gpuArray(volMask); volBinary_tmp = single(find( volMask_tmp > 0.01 )); peakMaskInterpolator = ''; @@ -609,8 +627,11 @@ mip.('mask') = gpuArray(stat_mask); end + wCCC_tmp = cell(length(wCCC)); + + for iRef = 1:nReferences(1) for iWccc = 1:length(wCCC{iRef}) if (flgWeightCCC) @@ -619,8 +640,28 @@ % The check in xcf_rotational looks for a cell wCCC_tmp{iRef} = 0; end - end % iWccc - end % iRef + end + end + + + + for iGold = 1:2 + for iRef = 1:nReferences(iGold) + if emc.multi_reference_alignment <= 2 + ref_FT1_tmp{iGold}{iRef} = gpuArray(ref_FT1{iGold}{iRef}); + ref_FT2_tmp{iGold}{iRef} = gpuArray(ref_FT2{iGold}{iRef}); + ref_WGT_tmp{iGold}{iRef} = gpuArray(refWGT{iGold}{iRef}); + ref_WGT_rot{iGold}{iRef} = gpuArray(refWgtROT{iGold}{iRef}); + else + % Temp workaround, six big ribo refs crashing + ref_FT1_tmp{iGold}{iRef} = (ref_FT1{iGold}{iRef}); + ref_FT2_tmp{iGold}{iRef} = (ref_FT2{iGold}{iRef}); + ref_WGT_tmp{iGold}{iRef} = (refWGT{iGold}{iRef}); + ref_WGT_rot{iGold}{iRef} = (refWgtROT{iGold}{iRef}); + end + end + end + % sprintf('\nWorking on %d/%d volumes',iTomo,nTomograms) tic; @@ -631,16 +672,17 @@ tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); - includeList = positionList(:,26:26:26*emc.nPeaks ) ~= -9999; - includeList = any(includeList,2); - position_list = positionList(includeList,:); + binShift = [0,0,0]; nSubTomos = size(positionList,1); + iTiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoName).tiltName; + + % Can't clear inside the parfor, but make sure we don't have two tomograms % in memory at once. @@ -662,6 +704,7 @@ volHeader = getHeader(volumeData); end + % For now, set up for full grid-search only, as I intend to just do % translational and in-plane searches for now anyhow. @@ -672,30 +715,21 @@ iv1 = []; iv2 = []; iv3 = []; cccStorageBest = cell(emc.nPeaks,1); cccStorageRefine = cell(emc.nPeaks,1); - cccStorage2 = cell(emc.nPeaks,1); - cccInitial_arr = cell(emc.nPeaks,1); - cccStorageTrans= zeros(1,10, 'single', 'gpuArray'); - for iPeak = 1:emc.nPeaks - cccStorageBest{iPeak} = zeros(nSubTomos,10, 'single'); - cccStorageBest{iPeak}(:,6) = -9999; - cccStorageRefine{iPeak}= zeros(nSubTomos,10, 'single'); - cccStorageRefine{iPeak}(:,6) = -9999; - cccStorage2{iPeak} = zeros(nSubTomos,10, 'single', 'gpuArray'); - cccStorage2{iPeak}(:,6) = -9999; - cccInitial_arr{iPeak} = zeros(nSubTomos,10, 'single', 'gpuArray'); - cccInitial_arr{iPeak}(:,6) = -9999; - - + cccStorageBest{iPeak} = zeros(nSubTomos,10); + cccStorageRefine{iPeak}= zeros(nSubTomos,10); end % reset for each tomogram wdgIDX = 0; + for iSubTomo = 1:nSubTomos + + make_SF3D = true; breakPeak = 0; % for try catch on cut out vols - - + for iPeak = 1:emc.nPeaks + if (emc.track_stats) measure_noise = true; mip.('x') = {}; @@ -706,410 +740,497 @@ continue; end getInitialCCC = 1; + cccInitial = zeros(nReferences(1),10,flgPrecision, 'gpuArray'); + cccStorage2= zeros(nAngles(1).*nReferences(1),10,'gpuArray'); + + % Used in refinment loop + angCount = 1; % Check that the given subTomo is not to be ignored classIDX = positionList(iSubTomo, 26+26*(iPeak-1)); particleIDX = positionList(iSubTomo, 4); half_set = positionList(iSubTomo, 7); - - if (classIDX == -9999) - continue; - end - - center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; - angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); - % Find range to extract, and check for domain error. - if (emc.flgCutOutVolumes) - % Need some check that the windowsize has not changed! TODO TODO - - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid(2*CUTPADDING+sizeWindow, ... - sizeWindow,maskRadius, center); - else - [ indVAL, padVAL, shiftVAL ] = ... - BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... - sizeWindow,maskRadius, center); - end - if ischar(indVAL) - fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) - nIgnored = nIgnored + 1; - geometry_tmp.(tomoList{iTomo})(geometry_tmp.(tomoList{iTomo})(:,4) == particleIDX, 26) = -9999; - continue; - else - if (emc.flgCutOutVolumes) - % Test with some generic padding , only to be used on bin 1 at - % first!!! TODO add a flag to check this. - try - particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); - iparticle = gpuArray(OPEN_IMG('single',particleOUT_name,[indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - catch - fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); - geometry_tmp.(tomoList{iTomo})(geometry_tmp.(tomoList{iTomo})(:,4) == particleIDX, 26) = -9999; - breakPeak = 1; - continue; - end - else - - iparticle = gpuArray(OPEN_IMG('single', volumeData, [indVAL(1,1),indVAL(2,1)], ... - [indVAL(1,2),indVAL(2,2)], ... - [indVAL(1,3),indVAL(2,3)],'keep')); - end - [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... - padVAL(2,1:3), 'GPU', 'singleTaper'); - - if (make_SF3D) - make_SF3D = false; - % For now excluding the soften weight. - [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center, reconCoords, emc.wiener_constant); - imgWdgInterpolator = ''; - % The unshifted mask is kept in texture mem until no longer - % needed - [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); - - particleInterpolator= ''; - - particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); - end - end % Else clause on windowing (if is a continue) + % if classVector{half_set}(1,:) == 0 + % classPosition = 1; + % flgAllClasses = true; + % else + % classPosition = find(classVector{half_set}(1,:) == classIDX); + % flgAllClasses = false; + % end + % Align all valid subtomos, even if the do not belong to the classes we've selected as references. + % To ignore particles, remove them with geometry RemoveClases.m + % FIXME: what was this for? + flgAllClasses = true; + + - for iRefIdx = 1:nReferences(1); - % Just use C1 to initialize, whether or not this is the final - refInterpolator = ''; - refWdgInterpolator= ''; - ref_FT1_thread_local = gpuArray(ref_FT1{half_set}{iRefIdx}); - refWgtROT_thread_local = gpuArray(refWgtROT{half_set}{iRefIdx}); - refInterpolator = interpolator(ifftn(ref_FT1_thread_local),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator = interpolator(refWgtROT_thread_local,[0,0,0],[0,0,0],'Bah','forward','C1',false); - bandpassFilt_tmp{1} = gpuArray(bandpassFilt{iRefIdx}); - bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{iRefIdx}); + if (classIDX ~= -9999) && ... % All previously ignored particles + ( flgAllClasses || ismember(classIDX, classVector{half_set}(1,:)) ) - % In the case we are only aligning refs to those classes they came from, we - % need to check to see if we duck out here. - switch emc.multi_reference_alignment - case 0 - refToAlign = 1; - case 1 - refToAlign = classVector{half_set}(1,iRefIdx); - case 2 - refToAlign = find(classIDX == classVector{half_set}(1,:)); - if isempty(refToAlign) - fprintf('WARNING: wanted class %d not found in classVector\n',classIDX); - for i = 1:size(classVector{half_set},2) - fprintf('%d ',classVector{half_set}(1,i)); - end - error('classIDX not found in classVector'); - end - if (refToAlign ~= iRefIdx) - continue; - end - end + center = positionList(iSubTomo,[11:13]+26*(iPeak-1))./samplingRate + binShift; + angles = positionList(iSubTomo,[17:25]+26*(iPeak-1)); + % Find range to extract, and check for domain error. + if (emc.flgCutOutVolumes) + % Need some check that the windowsize has not changed! TODO TODO - if (emc.use_new_grid_search) - theta_search = 1:gridSearch.number_of_out_of_plane_angles; + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid(2*CUTPADDING+sizeWindow, ... + sizeWindow,maskRadius, center); else - theta_search = 1:size(angleStep,1); + [ indVAL, padVAL, shiftVAL ] = ... + BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... + sizeWindow,maskRadius, center); end - for iAngle = theta_search + + + + if ischar(indVAL) + fprintf('\nnow ignoring particle %d from tomo %d', iSubTomo,iTomo) + nIgnored = nIgnored + 1; + geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; + else - if (emc.use_new_grid_search) - theta = gridSearch.parameter_map.theta(iAngle); - if length(gridSearch.parameter_map.phi{iAngle}) > 1 - phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); - else - phiInc = 0; + + if (emc.flgCutOutVolumes) + % Test with some generic padding , only to be used on bin 1 at + % first!!! TODO add a flag to check this. + try + particleOUT_name = sprintf('cache/subtomo_%0.7d_%d.mrc',positionList(iSubTomo,4),iPeak); + iparticle = gpuArray(OPEN_IMG('single',particleOUT_name,[indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); + catch + fprintf('\n\nDid not load cut out vol. on subTomo %d FixMEEEEEE\n\n',iSubTomo); + geometry_tmp.(tomoList{iTomo})(iSubTomo, 26) = -9999; + breakPeak = 1; + continue; end - thetaInc = gridSearch.theta_step; - numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); else - theta = angleStep(iAngle,1); - phiInc = angleStep(iAngle,3); - thetaInc = angleStep(iAngle,4); - numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; + + iparticle = gpuArray(OPEN_IMG('single', volumeData, [indVAL(1,1),indVAL(2,1)], ... + [indVAL(1,2),indVAL(2,2)], ... + [indVAL(1,3),indVAL(2,3)],'keep')); end + [ iparticle ] = BH_padZeros3d(iparticle, padVAL(1,1:3), ... + padVAL(2,1:3), 'GPU', 'singleTaper'); - % To prevent only searching the same increments each time in a limited - % grid search, radomly offset the azimuthal angle by a random number - % between 0 and 1/2 the azimuthal increment. - azimuthalRandomizer = (rand(1)-0.5)*phiInc; - - % Calculate the increment in phi so that the azimuthal sampling is - % consistent and equal to the out of plane increment. + if (make_SF3D) + make_SF3D = false; + % For now excluding the soften weight. + [ iMaxWedgeIfft ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center, reconCoords, emc.wiener_constant); + imgWdgInterpolator = ''; + % The unshifted mask is kept in texture mem until no longer + % needed + [imgWdgInterpolator, ~] = interpolator(iMaxWedgeIfft,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + iMaxWedgeIfft =ifftshift(iMaxWedgeIfft); + + % Just use C1 to initialize, whether or not this is the final + refInterpolator = ''; + refWdgInterpolator= ''; + particleInterpolator= ''; + + [refInterpolator, ~] = interpolator(gpuArray(ref_FT2_tmp{1}{1}),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + refWdgInterpolator = interpolator(gpuArray(ref_WGT_rot{half_set}{iRef}),[0,0,0],[0,0,0],'Bah','forward','C1',false); + particleInterpolator = interpolator(gpuArray(iparticle),[0,0,0],[0,0,0], 'Bah', 'inv', 'C1', false); + end if (emc.use_new_grid_search) - % FIXME randomizer passed as bool to eulerSearch - phi_search = gridSearch.parameter_map.phi{iAngle}; + theta_search = 1:gridSearch.number_of_out_of_plane_angles; else - phi_search = 0:angleStep(iAngle,2); + theta_search = 1:size(angleStep,1); end - for iAzimuth = phi_search + for iAngle = theta_search + + if (emc.use_new_grid_search) + theta = gridSearch.parameter_map.theta(iAngle); + if length(gridSearch.parameter_map.phi{iAngle}) > 1 + phiInc = gridSearch.parameter_map.phi{iAngle}(2)-gridSearch.parameter_map.phi{iAngle}(1); + else + phiInc = 0; + end + thetaInc = gridSearch.theta_step; + numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); + else + theta = angleStep(iAngle,1); + phiInc = angleStep(iAngle,3); + thetaInc = angleStep(iAngle,4); + numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; + + end + + % To prevent only searching the same increments each time in a limited + % grid search, radomly offset the azimuthal angle by a random number + % between 0 and 1/2 the azimuthal increment. + + azimuthalRandomizer = (rand(1)-0.5)*phiInc; + + + % Calculate the increment in phi so that the azimuthal sampling is + % consistent and equal to the out of plane increment. + + if (emc.use_new_grid_search) - phi = rem(iAzimuth + azimuthalRandomizer,360); - psiInc = gridSearch.psi_step; + % FIXME randomizer passed as bool to eulerSearch + phi_search = gridSearch.parameter_map.phi{iAngle}; else - phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); - psiInc = angleStep(iAngle,5); + phi_search = 0:angleStep(iAngle,2); end - for iInPlane = inPlaneSearch - psi = iInPlane; + + for iAzimuth = phi_search - RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - + if (emc.use_new_grid_search) + phi = rem(iAzimuth + azimuthalRandomizer,360); + psiInc = gridSearch.psi_step; + else + phi = rem((phiInc * iAzimuth)+azimuthalRandomizer,360); + psiInc = angleStep(iAngle,5); + + end + + + for iInPlane = inPlaneSearch + psi = iInPlane; + %[phi,theta,psi-phi]; + + + RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; + + cccStorageTrans= zeros(1.*nReferences(1),10,'gpuArray'); + + for alignLoop = 1:2 + + + switch alignLoop + + case 1 + % This takes care of non-inter shift in the origin that is + % ignored during the windowing of the particle. + estPeakCoord = shiftVAL; + % Estimate the peakshift by rotating the ref not the particle. + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + case 2 + + bestOfRefs = sortrows(gather(cccStorageTrans), -6); + %sortrows(gather(cccStorage1(angCount:angCount+nReferences(1)-1,:)),-6); + + estPeakCoord = bestOfRefs(1,8:10); + + [ iTrimParticle ] = particleInterpolator.interp3d(... + RotMat,... + estPeakCoord,rotConvention ,... + 'inv',particle_symmetry); + + if (getInitialCCC) + [ iTrimInitial ] = particleInterpolator.interp3d(... + reshape(angles,3,3),... + shiftVAL,rotConvention ,... + 'inv',particle_symmetry); + + [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... + reshape(angles,3,3),... + [0,0,0],rotConvention ,... + 'inv',particle_symmetry); + end + + [ iWedgeMask ] = imgWdgInterpolator.interp3d(... + RotMat,... + [0,0,0],rotConvention ,... + 'inv',particle_symmetry); + end % switch on align loop + + + switch emc.multi_reference_alignment + case 0 + refToAlign = 1; + case 1 + refToAlign = 1:max(nReferences(:)); + case 2 + refToAlign = classIDX; + otherwise + error('emc.multi_reference_alignment is not 0,1,2') + end + + for iRef = refToAlign + + switch alignLoop + + case 1 + % use transpose of RotMat + + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + estPeakCoord,rotConvention ,... + 'forward','C1'); + + + + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + + + + % maybe I should be rotating peak mask here in case it has + % an odd shape, since we are leaving the proper frame + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{iRef} ,'',... + padCalc,flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{iRef} ,'',padCalc,flgPrecision); + + if (emc.track_stats && measure_noise) + + + [ ~, mip ] = BH_multi_xcf_Translational_2( ... + rotPart_FT, ... + conj(iRotRef),... + ifftshift(iRotWdg),... + iMaxWedgeIfft,... + iRotMask, peakCOM,... + mip); + + + end + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + cccStorageTrans(iRef,:) = [iRef, particleIDX, ... + phi, theta, psi - phi, ... + 0, 0, ... + peakCoord + estPeakCoord]; + case 2 + + % get starting point + if (getInitialCCC) + + initialRotPart_FT = BH_bandLimitCenterNormalize(... + iTrimInitial.*volMask_tmp,... + bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); + + + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( initialRotPart_FT, ... + ref_FT1_tmp{half_set}{iRef}, ... + ifftshift(iWedgeInitial),... + ref_WGT_tmp{half_set}{iRef}, ... + wCCC_tmp{iRef}); + + + + + cccInitial(iRef,:) = [iRef, particleIDX, ... + 0,0,0, ... + iCCC, 1, ... + shiftVAL]; + + + initialRotPart_FT = []; + + + end + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle.*volMask_tmp,... + bandpassFilt_tmp{iRef} ,volBinary_tmp,padCalc,flgPrecision); + + + + + [ iCCC, ~ ] = ... + BH_multi_xcf_Rotational( rotPart_FT, ... + ref_FT1_tmp{half_set}{iRef},... + ifftshift(iWedgeMask),... + ref_WGT_tmp{half_set}{iRef}, ... + wCCC_tmp{iRef}); + + + + + + + % Note that no new translational estimate is made, so no + % need to multiply by RotMat + cccStorage2(angCount,:) = ... + [iRef, particleIDX, ... + phi, theta, psi , ... + iCCC, 1, ... + estPeakCoord]; + + + angCount = angCount + 1; + end + + + end % loop over references. + + + + end + % This volume won't be needed until the next subTomo is considered, + % which is also where getInitialCCC Boolean is set to True again. + iTrimInitial = []; + getInitialCCC = 0; + + end % in plane angles + end % azimuth + end % polar + + % % % fprintf('Power ratio is %3.3f\n',powerOut./powerInitial); + + cccPreRefineSort = sortrows(gather(cccStorage2),-6); + + if (length(refToAlign) > 1) + cccInitial = sortrows(gather(cccInitial), -6); + cccInitial = cccInitial(1,:); + else + cccInitial = gather(cccInitial(refToAlign,:)); + + end + + if cccInitial(1,6 ) > cccPreRefineSort(1,6) + cccPreRefineSort(1,:) = cccInitial(1,:); + end + + + + % This only seems to be a problem with cut out volumes. + % Normalization maybe? + if ~any(cccPreRefineSort(1,:)) + cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); + continue + end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + if (flgRefine) + + + + % Get the results from just this subTomo and sort on CCC + + rRef = cccPreRefineSort(1,1); + rPart = cccPreRefineSort(1,2); + rPhi = cccPreRefineSort(1,3); + rPhiInc = phiInc / 4; + rTheta= cccPreRefineSort(1,4); + rTheInc = thetaInc /2; + rPsi = cccPreRefineSort(1,5); + rPsiInc = psiInc /2; + % Confirm shiftVAL is doing what it should be + rXYZest = cccPreRefineSort(1,8:10); + + if (rTheInc) + % For a larger out of plane step, search a larger range in plane + psiRefineStep = floor(sqrt(rTheInc)); + else + psiRefineStep = 1; + end + + thetaRefineStep =1; + phiRefineStep=2; + totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; + totalRefineStep = prod((2.*totalRefineStep)+1); + + cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); + + if (rPsiInc == 0) + inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; + else + inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; + end + polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; + azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; + + searchList = zeros(totalRefineStep,3); + nSearch = 1; + for iPhi = azimuthalRefine + for iTheta = polarRefine + for iPsi = inPlaneRefine + % best iPsi is origin Psi - Phi, no need to subtract here. + + searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; + + nSearch = nSearch + 1; + end + end + end % end of building angle list + + for iRefine = 1:nSearch-1 for alignLoop = 1:2 + if alignLoop == 1 + rXYZ = rXYZest; + elseif alignLoop == 2 + rXYZ = cccStorage3(iRefine,8:10); + end + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%5 + + RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; + + + + switch alignLoop + % This keeps seperate shifts due to windowing and binning from + % shifts found in CCC case 1 - % This takes care of non-inter shift in the origin that is - % ignored during the windowing of the particle. - estPeakCoord = shiftVAL; + % Estimate the peakshift by rotating the ref not the particle. iTrimParticle = ... iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... padWindow(1,2) + 1:end - padWindow(2,2) , ... padWindow(1,3) + 1:end - padWindow(2,3) ); - case 2 - estPeakCoord = gather(cccStorageTrans(1,8:10)); - + case 2 + [ iTrimParticle ] = particleInterpolator.interp3d(... RotMat,... - estPeakCoord,rotConvention ,... + rXYZ,rotConvention ,... 'inv',particle_symmetry); - if (getInitialCCC) - [ iTrimInitial ] = particleInterpolator.interp3d(... - reshape(angles,3,3),... - shiftVAL,rotConvention ,... - 'inv',particle_symmetry); - - [ iWedgeInitial ] = imgWdgInterpolator.interp3d(... - reshape(angles,3,3),... - [0,0,0],rotConvention ,... - 'inv',particle_symmetry); - end [ iWedgeMask ] = imgWdgInterpolator.interp3d(... RotMat,... [0,0,0],rotConvention ,... 'inv',particle_symmetry); - end % switch on align loop + + end - switch alignLoop - case 1 - % use transpose of RotMat - - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - estPeakCoord,rotConvention ,... - 'forward','C1'); - - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - % maybe I should be rotating peak mask here in case it has - % an odd shape, since we are leaving the proper frame - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{1} ,'',... - [0,0,0;0,0,0],flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{1} ,'',padCalc,flgPrecision); - - if (emc.track_stats && measure_noise) - - [ ~, mip ] = BH_multi_xcf_Translational_2( ... - rotPart_FT, ... - conj(iRotRef),... - ifftshift(iRotWdg),... - iMaxWedgeIfft,... - iRotMask, peakCOM,... - mip); - - end - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - cccStorageTrans(1,:) = [iRefIdx, particleIDX, ... - phi, theta, psi - phi, ... - 0, 0, ... - peakCoord + estPeakCoord]; - case 2 - % get starting point - if (getInitialCCC) - initialRotPart_FT = BH_bandLimitCenterNormalize(... - iTrimInitial.*volMask_tmp,... - bandpassFilt_tmp{1} ,volBinary_tmp,padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( initialRotPart_FT, ... - ref_FT1_thread_local, ... - ifftshift(iWedgeInitial),... - refWGT{half_set}{iRefIdx}, ... - wCCC_tmp{iRefIdx}); - - if ( iCCC > cccInitial_arr{iPeak}(iSubTomo,6)) - cccInitial_arr{iPeak}(iSubTomo,:) = [iRefIdx, particleIDX, ... - 0,0,0, ... - iCCC, 1, ... - shiftVAL]; - initialRotPart_FT = []; - end - end - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{1} ,volBinary_tmp,padCalc,flgPrecision); - - [ iCCC, ~ ] = ... - BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_thread_local,... - ifftshift(iWedgeMask),... - refWGT{half_set}{iRefIdx}, ... - wCCC_tmp{iRefIdx}); - - % Note that no new translational estimate is made, so no - % need to multiply by RotMat - if (iCCC > cccStorage2{iPeak}(iSubTomo,6) ) - - cccStorage2{iPeak}(iSubTomo,:) = ... - [iRefIdx, particleIDX, ... - phi, theta, psi , ... - iCCC, 1, ... - estPeakCoord]; - end + + + if alignLoop == 1 + - end % switch - % This volume won't be needed until the next subTomo is considered, - % which is also where getInitialCCC Boolean is set to True again. - iTrimInitial = []; - end % alignLoop (trans then rotation) - getInitialCCC = 0; % needs to be calculated on the first pass through on the second iter of align loop - end % in plane angles (psi) - end % azimuth (phi) - end % polar (theta) - end % loop over references - - - cccInitial = gather( cccInitial_arr{iPeak}(iSubTomo,:) ); - if (cccInitial(1,6) == -9999) - continue; - end - cccPreRefineSort = gather(cccStorage2{iPeak}(iSubTomo,:)); - - % This only seems to be a problem with cut out volumes. - % Normalization maybe? - if ~any(cccPreRefineSort(1,:)) - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - fprintf('all Zeros in PreRefine search, revert on subtomo %d peak %d\n',iSubTomo,iPeak); - continue; - end - %% - - if cccInitial(1,6) > cccPreRefineSort(1,6) - cccPreRefineSort(1,:) = cccInitial(1,:); - end - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - if (flgRefine) - % Get the results from just this subTomo and sort on CCC - - rRef = cccPreRefineSort(1,1); - rPart = cccPreRefineSort(1,2); - rPhi = cccPreRefineSort(1,3); - rPhiInc = phiInc / 4; - rTheta= cccPreRefineSort(1,4); - rTheInc = thetaInc /2; - rPsi = cccPreRefineSort(1,5); - rPsiInc = psiInc /2; - % Confirm shiftVAL is doing what it should be - rXYZest = cccPreRefineSort(1,8:10); - - % Host to Device for best reference - bandpassFilt_tmp{1} = gpuArray(bandpassFilt{rRef}); - bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{rRef}); - refInterpolator = ''; - refWdgInterpolator= ''; - ref_FT1_thread_local = gpuArray(ref_FT1{half_set}{rRef}); - refWgtROT_thread_local = gpuArray(refWgtROT{half_set}{rRef}); - refInterpolator = interpolator(ifftn(ref_FT1_thread_local),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator = interpolator(gpuArray(refWgtROT_thread_local),[0,0,0],[0,0,0],'Bah','forward','C1',false); - - if (rTheInc) - % For a larger out of plane step, search a larger range in plane - psiRefineStep = floor(sqrt(rTheInc)); - else - psiRefineStep = 1; - end - - thetaRefineStep =1; - phiRefineStep=2; - totalRefineStep = [psiRefineStep, thetaRefineStep, phiRefineStep]; - totalRefineStep = prod((2.*totalRefineStep)+1); - - cccStorage3 = zeros(totalRefineStep,10,'gpuArray'); - - if (rPsiInc == 0) - inPlaneRefine = rPsi - psiRefineStep*rTheInc./2:rTheInc./2: rPsi+psiRefineStep*rTheInc./2; - else - inPlaneRefine = rPsi- psiRefineStep*rPsiInc : rPsiInc : rPsi + psiRefineStep*rPsiInc; - end - polarRefine = rTheta-thetaRefineStep*rTheInc : rTheInc : rTheta + thetaRefineStep*rTheInc; - azimuthalRefine= rPhi-phiRefineStep*rPhiInc : rPhiInc : rPhi + phiRefineStep*rPhiInc; - - searchList = zeros(totalRefineStep,3); - nSearch = 1; - for iPhi = azimuthalRefine - for iTheta = polarRefine - for iPsi = inPlaneRefine - % best iPsi is origin Psi - Phi, no need to subtract here. - - searchList(nSearch, :) = [iPhi, iTheta, iPsi-iPhi]; - - nSearch = nSearch + 1; - end - end - end % end of building angle list - for iRefine = 1:nSearch-1 - for alignLoop = 1:2 - - RotMat = BH_defineMatrix(searchList(iRefine,:),rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - switch alignLoop - % This keeps seperate shifts due to windowing and binning from - % shifts found in CCC - case 1 - rXYZ = rXYZest; - % Estimate the peakshift by rotating the ref not the particle. - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - [ iRotRef ] = refInterpolator.interp3d(... RotMat',... rXYZ,rotConvention ,... @@ -1128,225 +1249,211 @@ iRotRef = BH_bandLimitCenterNormalize(... iRotRef,... - bandpassFiltREF_tmp{1},'',... - [0,0,0;0,0,0],flgPrecision); + bandpassFiltREF_tmp{rRef},'',... + padCalc,flgPrecision); rotPart_FT = BH_bandLimitCenterNormalize(... iTrimParticle,... - bandpassFilt_tmp{1} ,'',padCalc,flgPrecision); - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - + bandpassFilt_tmp{rRef} ,'',padCalc,flgPrecision); + + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + % 2016-11-11 also took out (+ rXYZ) cccStorage3(iRefine,:) = [rRef, rPart, ... searchList(iRefine,:), ... - -9999, 1, ... - peakCoord+rXYZ]; - case 2 - rXYZ = cccStorage3(iRefine,8:10); - - [ iTrimParticle ] = particleInterpolator.interp3d(... - RotMat,... - rXYZ,rotConvention ,... - 'inv',particle_symmetry); - - [ iWedgeMask ] = imgWdgInterpolator.interp3d(... - RotMat,... - [0,0,0],rotConvention ,... - 'inv',particle_symmetry); - + 1, 1, ... + peakCoord+rXYZ]; + else rotPart_FT = BH_bandLimitCenterNormalize(... iTrimParticle.*volMask_tmp,... - bandpassFilt_tmp{1},volBinary_tmp,... + bandpassFilt_tmp{rRef},volBinary_tmp,... padCalc,flgPrecision); [ iCCC, ~ ] = ... BH_multi_xcf_Rotational( rotPart_FT, ... - ref_FT1_thread_local,... + ref_FT1_tmp{half_set}{rRef},... ifftshift(iWedgeMask),... - refWGT{half_set}{rRef}, ... - wCCC_tmp{rRef}); + ref_WGT_tmp{half_set}{rRef}, ... + wCCC_tmp{iRef}); cccStorage3(iRefine,:) = [rRef, rPart, ... searchList(iRefine,:), ... iCCC, 1, ... rXYZ] ; - end % switch align loop - - end % end of alignLoop - end % end of iRefine loop - sortRef = sortrows(gather(cccStorage3),-6); - - if (sortRef(1,6) > cccStorageRefine{iPeak}(iSubTomo,6)) - cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); - end - - end % end of if flgRegine - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % Get the final translational shift for the best scoring angular - % match. - try - if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) - bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); - else - bestRotPeak = cccPreRefineSort(1,:); - bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); - end - catch - fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); - cccStorageRefine{iPeak}(iSubTomo,:) - cccPreRefineSort(1,:) - end - finalRef = bestRotPeak(1,1); - finalPart = bestRotPeak(1,2); - finalPhi = bestRotPeak(1,3); - finalTheta= bestRotPeak(1,4); - finalPsi = bestRotPeak(1,5); - % Confirm shiftVAL is doing what it should be - finalrXYZest = bestRotPeak(1,8:10); - - - RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); - RotMat = reshape(angles,3,3) * RotMat; - - if (~flgRefine) - % We can use the same reference and interpolators if we've reloaded them for the refine step, otherwise - % we need to transfer them host to device here to get the final shifts. - bandpassFilt_tmp{1} = gpuArray(bandpassFilt{finalRef}); - bandpassFiltREF_tmp{1} = gpuArray(bandpassFiltREF{finalRef}); - refInterpolator = ''; - refWdgInterpolator= ''; - ref_FT1_thread_local = gpuArray(ref_FT1{half_set}{finalRef}); - refWgtROT_thread_local = gpuArray(refWgtROT{half_set}{finalRef}); - refInterpolator = interpolator(ifftn(ref_FT1_thread_local),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - refWdgInterpolator = interpolator(gpuArray(refWgtROT_thread_local),[0,0,0],[0,0,0],'Bah','forward','C1',false); - end - - iTrimParticle = ... - iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... - padWindow(1,2) + 1:end - padWindow(2,2) , ... - padWindow(1,3) + 1:end - padWindow(2,3) ); - - % use transpose of RotMat - %%% 2016-11-11 estPeakCoord should have been finalrXYZest in - %%% the last writing, but now switching to zeros - try - [ iRotRef ] = refInterpolator.interp3d(... - RotMat',... - finalrXYZest,rotConvention ,... - 'forward','C1'); - [ iRotWdg ] = refWdgInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - - [ iRotMask ] = peakMaskInterpolator.interp3d(... - RotMat',... - [0,0,0],rotConvention ,... - 'forward','C1'); - catch - fprintf('\n\nFinal ref,part,phi,theta,psi %f %f %f %f %f\n\n',... - bestRotPeak(:,1:5)); - bestRotPeak(1,1:5) - fprintf('BreakPeak %d\n',breakPeak); - error('errrorsoedfsdf') - end - - iRotRef = BH_bandLimitCenterNormalize(... - iRotRef,... - bandpassFiltREF_tmp{1} ,'',... - [0,0,0;0,0,0],flgPrecision); - - rotPart_FT = BH_bandLimitCenterNormalize(... - iTrimParticle,... - bandpassFilt_tmp{1} ,'',padCalc,flgPrecision ); - - - [ peakCoord ] = BH_multi_xcf_Translational( ... - rotPart_FT.*ifftshift(iRotWdg), ... - conj(iRotRef).*iMaxWedgeIfft,... - iRotMask, peakCOM); - - - % Subtract shiftVAL since this is due to windowing, not the actual - % position. - cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), peakCoord + finalrXYZest - shiftVAL]) ; - %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - % It is probably more useful see the shifts in the particle - % reference frame vs. the avg which was the original - if (emc.printShiftsInParticleBasis) - printShifts = zeros(3,3); - printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); - printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); - printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); - else - printShifts = [cccInitial(1,end-2:end); ... - cccPreRefineSort(1,end-2:end);... - cccStorageBest{iPeak}(iSubTomo,end-2:end)]; - end + end + end + + end + + sortRef = sortrows(gather(cccStorage3),-6); + cccStorageRefine{iPeak}(iSubTomo,:) = sortRef(1,:); + + end % end of refinement loop + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Get the final translational shift for the best scoring angular + % match. + try + if (flgRefine) && any(cccStorageRefine{iPeak}(iSubTomo,:)) + bestRotPeak = cccStorageRefine{iPeak}(iSubTomo,:); + else + bestRotPeak = cccPreRefineSort(1,:); + bestRotPeak(1,5) = bestRotPeak(1,5) - bestRotPeak(1,3); + end + catch + fprintf('\nflgRefine %d, iPeak %d, iSubTomo %d\n',flgRefine,iPeak,iSubTomo); + cccStorageRefine{iPeak}(iSubTomo,:) + cccPreRefineSort(1,:) + end + + finalRef = bestRotPeak(1,1); + finalPart = bestRotPeak(1,2); + finalPhi = bestRotPeak(1,3); + finalTheta= bestRotPeak(1,4); + finalPsi = bestRotPeak(1,5); + % Confirm shiftVAL is doing what it should be + finalrXYZest = bestRotPeak(1,8:10); + + RotMat = BH_defineMatrix([finalPhi, finalTheta, finalPsi],rotConvention, 'inv'); + RotMat = reshape(angles,3,3) * RotMat; + - % Print out in Angstrom - printShifts = printShifts .* emc.pixel_size_angstroms; - - - deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); - if (emc.print_alignment_stats && deltaCCC < 0 && abs(deltaCCC) > 0.15*cccInitial(1,6)) - fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); - fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); - - end - - if (emc.track_stats) - - if thetaInc > 0 - cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); - else - cccStorageBest{iPeak}(iSubTomo,end-3) = 0; - end - - % % I'm not sold on what do do with this. The distribution over the - % % shift parameters doesn't really seem to make sense to me. There - % % are too many factors that can lead to large shifts (e.g. - % % tomoCPR) If we were searching the full angular space each - % % iteration, then this would make sense. - % mip_mean = mip.X./mip.N; - % mip_covar = mip.X2./mip.N - transpose(mip_mean)*(mip_mean); - % mip_covar_inv = mip_covar\eye(3); - % gauss_norm = ((2.*pi).^(3/2).*abs(mip_covar)).^-1; - % gauss_exp = exp(-0.5.*(printShifts(2,:)-mip_mean)*mip_covar_inv*transpose(printShifts(2,:)-mip_mean)); - - end - cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); - cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); - if (emc.print_alignment_stats && flgRefine) - cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:), ... - 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... - cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... - 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + + iTrimParticle = ... + iparticle(padWindow(1,1) + 1:end - padWindow(2,1) , ... + padWindow(1,2) + 1:end - padWindow(2,2) , ... + padWindow(1,3) + 1:end - padWindow(2,3) ); + + + % use transpose of RotMat + %%% 2016-11-11 estPeakCoord should have been finalrXYZest in + %%% the last writing, but now switching to zeros + try + [ iRotRef ] = refInterpolator.interp3d(... + RotMat',... + finalrXYZest,rotConvention ,... + 'forward','C1'); + [ iRotWdg ] = refWdgInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + + [ iRotMask ] = peakMaskInterpolator.interp3d(... + RotMat',... + [0,0,0],rotConvention ,... + 'forward','C1'); + catch + fprintf('\n\nFinal ref,part,phi,theta,psi %f %f %f %f %f\n\n',... + bestRotPeak(:,1:5)); + bestRotPeak(1,1:5) + fprintf('BreakPeak %d\n',breakPeak); + error('errrorsoedfsdf') + end + + + iRotRef = BH_bandLimitCenterNormalize(... + iRotRef,... + bandpassFiltREF_tmp{finalRef} ,'',... + padCalc,flgPrecision); + + rotPart_FT = BH_bandLimitCenterNormalize(... + iTrimParticle,... + bandpassFilt_tmp{finalRef} ,'',padCalc,flgPrecision ); + + + [ peakCoord ] = BH_multi_xcf_Translational( ... + rotPart_FT.*ifftshift(iRotWdg), ... + conj(iRotRef).*iMaxWedgeIfft,... + iRotMask, peakCOM); + + + % Subtract shiftVAL since this is due to windowing, not the actual + % position. + cccStorageBest{iPeak}(iSubTomo,:) = gather([bestRotPeak(1,1:7), ... + peakCoord + finalrXYZest - shiftVAL]) ; + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + % It is probably more useful see the shifts in the particle + % reference frame vs. the avg which was the original + if (emc.printShiftsInParticleBasis) + printShifts = zeros(3,3); + printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); + printShifts(2,:) = RotMat * reshape(cccPreRefineSort(1,end-2:end),3,1); + printShifts(3,:) = RotMat * reshape(cccStorageBest{iPeak}(iSubTomo,end-2:end),3,1); + else + printShifts = [cccInitial(1,end-2:end); ... + cccPreRefineSort(1,end-2:end);... + cccStorageBest{iPeak}(iSubTomo,end-2:end)]; + end + + % Print out in Angstrom + printShifts = printShifts .* emc.pixel_size_angstroms; + + + deltaCCC = cccStorageBest{iPeak}(iSubTomo,6) - cccInitial(1,6); + if (emc.print_alignment_stats && deltaCCC < 0 && abs(deltaCCC) > 0.15*cccInitial(1,6)) + fprintf('Drop in CCC greater than 15 pph (%2.3f), reverting to prior.\n', deltaCCC); + fprintf(['\n%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine', iPeak,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + cccStorageBest{iPeak}(iSubTomo,:) = cccInitial(1,:); + + end + + if (emc.track_stats) + + if thetaInc > 0 + cccStorageBest{iPeak}(iSubTomo,end-3) = gather(mean(mip.x , 'all')./std(mip.x,0,'all')./thetaInc); + else + cccStorageBest{iPeak}(iSubTomo,end-3) = 0; + end + + % % I'm not sold on what do do with this. The distribution over the + % % shift parameters doesn't really seem to make sense to me. There + % % are too many factors that can lead to large shifts (e.g. + % % tomoCPR) If we were searching the full angular space each + % % iteration, then this would make sense. + % mip_mean = mip.X./mip.N; + % mip_covar = mip.X2./mip.N - transpose(mip_mean)*(mip_mean); + % mip_covar_inv = mip_covar\eye(3); + % gauss_norm = ((2.*pi).^(3/2).*abs(mip_covar)).^-1; + % gauss_exp = exp(-0.5.*(printShifts(2,:)-mip_mean)*mip_covar_inv*transpose(printShifts(2,:)-mip_mean)); + + end + + + cccInitial(1,1) = classVector{iGold}(cccInitial(1,1)); + cccStorageBest{iPeak}(iSubTomo,1) = classVector{iGold}(cccStorageBest{iPeak}(iSubTomo,1)); + if (emc.print_alignment_stats && flgRefine) + cccPreRefineSort(1,1) = classVector{iGold}(cccPreRefineSort(1,1)); + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:), ... + 'PreRefine', iPeak,classIDX,[cccPreRefineSort(1,1:4),cccPreRefineSort(1,5)-... + cccPreRefineSort(1,3),cccPreRefineSort(1,6:7),printShifts(2,:)], ... + 'PostRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + + else + if (emc.print_alignment_stats) + fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... + '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... + 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... + 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); + end + end + + + end % if condition on newly ignored particles - else - if (emc.print_alignment_stats) - fprintf(['\n%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n', ... - '%s\t%d, %d,%d,%d,%6.3f,%6.3f,%6.3f,%6.6f,%6.6f,%6.3f,%6.3f,%6.3f\n'], ... - 'PreInitial',iPeak,classIDX, cccInitial(1,1:end-3),printShifts(1,:),... - 'PreRefine',iPeak,classIDX,cccStorageBest{iPeak}(iSubTomo,1:end-3),printShifts(3,:)); - end end - if ~(rem(iSubTomo,100)) timeClass = toc; @@ -1364,8 +1471,10 @@ rotPart_FT = []; rotParticle = []; end % end loop over possible peaks + + iMaxWedgeIfft = []; end % loop over subTomos - iMaxWedgeIfft = []; + for iPeak = 1:emc.nPeaks @@ -1380,6 +1489,7 @@ sortCCC(:,1+10*(iPeak-1):10+10*(iPeak-1)) = cccStorageBest{iPeak}; end + % % % % I think this is redundant now, but leaving until I double check. % % % save('sortCCC.mat','sortCCC'); [~,a,~] = unique(sortCCC(:,2), 'stable','rows'); @@ -1399,7 +1509,8 @@ fprintf(angOut,'\n'); end fclose(angOut); - end % if/ -> else clause to check for previous alignment, return to loop on references + + end % if clause to check for previous alignment end % loop over tomos bestAnglesResults{iParProc} = bestAngles_tmp; geometryResults{iParProc} = geometry_tmp; diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 22ea6a21..f6266ebb 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -331,7 +331,7 @@ % This will be re-calculated once the tilt-series size is known. targetPatchSize = emc.tomoCPR_target_n_patches_x_y; else - targetPatchSize = ceil(max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch)))); + targetPatchSize = [1,1].*ceil(max(500, ceil(2.*(particle_radius).*sqrt(nFiducialsPerPatch)))); end @@ -599,7 +599,7 @@ iRefIDX = 1; iClassIDX = positionList(iSubTomo,26); - use_this_class = true + use_this_class = true; if (nRefs > 1) % Assuming generally there are fewer classes seleceted as references than there are total classes % For those that aren't one of the select ones, we could try to track the best matched reference from the most recent @@ -634,7 +634,7 @@ if ischar(indVAL) || ~use_this_class - if ischar(indVal) + if ischar(indVAL) fprintf('ignoring subTomo %d for out of bounds conditions.\n', iSubTomo); else fprintf('ignoring subTomo %d for class %d because it is not one of the references.\n', iSubTomo, iClassIDX); diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 999ebb07..a9712826 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,8 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=0 -nightly=6 - +nightly=7 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From ae26c9407760f7e3a1fc13280e341bb1c2de0666 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 10 Jul 2024 10:07:33 -0400 Subject: [PATCH 080/151] Roughed in a very basic prallelization for PCA over scaleSpace or multiRef. --- statistics/BH_pcaPub.m | 497 ++++++++++++++++--------------- synthetic/BH_synthetic_mapBack.m | 28 +- 2 files changed, 276 insertions(+), 249 deletions(-) diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index a425bdab..c212c6ad 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -169,7 +169,6 @@ outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); -%%%flgGold = emc.('flgGoldStandard'); flgNorm = 1;% emc.('flgNormalizeWMDs'); @@ -199,16 +198,6 @@ -% Removed flgGold everywhere else, but keep ability to classify full data set at -% the end (after all alignment is finished.) - - -if (emc.classification) - flgGold = 0; -else - flgGold = 1; -end - load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; @@ -275,17 +264,10 @@ % If emc.classification is negative combine the data for clustering, but don't set % any of the alignment changes to be persistant so that extracted class % averages are still independent half-sets. -if (flgGold) - oddRot = eye(3); -else - iRefPrev = 1; - - aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)) - oddRot = reshape(aliParams(1,:),3,3)'; - % refine the translation per particle. - clear iRefPrev -end +aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',1)); +oddRot = reshape(aliParams(1,:),3,3)'; + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -355,39 +337,35 @@ end % IF combining for analysis, resample prior to any possible binning. -if ~(flgGold) - if (nReferences(1) ~= nReferences(2)) - error('When combining half sets, the number of references must match') - end - size(averageMotif) - if (nReferences(1) > 1) - for iRef = 1:nReferences(1) - averageMotif{1}{iRef} = averageMotif{2}{iRef} + ... - BH_resample3d(gather(averageMotif{1}{iRef}), ... - oddRot, ... - aliParams(2,1:3), ... - {'Bah',1,'spline'}, 'cpu', ... - 'forward'); - averageMotif{2}{iRef} = []; - end - else - averageMotif{1} = averageMotif{2} + ... - BH_resample3d(gather(averageMotif{1}), ... +if (nReferences(1) ~= nReferences(2)) + error('When combining half sets, the number of references must match') +end +size(averageMotif) +if (nReferences(1) > 1) + for iRef = 1:nReferences(1) + averageMotif{1}{iRef} = averageMotif{2}{iRef} + ... + BH_resample3d(gather(averageMotif{1}{iRef}), ... oddRot, ... aliParams(2,1:3), ... {'Bah',1,'spline'}, 'cpu', ... 'forward'); - averageMotif{2} = []; + averageMotif{2}{iRef} = []; end - +else + averageMotif{1} = averageMotif{2} + ... + BH_resample3d(gather(averageMotif{1}), ... + oddRot, ... + aliParams(2,1:3), ... + {'Bah',1,'spline'}, 'cpu', ... + 'forward'); + averageMotif{2} = []; end + %%% incomplete, the idea is to generate an antialiased scaled volume for PCA if ( refSamplingRate ~= samplingRate ) fprintf('Resampling from %d refSampling to %d pcaSampling\n',refSamplingRate,samplingRate); - for iGold = 1:1+flgGold - averageMotif{iGold} = BH_reScale3d(averageMotif{iGold},'',sprintf('%f',1/samplingRate),'GPU'); - end + averageMotif{1} = BH_reScale3d(averageMotif{1},'',sprintf('%f',1/samplingRate),'GPU'); if (flgLoadMask) externalMask = BH_reScale3d(externalMask,'',sprintf('%f',1/samplingRate),'GPU'); @@ -399,41 +377,29 @@ prevVarianceMaps = struct(); if (flgVarianceMap) - for iGold = 1:1+flgGold - - if (flgGold) - if iGold == 1; - halfSet = 'ODD'; - else - halfSet = 'EVE'; - end - else - halfSet = 'STD'; - end - - % For randomsubset (PREVIOUS_PCA = 0) the suffix is *_pcaPart.mat) but - % presumably we could have also just done full, so try that first - - try - load(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet)) - catch - fprintf('\nDid not find, %s_%s_pcaFUll.mat, trying *_pcaPart.mat\n',outputPrefix,halfSet); - load(sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet)); - end + halfSet = 'STD'; + + % For randomsubset (PREVIOUS_PCA = 0) the suffix is *_pcaPart.mat) but + % presumably we could have also just done full, so try that first + try + load(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet)) + catch + fprintf('\nDid not find, %s_%s_pcaFUll.mat, trying *_pcaPart.mat\n',outputPrefix,halfSet); + load(sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet)); + end + + % In most cases, this is the number of "features" specified in the + % parameter file, but in some data not even this may non-zero singluar + % values are found, so the number could be different (lower) + for iScale = 1:emc.n_scale_spaces + eigsFound = size(coeffs{iScale},1); + fname = sprintf('%s_varianceMap%d-%s-%d.mrc', ... + outputPrefix, eigsFound, halfSet, iScale); - % In most cases, this is the number of "features" specified in the - % parameter file, but in some data not even this may non-zero singluar - % values are found, so the number could be different (lower) - for iScale = 1:emc.n_scale_spaces - eigsFound = size(coeffs{iScale},1); - fname = sprintf('%s_varianceMap%d-%s-%d.mrc', ... - outputPrefix, eigsFound, halfSet, iScale); - - prevVarianceMaps.(sprintf('h%d',iGold)).(sprintf('s%d',iScale)) = ... - OPEN_IMG('single', fname).^flgStdDev; - end - clear v coeffs eigsFound idxList + prevVarianceMaps.(sprintf('h%d',iGold)).(sprintf('s%d',iScale)) = ... + OPEN_IMG('single', fname).^flgStdDev; end + clear v coeffs eigsFound idxList end @@ -456,8 +422,7 @@ end % when combining the addition is harmless, but is a convenient way to % include when sets are left 100% separate. - % volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); - volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, ... + volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1}, pixelSize, ... {'pca', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); end @@ -476,27 +441,25 @@ volMask = struct(); nPixels = zeros(2,emc.n_scale_spaces); for iScale = 1:emc.n_scale_spaces - for iGold = 1:1+flgGold - stHALF = sprintf('h%d',iGold); - stSCALE = sprintf('s%d',iScale); - if (flgVarianceMap) - if (test_multi_ref_diffmap) - error('test_multi_ref_diffmap is incompatible with flgVarianceMap') - end - volTMP = gather(volumeMask.*prevVarianceMaps.(stHALF).(stSCALE)); - else - volTMP = gather(volumeMask); + stHALF = sprintf('h%d',1); + stSCALE = sprintf('s%d',iScale); + if (flgVarianceMap) + if (test_multi_ref_diffmap) + error('test_multi_ref_diffmap is incompatible with flgVarianceMap') end - - masks.('volMask').(stHALF).(stSCALE) = (volTMP); - masks.('binary').(stHALF).(stSCALE) = (volTMP >= bh_global_binary_pcaMask_threshold); - masks.('binary').(stHALF).(stSCALE) = ... - masks.('binary').(stHALF).(stSCALE)(:); - masks.('binaryApply').(stHALF).(stSCALE) = (volTMP >= 0.01); - - nPixels(iGold,iScale) = gather(sum(masks.('binary').(stHALF).(stSCALE))); - clear volTMP stHALF stSCALE + volTMP = gather(volumeMask.*prevVarianceMaps.(stHALF).(stSCALE)); + else + volTMP = gather(volumeMask); end + + masks.('volMask').(stHALF).(stSCALE) = (volTMP); + masks.('binary').(stHALF).(stSCALE) = (volTMP >= bh_global_binary_pcaMask_threshold); + masks.('binary').(stHALF).(stSCALE) = ... + masks.('binary').(stHALF).(stSCALE)(:); + masks.('binaryApply').(stHALF).(stSCALE) = (volTMP >= 0.01); + + nPixels(1,iScale) = gather(sum(masks.('binary').(stHALF).(stSCALE))); + clear volTMP stHALF stSCALE end clear volumeMask @@ -512,7 +475,7 @@ kernelSize = ceil(threeSigma(iScale).*3) + 3; kernelSize = kernelSize + (1-mod(kernelSize,2)); % masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); - masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'gpu', {}); + masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'cpu', {}); % SAVE_IMG( masks.('scaleMask').(sprintf('s%d',iScale)), ... @@ -522,43 +485,45 @@ end end -avgMotif_FT = cell(1+flgGold,emc.n_scale_spaces); -avgFiltered = cell(1+flgGold,emc.n_scale_spaces); -% Here always read in both, combine if flgGold = 0 -for iGold = 1:1+flgGold - for iScale = 1:emc.n_scale_spaces - - if (test_multi_ref_diffmap) - tmp_avg = averageMotif{iGold}{iScale}; - else - tmp_avg = averageMotif{iGold}; - end - - - tmp_avg = tmp_avg - mean(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); - tmp_avg = tmp_avg ./ rms(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); - tmp_avg = tmp_avg .* masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)); - % FIXME: ideally we would do both, but for testing I am stealing scaleSpace for iRef - if ~(test_multi_ref_diffmap) - tmp_avg = EMC_convn(single(gpuArray(tmp_avg)) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); - end - avgMotif_FT{iGold, iScale} = ... - BH_bandLimitCenterNormalize(tmp_avg,... - BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... - masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)),... - [0,0,0;0,0,0],'single'); - - avgFiltered{iGold, iScale} = real(ifftn(avgMotif_FT{iGold, iScale})); - - avgFiltered{iGold, iScale} = avgFiltered{iGold, iScale} - mean(avgFiltered{iGold, iScale}(masks.('binary').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); - avgFiltered{iGold, iScale} = gather(avgFiltered{iGold, iScale} ./rms(avgFiltered{iGold, iScale}(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))) .* ... - masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale))); +avgMotif_FT = cell(emc.n_scale_spaces,1); +avgFiltered = cell(emc.n_scale_spaces,1); +for iScale = 1:emc.n_scale_spaces + + if (test_multi_ref_diffmap) + tmp_avg = averageMotif{1}{iScale}; + else + tmp_avg = averageMotif{1}; end + + + tmp_avg = tmp_avg - mean(tmp_avg(masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale)))); + tmp_avg = tmp_avg ./ rms(tmp_avg(masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale)))); + tmp_avg = tmp_avg .* masks.('volMask').(sprintf('h%d',1)).(sprintf('s%d',iScale)); + % FIXME: ideally we would do both, but for testing I am stealing scaleSpace for iRef + if ~(test_multi_ref_diffmap) + tmp_avg = EMC_convn(single(gpuArray(tmp_avg)) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); + end + avgMotif_FT{iScale} = ... + BH_bandLimitCenterNormalize(tmp_avg,... + BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... + masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale)),... + [0,0,0;0,0,0],'single'); + + avgFiltered{iScale} = real(ifftn(avgMotif_FT{iScale})); + + avgFiltered{iScale} = avgFiltered{iScale} - mean(avgFiltered{iScale}(masks.('binary').(sprintf('h%d',1)).(sprintf('s%d',iScale)))); + avgFiltered{iScale} = gather(avgFiltered{iScale} ./rms(avgFiltered{iScale}(masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale)))) .* ... + masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale))); + + % Gather these so they are not destroyed when assigning gpuDevice + avgMotif_FT{iScale} = gather(avgMotif_FT{iScale}); + avgFiltered{iScale} = gather(avgFiltered{iScale}); end -montOUT = BH_montage4d(avgFiltered(1,:),''); + +montOUT = BH_montage4d(avgFiltered(:),''); SAVE_IMG(MRCImage(montOUT), sprintf('%s_filt.mrc', outputPrefix),pixelSize); clear montOUT @@ -566,23 +531,12 @@ % If emc.Pca_randSubset is string with a previous matfile use this, without any % decomposition. -for iGold = 1:1+flgGold +for iGold = 1 flgRefIsPadded = 0; - if (flgGold) - if iGold == 1; - halfSet = 'ODD'; - stHALF = sprintf('h%d',iGold); - randSet =1; - else - stHALF = sprintf('h%d',iGold); - halfSet = 'EVE'; - randSet = 2; - end - else - stHALF = sprintf('h%d',iGold); - halfSet = 'STD'; - randSet = [1,2]; - end + + stHALF = sprintf('h%d',iGold); + halfSet = 'STD'; + randSet = [1,2]; if (PREVIOUS_PCA) previousPCA = sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet); @@ -612,17 +566,19 @@ % Initialize array in main memory for pca clear dataMatrix tempDataMatrix - dataMatrix = cell(3,1); - tempDataMatrix = cell(3,1); + dataMatrix = cell(emc.n_scale_spaces,1); + % tempDataMatrix = cell(3,1); for iScale = 1:emc.n_scale_spaces dataMatrix{iScale} = zeros(nPixels(iGold,iScale), nSUBSET, 'single'); - tempDataMatrix{iScale} = zeros(nPixels(iGold,iScale), nParticlesOnGpuBeforeTransfer, 'single', 'gpuArray'); + % FIXME: if reverting parfor, this will be needed, for now, giving each worker its own copy. + % tempDataMatrix{iScale} = zeros(nPixels(iGold,iScale), nParticlesOnGpuBeforeTransfer, 'single', 'gpuArray'); end % Pull masks onto GPU (which are cleared along with everything else when % the device is reset at the end of each loop.) gpuMasks = struct(); + % FIXME: these gpumasks are no longer needed for iScale = 1:emc.n_scale_spaces stSCALE = sprintf('s%d',iScale); @@ -635,10 +591,11 @@ if ~(test_multi_ref_diffmap) gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); end - - - gpuMasks.('highPass').(stSCALE) = BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize); + + masks.('highPass').(stHALF).(stSCALE) = gather(BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize)); end + + % % % for iGold_inner = 1:1+flgGold % % % for iScale = 1:emc.n_scale_spaces @@ -648,25 +605,73 @@ % % % end - nExtracted = 1; - nTemp = 1; - nTempPrev = 0; - idxList = zeros(1,nSUBSET); - peakList = zeros(1,nSUBSET); + nExtracted = cell(emc.n_scale_spaces,1); + nTemp = cell(emc.n_scale_spaces,1); + nTempPrev = cell(emc.n_scale_spaces,1); + idxList = cell(emc.n_scale_spaces,1); + peakList = cell(emc.n_scale_spaces,1); + reconCoords = cell(emc.n_scale_spaces,1); + nIgnored = cell(emc.n_scale_spaces,1); + TLT = cell(emc.n_scale_spaces,1); + geom = cell(emc.n_scale_spaces,1); + + for iScale = 1:emc.n_scale_spaces + nExtracted{iScale} = 1; + nTemp{iScale} = 1; + nTempPrev{iScale} = 0; + nIgnored{iScale} = 0; + idxList{iScale} = zeros(1,nSUBSET); + peakList{iScale} = zeros(1,nSUBSET); + reconCoords{iScale} = subTomoMeta.mapBackGeometry.tomoCoords; + geom{iScale} = subTomoMeta.(cycleNumber).(geom_name); + TLT{iScale} = subTomoMeta.('tiltGeometry'); + + end + % nExtracted = 1; + % nTemp = 1; + % nTempPrev = 0; + % idxList = zeros(1,nSUBSET); + % peakList = zeros(1,nSUBSET); firstLoop = true; sI = 1; - nIgnored = 0; + + + % FIXME: hardcoded for initial testing + nWorkers = 4+1; + if ~isempty(gcp('nocreate')) + delete(gcp('nocreate')) + EMC_parpool(nWorkers); + else + EMC_parpool(nWorkers); + end + fprintf('init with %d workers\n',nWorkers); + + parfor iScaleSpace = 1:emc.n_scale_spaces + % for iScaleSpace = 1:emc.n_scale_spaces + + % FIXME: need to check gpu idx that are available as in other parallel code + gpuDevice(iScaleSpace); + fprintf('parProc %d/%d assigned to GPU %d\n',iScaleSpace,nWorkers,iScaleSpace); + + volMask = gpuArray(masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScaleSpace))); + binaryMask = gpuArray(masks.('binary').(sprintf('h%d',iGold)).(sprintf('s%d',iScaleSpace))); + if ~(test_multi_ref_diffmap) + scaleMask = gpuArray(masks.('scaleMask').(sprintf('s%d',iScaleSpace))); + end + highPass = gpuArray(masks.('highPass').(sprintf('h%d',iGold)).(sprintf('s%d',iScaleSpace))); + + d_avgMotif_FT = gpuArray(avgMotif_FT{iScaleSpace}); + + % Initialize the tempDataMatrix + tempDataMatrix = zeros(nPixels(iGold,iScaleSpace), nParticlesOnGpuBeforeTransfer, 'single', 'gpuArray'); + for iTomo = 1:nTomograms tomoName = tomoList{iTomo}; iGPU = 1; - tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; - tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - reconCoords = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); - TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); if (emc.flgCutOutVolumes) @@ -682,10 +687,6 @@ end - iTiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoName).tiltName; - - tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); - fprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); @@ -706,10 +707,10 @@ % bins = 1./[1000,800,600,400,300,200,150,100,80,60,50,40,35,30,28,26,24,22,20,18,16,14,12,10,8,6,4,2]; % bins = [0, bins]; - [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(size(avgMotif_FT{iGold, iScale}),'Cartesian',... + [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(size(d_avgMotif_FT),'Cartesian',... 'GPU',{'none'},1,0,1); - bins = radialGrid(1:floor(size(avgMotif_FT{iGold, iScale},1)/2),1,1); + bins = radialGrid(1:floor(size(d_avgMotif_FT,1)/2),1,1); bins = bins(bins < 0.5); radialMask = cell(length(bins)-1,1); @@ -722,7 +723,7 @@ end - wdgBP = ifftshift(gpuMasks.('highPass').(sprintf('s%d',iScale))); + wdgBP = ifftshift(highPass); for iSubTomo = 1:nSubTomos %%%%% %%%%% @@ -734,7 +735,6 @@ % subset of peaks. FIXME includeParticle = positionList(iSubTomo, 8); particleIDX = positionList(iSubTomo, 4); % Same for all peaks - iPeak=0; % make sure this exists if we are no including the particle if (includeParticle) make_sf3d = true; for iPeak = 0:emc.nPeaks-1 @@ -750,7 +750,7 @@ make_sf3d = false; radialGrid = ''; padWdg = [0,0,0;0,0,0]; - [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, TLT, center, reconCoords, wiener_constant); + [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, TLT{iScaleSpace}.(tomoList{iTomo}), center, reconCoords{iScaleSpace}.(tomoList{iTomo}), wiener_constant); end % If flgGold there is no change, otherwise temporarily resample the @@ -772,9 +772,7 @@ sizeWindow, maskRadius, center); end - if ~(flgGold) - shiftVAL = shiftVAL + aliParams(2,1:3)./samplingRate; - end + shiftVAL = shiftVAL + aliParams(2,1:3)./samplingRate; using_this_subtomo = true; if ~ischar(indVAL) @@ -831,92 +829,100 @@ + if (test_multi_ref_diffmap) + iPrt = iTrimParticle; + else + iPrt = EMC_convn(iTrimParticle , scaleMask); + end - for iScale = 1:emc.n_scale_spaces - if (test_multi_ref_diffmap) - iPrt = iTrimParticle; - else - iPrt = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); - end - - iPrt = BH_bandLimitCenterNormalize( ... - iPrt .* ... - gpuMasks.('volMask').(sprintf('s%d',iScale)), ... - gpuMasks.('highPass').(sprintf('s%d',iScale)),... - gpuMasks.('binary').(sprintf('s%d',iScale)),... - [0,0,0;0,0,0],'single'); - - [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iPrt,ifftshift(iWedge),... - flgNorm,pixelSize,radialMask, padWdg); - - - % using_this_subtomo is set true for each particle, but if we are not finite for ANY scale space, we want to skip this particle for all - % scale spaces. - if all(isfinite(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale))))) - tempDataMatrix{iScale}(:,nTemp) = single(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale)))); - else - fprintf('inf or nan in subtomo %d scalePace %d',particleIDX, iScale); - using_this_subtomo = false; - end - - end % loop on scale spaces - clear iAvg iWmd iTrimParticle + + iPrt = BH_bandLimitCenterNormalize( ... + iPrt .* ... + volMask,... + highPass,... + binaryMask,... + [0,0,0;0,0,0],'single'); + + [iWmd,~] = BH_diffMap(d_avgMotif_FT,iPrt,ifftshift(iWedge),... + flgNorm,pixelSize,radialMask, padWdg); + + + % using_this_subtomo is set true for each particle, but if we are not finite for ANY scale space, we want to skip this particle for all + % scale spaces. + if all(isfinite(iWmd(binaryMask))) + tempDataMatrix(:,nTemp{iScaleSpace}) = single(iWmd(binaryMask)); + else + fprintf('inf or nan in subtomo %d scalePace %d',particleIDX, iScaleSpace); + using_this_subtomo = false; + end + + iWmd = []; + iTrimParticle = []; if (using_this_subtomo) - idxList(1, nExtracted) = particleIDX; - peakList(1,nExtracted) = iPeak+1; - nExtracted = nExtracted + 1; - nTemp = nTemp + 1; + idxList{iScaleSpace}(1, nExtracted{iScaleSpace}) = particleIDX; + peakList{iScaleSpace}(1,nExtracted{iScaleSpace}) = iPeak+1; + nExtracted{iScaleSpace} = nExtracted{iScaleSpace} + 1; + nTemp{iScaleSpace} = nTemp{iScaleSpace} + 1; % pull data of the gpu every 1000 particls (adjust this to max mem) - if nTemp - 1 == nParticlesOnGpuBeforeTransfer - for iScale = 1:emc.n_scale_spaces - dataMatrix{iScale}(:,1+nTempPrev:nTemp+nTempPrev-1) = ... - gather(tempDataMatrix{iScale}(:,1:nTemp-1)); - end + if nTemp{iScaleSpace} - 1 == nParticlesOnGpuBeforeTransfer + dataMatrix{iScaleSpace}(:,1+nTempPrev{iScaleSpace}:nTemp{iScaleSpace}+nTempPrev{iScaleSpace}-1) = gather(tempDataMatrix(:,1:nTemp{iScaleSpace}-1)); - nTempPrev = nTempPrev + nTemp - 1; - nTemp = 1; + nTempPrev{iScaleSpace} = nTempPrev{iScaleSpace} + nTemp{iScaleSpace} - 1; + nTemp{iScaleSpace} = 1; end else - nIgnored = nIgnored + 1; + nIgnored{iScaleSpace} = nIgnored{iScaleSpace} + 1; fprintf('Ignoring subtomo %d from %s\n',particleIDX, tomoList{iTomo}); - eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; - subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; + eraseIDX = geom{iScaleSpace}.(tomoList{iTomo})(:,4) == particleIDX; + % Only update with the first worker + if (iScaleSpace == 1) + geom{iScaleSpace}.(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; + end end else - nIgnored = nIgnored + 1; + nIgnored{iScaleSpace} = nIgnored{iScaleSpace} + 1; fprintf('Ignoring inside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); - eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; - subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; + eraseIDX = geom{iScaleSpace}.(tomoList{iTomo})(:,4) == particleIDX; + % Only update with the first worker + if (iScaleSpace == 1) + geom{iScaleSpace}.(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; + end end % end of update data matrix or if ignoring update metaData end % end of loop over peaks else - nIgnored = nIgnored + 1; + nIgnored{iScaleSpace} = nIgnored{iScaleSpace} + 1; if ~(emc.Pca_randSubset) fprintf('Ignoring outside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); - eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; - subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; + eraseIDX = geom{iScaleSpace}.(tomoList{iTomo})(:,4) == particleIDX; + % Only update with the first worker + if (iScaleSpace == 1) + geom{iScaleSpace}.(tomoList{iTomo})(eraseIDX, 26) = -9999; + end end end % end of ignore if statment from extracted window out of bounds (ischar(indVAL)) if ~rem(iSubTomo,100) fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... iSubTomo, nSubTomos,iPeak+1,emc.nPeaks, iTomo,nTomograms); - fprintf('Total nExtracted = %d\n', nExtracted-1); - fprintf('Total nIgnored = %d\n', nIgnored); - + if (iScaleSpace == 1) + fprintf('Total nExtracted = %d\n', nExtracted{iScaleSpace}-1); + fprintf('Total nIgnored{iScaleSpace} = %d\n', nIgnored{iScaleSpace}); + end end end % end of the loop over subTomos - clear volumeData + volumeData = []; end % end of the loop over Tomograms, + dataMatrix{iScaleSpace}(:,1+nTempPrev{iScaleSpace}:nTemp{iScaleSpace}-1+nTempPrev{iScaleSpace}) = gather(tempDataMatrix(:,1:nTemp{iScaleSpace}-1)); + end % end of the parallel loop over scale spaces % % % volBinaryMask = reshape(gather(volBinaryMask),sizeMask); for iScale = 1:emc.n_scale_spaces @@ -925,17 +931,36 @@ end - subTomoMeta.(cycleNumber).('newIgnored_PCA').(halfSet) = gather(nIgnored); - - subTomoMeta = subTomoMeta; + subTomoMeta.(cycleNumber).('newIgnored_PCA').(halfSet) = gather(nIgnored{1}); save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); + % FIXME: delete if removing the parallel loop + % for iScale = 1:emc.n_scale_spaces + % dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... + % gather(tempDataMatrix{iScale}(:,1:nTemp-1)); + % end + + + % Check to make sure the values for each scale space match as they should for iScale = 1:emc.n_scale_spaces - dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... - gather(tempDataMatrix{iScale}(:,1:nTemp-1)); + for val = 1:length(idxList{iScale}) + if idxList{1}(val) ~= idxList{iScale}(val) + fprintf('WARNING: idxList is not being filled correctly, expected %f got %f\n',idxList{1}(val),idxList{iScale}(val)); + end + end end - - clear tempDataMatrix + idxList = idxList{1}; + % Check to make sure the values for each scale space match as they should + for iScale = 1:emc.n_scale_spaces + for val = 1:length(peakList{iScale}) + if peakList{1}(val) ~= peakList{iScale}(val) + fprintf('WARNING: peakList is not being filled correctly, expected %f got %f\n',peakList{1}(val),peakList{iScale}(val)); + end + end + end + peakList = peakList{1}; + + % Get rid of any zero vals from newly ignored particles which are there due to % pre-allocation. Assuming no zeros have found their way in anywhere else which % would be a major problem. @@ -1046,7 +1071,7 @@ eigenImage = eigenImage - mean(eigenImage(masks.('binaryApply').(stHALF).(sprintf('s%d',iScale)))); eigenImage = eigenImage ./rms(eigenImage(masks.('binaryApply').(stHALF).(sprintf('s%d',iScale)))).* masks.('binary').(stHALF).(sprintf('s%d',iScale)) ; eigList{iEig,1} = gather(eigenImage); - eigList_SUM{iEig,1} = gather((eigenImage + avgFiltered{iGold, iScale} )./2); + eigList_SUM{iEig,1} = gather((eigenImage + avgFiltered{iScale} )./2); end diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index f6266ebb..ccbf7f5f 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -390,7 +390,8 @@ avgColor = zeros(reconstruction_size, 'int16'); end - if (emc.save_mapback_classes) + save_mapback_masks = false; + if (emc.save_mapback_classes && save_mapback_masks) avgColor = zeros(reconstruction_size, 'int16'); end @@ -455,7 +456,7 @@ - if (emc.save_mapback_classes) + if (emc.save_mapback_classes && save_mapback_masks) avgColor = zeros(reconstruction_size, 'int16'); end @@ -663,15 +664,16 @@ iColorMap(iColorMap >= 0.05) = iRefIDX; iColorMap = gather(int16(iColorMap)); end - - avgColor(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3)) = avgColor(indVAL(1,1):indVAL(2,1), ... - indVAL(1,2):indVAL(2,2), ... - indVAL(1,3):indVAL(2,3)) + ... - iColorMap(1+padVAL(1,1):end-padVAL(2,1),... - 1+padVAL(1,2):end-padVAL(2,2),... - 1+padVAL(1,3):end-padVAL(2,3)); + if (save_mapback_masks) + avgColor(indVAL(1,1):indVAL(2,1), ... + indVAL(1,2):indVAL(2,2), ... + indVAL(1,3):indVAL(2,3)) = avgColor(indVAL(1,1):indVAL(2,1), ... + indVAL(1,2):indVAL(2,2), ... + indVAL(1,3):indVAL(2,3)) + ... + iColorMap(1+padVAL(1,1):end-padVAL(2,1),... + 1+padVAL(1,2):end-padVAL(2,2),... + 1+padVAL(1,3):end-padVAL(2,3)); + end end @@ -777,13 +779,13 @@ end clear avgTomo - if (emc.save_mapback_classes || flgClassAvg) + if ((emc.save_mapback_classes || flgClassAvg) && save_mapback_masks) SAVE_IMG(single(avgColor),{sprintf('%smapBack%d/%s.tmpTomoColor', mbOUT{1:3}),'half'},pixel_size); clear avgColor end tmpTomoBin = 1; - if (emc.save_mapback_classes || flgClassAvg && tmpTomoBin > 1) + if ((emc.save_mapback_classes || flgClassAvg && tmpTomoBin > 1) && save_mapback_masks) system(sprintf(['binvol -bin %d %smapBack%d/%s.tmpTomoColor ',... '%smapBack%d/%s.bin%dTomoColor.mrc'], ... tmpTomoBin,mbOUT{1:3},mbOUT{1:3},tmpTomoBin)); From 510d5d5ac4d306216ac6f2cad87b28666dcb6bee Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 19 Jul 2024 09:23:29 -0400 Subject: [PATCH 081/151] Add a more rigrous check on file validity to catch the case where the header is intact but the image file is still truncated. Add an (untested) method to remove all -9999 ignored particles from the metadata which should improve performance and disk usage on large projects where each copy of the metadata can be hundreds of MB. --- @MRCImage/checkFullFile.m | 11 +++++++++++ metaData/BH_geometryAnalysis.m | 21 +++++++++++++++++++++ metaData/OPEN_IMG.m | 7 +++++-- metaData/emc_check_for_valid_image_file.m | 10 ++++++---- testScripts/mCompile.sh | 2 +- 5 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 @MRCImage/checkFullFile.m diff --git a/@MRCImage/checkFullFile.m b/@MRCImage/checkFullFile.m new file mode 100644 index 00000000..426bc0cf --- /dev/null +++ b/@MRCImage/checkFullFile.m @@ -0,0 +1,11 @@ +function is_file_the_expected_size = checkFullFile(mrc_image_obj) + + + bytesPerElement = getModeBytes(mrc_image_obj); + dimensions = getDimensions(mrc_image_obj); + + expected_size = dimensions(1) * dimensions(2) * dimensions(3) * bytesPerElement + 1024 + mrc_image_obj.header.nBytesExtended; + + + is_file_the_expected_size = getFileNBytes(mrc_image_obj) == expected_size; +end \ No newline at end of file diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 70443f26..eb65025e 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -485,7 +485,28 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... file_out = sprintf('%s-cccCutoff.pdf', outputPrefix); saveas(gcf, file_out,'pdf') + + case 'RemoveIgnoredParticles' + % Get distribution of CCC from given cycle rawAlignment + % Save histogram, also remove given bottom percentage and report CCC cutoff + printf('\n\t\nRemoving ignored particles from the meta data to save space, this must be run after alignment or after a call to emClarity skip, post classification\n\n.'); + if ~(strcmpi(STAGEofALIGNMENT, 'RawAlignment')) + error('Can only remove fraction at RawAlignment Stage') + end + n_removed = 0; + n_total = 0; + for iTomo = 1:nTomograms + % Get the non ignored values + non_ignored = any(geometry.(tomoList{iTomo})(:,26:26*emc.nPeaks) ~= -9999, 2); + n_total = n_total + size(geometry.(tomoList{iTomo}),1); + n_removed = n_removed + sum(~non_ignored); + geometry.(tomoList{iTomo}) = geometry.(tomoList{iTomo})(non_ignored,:); + end + + fprintf('Removed %d/%d particles\n',n_removed,n_total); + + case 'ListPercentiles' cccVector = []; % Gather included scores diff --git a/metaData/OPEN_IMG.m b/metaData/OPEN_IMG.m index e76a4534..59608ba4 100644 --- a/metaData/OPEN_IMG.m +++ b/metaData/OPEN_IMG.m @@ -1,10 +1,13 @@ function [ vol ] = OPEN_IMG(precision, filename, varargin) - if isa(filename, 'MRCImage') vol = getVolume(filename, varargin{:}); else - vol = getVolume(MRCImage(filename), varargin{:}); + if isfile(filename) + vol = getVolume(MRCImage(filename), varargin{:}); + else + error('\n\nFile does not exist: %s\n\n', filename); + end end if strcmp(precision, 'single') diff --git a/metaData/emc_check_for_valid_image_file.m b/metaData/emc_check_for_valid_image_file.m index 5d018c02..517a8ea7 100644 --- a/metaData/emc_check_for_valid_image_file.m +++ b/metaData/emc_check_for_valid_image_file.m @@ -1,19 +1,21 @@ function [ is_valid ] = emc_check_for_valid_image_file(wanted_filename) % Check if the file exists + fprintf('Checking if file exists: %s\n', wanted_filename); if isfile(wanted_filename) try + fprintf('Attempting to read file: %s\n', wanted_filename); test_header = MRCImage(wanted_filename, 0); - is_valid = true; - return; + fprintf('Reading file: %s\n', wanted_filename); + % From the @MRCImage class + is_valid = checkFullFile(test_header); catch + fprintf('Error reading file: %s\n', wanted_filename); system(['rm ' wanted_filename]); is_valid = false; - return; end else is_valid = false; - return; end end \ No newline at end of file diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index a9712826..b88c6577 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=0 -nightly=7 +nightly=8 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 1858b35343175004e9cc653afb0188bc9254ffcf Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 27 Sep 2024 12:07:23 -0400 Subject: [PATCH 082/151] I think this fixes the parallel multi ref pca --- @MRCImage/checkFullFile.m | 3 + alignment/BH_templateSearch3d_2.m | 4 +- logicals/BH_isWindowValid.m | 8 +- logicals/BH_multi_checkGPU.m | 3 +- metaData/BH_geometryAnalysis.m | 4 +- metaData/emc_check_for_valid_image_file.m | 18 +- statistics/BH_clusterPub.m | 7 +- statistics/BH_pcaPub.m | 237 +++++++++++----------- testScripts/emClarity.m | 2 +- testScripts/mCompile.sh | 4 +- 10 files changed, 150 insertions(+), 140 deletions(-) diff --git a/@MRCImage/checkFullFile.m b/@MRCImage/checkFullFile.m index 426bc0cf..b5011911 100644 --- a/@MRCImage/checkFullFile.m +++ b/@MRCImage/checkFullFile.m @@ -8,4 +8,7 @@ is_file_the_expected_size = getFileNBytes(mrc_image_obj) == expected_size; + if ~is_file_the_expected_size + fprintf('File is not the expected size: %d instead is %d\n', expected_size, getFileNBytes(mrc_image_obj)); + end end \ No newline at end of file diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 8f3d0b9c..be97c19a 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -165,7 +165,7 @@ % [ recGeom, ~, ~, ~] = BH_multi_recGeom( sprintf('recon/%s_recon.coords',tomoName), mapBackIter); -bp_vals(2) = 2.*max(latticeRadius); +% bp_vals(2) = 2.*max(latticeRadius); statsRadiusAng = stats_diameter_fraction.*[2,2,2].*max(latticeRadius); statsRadius = ceil(statsRadiusAng./pixelSize); % Convert to binned pixels maskRadius = ceil(0.5.*[1,1,1].*max(latticeRadius)./pixelSize); @@ -176,7 +176,7 @@ eraseMaskRadius = floor((eraseMaskRadius) ./ (pixelSize)); eraseMaskRadius = eraseMaskRadius + mod(eraseMaskRadius,2); -fprintf('EXPERIMENTAL setting the highpass to match the max particle diameter. %3.3f Ang\n\n', bp_vals(2)); +% fprintf('EXPERIMENTAL setting the highpass to match the max particle diameter. %3.3f Ang\n\n', bp_vals(2)); fprintf('\ntomograms normalized in %f Angstrom cubic window\n',statsRadiusAng(1)); diff --git a/logicals/BH_isWindowValid.m b/logicals/BH_isWindowValid.m index eb33e506..3f226835 100755 --- a/logicals/BH_isWindowValid.m +++ b/logicals/BH_isWindowValid.m @@ -1,5 +1,5 @@ function [ INDICES, PADVALUES, SHIFTS ] = ... - BH_isWindowValid( VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, CENTER ) + BH_isWindowValid( VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, CENTER, varargin ) %Address out of bounds conditions. % % @@ -39,6 +39,12 @@ % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +use_noise_instead_of_ignoring = false; +if nargin > 4 + if strcmpi(varargin{1}, 'pca') + use_noise_instead_of_ignoring = true; + end +end minSizeMask = (max(MASK_RADIUS)+6).*[2,2,2]; winLowCorner = ceil((WINDOW_SIZE-1) ./ 2); % if window size is odd then there should be as many pixels to the left and to diff --git a/logicals/BH_multi_checkGPU.m b/logicals/BH_multi_checkGPU.m index bd617ec1..039c4e8b 100755 --- a/logicals/BH_multi_checkGPU.m +++ b/logicals/BH_multi_checkGPU.m @@ -17,7 +17,8 @@ for iGpu = 1:nGpus try gpuDev = gpuDevice(iGpu); - mostMem(iGpu,:) = [iGpu, gpuDev.AvailableMemory/gpuDev.TotalMemory]; + % FIXME the min memory should be a global set somewhere else. + mostMem(iGpu,:) = [iGpu, gpuDev.AvailableMemory/gpuDev.TotalMemory * (gpuDev.AvailableMemory > 8e8)]; catch fprintf('\n\nWarning: matlab failed to instantiate a gpuDevice for iGPU %d\n\n'); mostMem(iGpu,:) = [iGpu, 0]; diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index eb65025e..7c33afa4 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -490,7 +490,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... case 'RemoveIgnoredParticles' % Get distribution of CCC from given cycle rawAlignment % Save histogram, also remove given bottom percentage and report CCC cutoff - printf('\n\t\nRemoving ignored particles from the meta data to save space, this must be run after alignment or after a call to emClarity skip, post classification\n\n.'); + fprintf('\n\t\nRemoving ignored particles from the meta data to save space, this must be run after alignment or after a call to emClarity skip, post classification\n\n.'); if ~(strcmpi(STAGEofALIGNMENT, 'RawAlignment')) error('Can only remove fraction at RawAlignment Stage') end @@ -540,7 +540,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... fprintf(fID,'%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t\n%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t\n',percentiles'); fclose(fID); otherwise - error('OPERATION must be WriteCsv, RemoveClasses, ShiftAll, RemoveFraction not %s', OPERATION) + error('OPERATION must be WriteCsv, RemoveClasses, ShiftAll, RemoveFraction, RemoveIgnoredParticles, not %s', OPERATION) end % Redundant for WriteCsv, otherwise update the new geometry, which was backed up diff --git a/metaData/emc_check_for_valid_image_file.m b/metaData/emc_check_for_valid_image_file.m index 517a8ea7..65ac3c35 100644 --- a/metaData/emc_check_for_valid_image_file.m +++ b/metaData/emc_check_for_valid_image_file.m @@ -1,21 +1,27 @@ function [ is_valid ] = emc_check_for_valid_image_file(wanted_filename) + is_valid = true; + is_file = false; % Check if the file exists - fprintf('Checking if file exists: %s\n', wanted_filename); + %fprintf('Checking if file exists: %s\n', wanted_filename); if isfile(wanted_filename) + is_file = true; try fprintf('Attempting to read file: %s\n', wanted_filename); test_header = MRCImage(wanted_filename, 0); - fprintf('Reading file: %s\n', wanted_filename); - % From the @MRCImage class - is_valid = checkFullFile(test_header); catch - fprintf('Error reading file: %s\n', wanted_filename); - system(['rm ' wanted_filename]); is_valid = false; end + % So all we know is that the header was readable, now check that there are at least the right number of bytes in the file + is_valid = checkFullFile(test_header); + else is_valid = false; end + if ( is_file && ~is_valid ) + % fprintf('Error reading file: %s\n', wanted_filename); + system([sprintf('rm -f %s',wanted_filename)]); + end + end \ No newline at end of file diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index 534217b3..837c8b7d 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -132,10 +132,13 @@ % length(relativeScale), nScaleSpace); % end % + for iScale = 1:nScaleSpace + size(coeffsUNTRIMMED{iScale}) + end if isa(coeffsUNTRIMMED, 'cell') - [nI,nJ] = size(coeffsUNTRIMMED{1}); + [nI,nJ] = size(coeffsUNTRIMMED{1}) for iScale = 1:nScaleSpace - nFeatures(iScale) = nnz(featureVector{iGold}(iScale,:)); + nFeatures(iScale) = nnz(featureVector{iGold}(iScale,:)) end coeffMat = zeros(sum(nFeatures), nJ, 'single'); nAdded = 0; diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index c212c6ad..a711e310 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -227,6 +227,7 @@ [ useGPU ] = BH_multi_checkGPU( -1 ); + gDev = gpuDevice(useGPU); @@ -248,11 +249,11 @@ cpuVols = struct; [ preSizeWindow, preSizeCalc, preSizeMask, prePadWindow, prePadCalc ] = ... - BH_multi_validArea(preMaskSize,preMaskRadius, emc.scale_calc_size ) + BH_multi_validArea(preMaskSize,preMaskRadius, emc.scale_calc_size ); [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); if (test_multi_ref_diffmap) @@ -276,15 +277,15 @@ refGroup = cell(2,1); classVector{1} = emc.('Raw_classes_odd')(1,:); classVector{2} = emc.('Raw_classes_eve')(1,:); -refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ] -refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ] +refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ]; +refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ]; for iGold = 1:2 % Sort low to high, because order is rearranged as such unstack refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:) + refVector{iGold} = refVectorFull{iGold}(1,:); % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:) + refGroup{iGold} = refVectorFull{iGold}(3,:); end @@ -610,8 +611,9 @@ nTempPrev = cell(emc.n_scale_spaces,1); idxList = cell(emc.n_scale_spaces,1); peakList = cell(emc.n_scale_spaces,1); + is_validData = cell(emc.n_scale_spaces,1); + tomoIDXList = cell(emc.n_scale_spaces,1); reconCoords = cell(emc.n_scale_spaces,1); - nIgnored = cell(emc.n_scale_spaces,1); TLT = cell(emc.n_scale_spaces,1); geom = cell(emc.n_scale_spaces,1); @@ -619,8 +621,9 @@ nExtracted{iScale} = 1; nTemp{iScale} = 1; nTempPrev{iScale} = 0; - nIgnored{iScale} = 0; idxList{iScale} = zeros(1,nSUBSET); + tomoIDXList{iScale} = zeros(1,nSUBSET); + is_validData{iScale} = zeros(1,nSUBSET); peakList{iScale} = zeros(1,nSUBSET); reconCoords{iScale} = subTomoMeta.mapBackGeometry.tomoCoords; geom{iScale} = subTomoMeta.(cycleNumber).(geom_name); @@ -646,11 +649,15 @@ end fprintf('init with %d workers\n',nWorkers); - parfor iScaleSpace = 1:emc.n_scale_spaces + parVect = 1:emc.n_scale_spaces + nGPUs = emc.nGPUs; + parfor iScaleSpace = parVect % for iScaleSpace = 1:emc.n_scale_spaces + gpuIDXList = mod(parVect + nGPUs, nGPUs)+1; + iGPUidx = gpuIDXList(iScaleSpace); + gpuDevice(iGPUidx); % FIXME: need to check gpu idx that are available as in other parallel code - gpuDevice(iScaleSpace); fprintf('parProc %d/%d assigned to GPU %d\n',iScaleSpace,nWorkers,iScaleSpace); volMask = gpuArray(masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScaleSpace))); @@ -667,13 +674,10 @@ for iTomo = 1:nTomograms - tomoName = tomoList{iTomo}; iGPU = 1; - - if (emc.flgCutOutVolumes) volumeData = []; else @@ -687,47 +691,35 @@ end - fprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName); + fprintf('Working on %d/%d volumes %s, iScaleSpace %d\n',iTomo,nTomograms,tomoName, iScaleSpace); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); - + % Loop over peaks inside each tomo to limit wedge mask xfer positionList = positionList(positionList(:,26) ~= -9999,:); nSubTomos = size(positionList,1); - - - % reset for each tomogram wdgIDX = 0; radialMask = ''; if (flgNorm) - - - % bins = 1./[1000,800,600,400,300,200,150,100,80,60,50,40,35,30,28,26,24,22,20,18,16,14,12,10,8,6,4,2]; - % bins = [0, bins]; - [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(size(d_avgMotif_FT),'Cartesian',... 'GPU',{'none'},1,0,1); - + bins = radialGrid(1:floor(size(d_avgMotif_FT,1)/2),1,1); bins = bins(bins < 0.5); - + radialMask = cell(length(bins)-1,1); - + for iBin = 1:length(bins)-1 radialMask{iBin} = find(radialGrid >= bins(iBin) & radialGrid < bins(iBin+1)); end - + radialGrid = ''; - end wdgBP = ifftshift(highPass); for iSubTomo = 1:nSubTomos - %%%%% %%%%% - - % Check that the given subTomo is not to be ignored - for now, treat % all peaks as included. The assumption is that using this will be % for initializing the project to get a good starting model. "True" @@ -771,10 +763,14 @@ BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... sizeWindow, maskRadius, center); end + + % Skipping particles has created friction several times over the years, so I'm going to make this an error for now. + if ischar(indVAL) + error("invalid windowing, is your mask larger than the one used in emClarity avg? indVAL"); + end shiftVAL = shiftVAL + aliParams(2,1:3)./samplingRate; - using_this_subtomo = true; if ~ischar(indVAL) % Read in and interpolate at single precision as the local values % in the interpolant suffer from any significant round off errors. @@ -827,14 +823,11 @@ padWindow(1,2)+1 : end - padWindow(2,2), ... padWindow(1,3)+1 : end - padWindow(2,3)); - - if (test_multi_ref_diffmap) iPrt = iTrimParticle; else iPrt = EMC_convn(iTrimParticle , scaleMask); end - iPrt = BH_bandLimitCenterNormalize( ... iPrt .* ... @@ -846,75 +839,42 @@ [iWmd,~] = BH_diffMap(d_avgMotif_FT,iPrt,ifftshift(iWedge),... flgNorm,pixelSize,radialMask, padWdg); - - % using_this_subtomo is set true for each particle, but if we are not finite for ANY scale space, we want to skip this particle for all - % scale spaces. - if all(isfinite(iWmd(binaryMask))) - tempDataMatrix(:,nTemp{iScaleSpace}) = single(iWmd(binaryMask)); - else - fprintf('inf or nan in subtomo %d scalePace %d',particleIDX, iScaleSpace); - using_this_subtomo = false; - end - iWmd = []; - iTrimParticle = []; - - + - if (using_this_subtomo) - idxList{iScaleSpace}(1, nExtracted{iScaleSpace}) = particleIDX; - peakList{iScaleSpace}(1,nExtracted{iScaleSpace}) = iPeak+1; - nExtracted{iScaleSpace} = nExtracted{iScaleSpace} + 1; - nTemp{iScaleSpace} = nTemp{iScaleSpace} + 1; + if (all(isfinite(iWmd(binaryMask)))) + tempDataMatrix(:,nTemp{iScaleSpace}) = single(iWmd(binaryMask)); + + is_validData{iScaleSpace}(1, nExtracted{iScaleSpace}) = 1; + % pull data of the gpu every 1000 particls (adjust this to max mem) - if nTemp{iScaleSpace} - 1 == nParticlesOnGpuBeforeTransfer - dataMatrix{iScaleSpace}(:,1+nTempPrev{iScaleSpace}:nTemp{iScaleSpace}+nTempPrev{iScaleSpace}-1) = gather(tempDataMatrix(:,1:nTemp{iScaleSpace}-1)); + if ( nTemp{iScaleSpace} == nParticlesOnGpuBeforeTransfer ) + dataMatrix{iScaleSpace}(:,1+nTempPrev{iScaleSpace}:nTemp{iScaleSpace}+nTempPrev{iScaleSpace}) = gather(tempDataMatrix(:,1:nTemp{iScaleSpace})); - nTempPrev{iScaleSpace} = nTempPrev{iScaleSpace} + nTemp{iScaleSpace} - 1; + nTempPrev{iScaleSpace} = nTempPrev{iScaleSpace} + nTemp{iScaleSpace} ; nTemp{iScaleSpace} = 1; end - else - nIgnored{iScaleSpace} = nIgnored{iScaleSpace} + 1; - fprintf('Ignoring subtomo %d from %s\n',particleIDX, tomoList{iTomo}); - eraseIDX = geom{iScaleSpace}.(tomoList{iTomo})(:,4) == particleIDX; - % Only update with the first worker - if (iScaleSpace == 1) - geom{iScaleSpace}.(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; - end - end - - - else - nIgnored{iScaleSpace} = nIgnored{iScaleSpace} + 1; - fprintf('Ignoring inside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); - eraseIDX = geom{iScaleSpace}.(tomoList{iTomo})(:,4) == particleIDX; - % Only update with the first worker - if (iScaleSpace == 1) - geom{iScaleSpace}.(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; - end - - end % end of update data matrix or if ignoring update metaData - + end % end of update data matrix or if ignoring due to non-finite processing post diffmap - update metaData + + iWmd = []; + iTrimParticle = []; + + end % end of if condition on valid windowing + idxList{iScaleSpace}(1, nExtracted{iScaleSpace}) = particleIDX; + peakList{iScaleSpace}(1,nExtracted{iScaleSpace}) = iPeak + 1; + nExtracted{iScaleSpace} = nExtracted{iScaleSpace} + 1; + nTemp{iScaleSpace} = nTemp{iScaleSpace} + 1; + tomoIDXList{iScaleSpace}(1,nExtracted{iScaleSpace}) = iTomo; end % end of loop over peaks - else - nIgnored{iScaleSpace} = nIgnored{iScaleSpace} + 1; - if ~(emc.Pca_randSubset) - fprintf('Ignoring outside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); - eraseIDX = geom{iScaleSpace}.(tomoList{iTomo})(:,4) == particleIDX; - % Only update with the first worker - if (iScaleSpace == 1) - geom{iScaleSpace}.(tomoList{iTomo})(eraseIDX, 26) = -9999; - end - end - end % end of ignore if statment from extracted window out of bounds (ischar(indVAL)) + + end % end of ignore for partial data set + if ~rem(iSubTomo,100) - fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... - iSubTomo, nSubTomos,iPeak+1,emc.nPeaks, iTomo,nTomograms); - if (iScaleSpace == 1) + fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... + iSubTomo, nSubTomos, iPeak+1, emc.nPeaks, iTomo, nTomograms); fprintf('Total nExtracted = %d\n', nExtracted{iScaleSpace}-1); - fprintf('Total nIgnored{iScaleSpace} = %d\n', nIgnored{iScaleSpace}); end end end % end of the loop over subTomos @@ -931,53 +891,81 @@ end - subTomoMeta.(cycleNumber).('newIgnored_PCA').(halfSet) = gather(nIgnored{1}); - save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - - % FIXME: delete if removing the parallel loop - % for iScale = 1:emc.n_scale_spaces - % dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... - % gather(tempDataMatrix{iScale}(:,1:nTemp-1)); - % end - + % Get a count of all the valid particles - note if one peak is being ignored, this is not valid FIXME! + is_validData_bool = true(1,nSUBSET); + for iScale = 1:emc.n_scale_spaces + is_validData_bool = is_validData_bool & is_validData{iScale}; + end + + n_validData = sum(is_validData_bool) - % Check to make sure the values for each scale space match as they should + % Sanity check that all the data idx are the same for iScale = 1:emc.n_scale_spaces - for val = 1:length(idxList{iScale}) - if idxList{1}(val) ~= idxList{iScale}(val) - fprintf('WARNING: idxList is not being filled correctly, expected %f got %f\n',idxList{1}(val),idxList{iScale}(val)); - end + idxList{iScale} = idxList{iScale} .* is_validData_bool; + peakList{iScale} = peakList{iScale} .* is_validData_bool; + if iScale > 1 + if any(idxList{iScale} ~= idxList{1}) + idxList{1} + idxList{iScale} + error('Error: found a difference in valid data idxList between scale spaces'); + end end end + idxList = idxList{1}; - % Check to make sure the values for each scale space match as they should + peakList = peakList{1}; + + % Remove any columns corresponding to an ignored particle and also update the geometry + clean_geometry = subTomoMeta.(cycleNumber).(geom_name); + for iScale = 1:emc.n_scale_spaces - for val = 1:length(peakList{iScale}) - if peakList{1}(val) ~= peakList{iScale}(val) - fprintf('WARNING: peakList is not being filled correctly, expected %f got %f\n',peakList{1}(val),peakList{iScale}(val)); + fprintf('Cleaning geometry for scale space %d\n',iScale); + fprintf('nPixels x n_validData = %d x %d\n',nPixels(1,iScale),n_validData); + tempDataMatrix = zeros(nPixels(1,iScale), n_validData, 'single'); + tempDataMatrix(:,1:n_validData) = dataMatrix{iScale}(:,is_validData_bool); + dataMatrix{iScale} = []; + dataMatrix{iScale} = tempDataMatrix; + end + dataMatrix + tempDataMatrix = []; + + % Remove empty columns from the geometry + for iErase = 1:length(idxList) + if ~(is_validData_bool(1,iErase)) + tomo_name = tomoList{tomoIDXList(1,iErase)}; + eraseIDX = clean_geometry.(tomo_name)(:,4) == idxList(1,iErase); + if (sum(eraseIDX) > emc.nPeaks || sum(eraseIDX) == 0) + error('Error: found %d particles with the same index %d',sum(eraseIDX),idxList{iScale}(1,iErase)); end + clean_geometry.(tomo_name)(eraseIDX(1), 26:26:26*emc.nPeaks) = -9999; end end - peakList = peakList{1}; + + % Now update the subtomo meta and save it + subTomoMeta.(cycleNumber).(geom_name) = clean_geometry; + save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); + + % Now shrink the idxList and peakList + idxList = idxList(1,is_validData_bool); + peakList = peakList(1,is_validData_bool); + + + % FIXME: delete if removing the parallel loop + % for iScale = 1:emc.n_scale_spaces + % dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... + % gather(tempDataMatrix{iScale}(:,1:nTemp-1)); + % end - % Get rid of any zero vals from newly ignored particles which are there due to - % pre-allocation. Assuming no zeros have found their way in anywhere else which - % would be a major problem. - cleanIDX = (idxList~=0)'; - idxList = idxList(cleanIDX); - peakList = peakList(cleanIDX); for iScale = 1:emc.n_scale_spaces - dataMatrix{iScale} = dataMatrix{iScale}(:,1:size(idxList,2)); % Center the rows for row = 1:size(dataMatrix{iScale},1) dataMatrix{iScale}(row,:) = dataMatrix{iScale}(row,:) - mean(double(dataMatrix{iScale}(row,:))); end end - - + dataMatrix %save('preparpoolSave.mat'); try @@ -1024,7 +1012,9 @@ U{iScale} = single(U{iScale}); S{iScale} = single(S{iScale}); V{iScale} = single(V{iScale}); - + size(U{iScale}) + size(S{iScale}) + size(V{iScale}) % [U{iScale},S{iScale},V{iScale}] = svd(dataMatrix{iScale}, 0); sDiag{iScale} = diag(S{iScale}); @@ -1035,7 +1025,8 @@ fprintf('Found %d / %d non-zero eigenvalues sum = %4.4f, in set %s.\n All singular values converged is t/f ( %d ) ', ... numNonZero, size(S{iScale}, 1), sum(sDiag{iScale}), halfSet, convergenceFlag); - coeffs{iScale} = S{iScale} * V{iScale}' + coeffs{iScale} = S{iScale} * V{iScale}'; + size(coeffs{iScale}) % Can be GB-TB if calculated full %varianceMap{iScale} = (U{iScale}*S{iScale}.^2*V{iScale} ./ numel(U{iScale}-1)); diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index e2402d8f..271e34b8 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -557,7 +557,7 @@ 'operation []\n',... ' SwitchCurrentCycle, UpdateTilts, WriteCsv, RemoveClasses,\n'... ' ShiftAll, ShiftBin, ListTomos, RemoveTomos,\n',... - ' ListPercentiles, RemoveFraction, RandomizeEulers\n',... + ' ListPercentiles, RemoveFraction, RemoveIgnoredParticles, RandomizeEulers\n',... 'vectOP [0,0,0]\n',... 'STD, EVE, ODD\n']); else diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index b88c6577..cacd5592 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -44,8 +44,8 @@ outName="$(basename ${mFile} .m)${post}" # bugs line. e.g. buggs=5testingFeature major=1 minor=8 -bugs=0 -nightly=8 +bugs=1 +nightly=0 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 9be83025542cd25e11f19ba0011e63b1469fa5eb Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 30 Sep 2024 19:44:00 -0400 Subject: [PATCH 083/151] Revert pca to single threaded as the results are not correct, and the acceleration is ~10-15% which is not worth trouble shooting right now. --- statistics/BH_pcaPub.m | 612 ++++++++++++++++++++--------------------- 1 file changed, 298 insertions(+), 314 deletions(-) diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index a711e310..a425bdab 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -169,6 +169,7 @@ outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); +%%%flgGold = emc.('flgGoldStandard'); flgNorm = 1;% emc.('flgNormalizeWMDs'); @@ -198,6 +199,16 @@ +% Removed flgGold everywhere else, but keep ability to classify full data set at +% the end (after all alignment is finished.) + + +if (emc.classification) + flgGold = 0; +else + flgGold = 1; +end + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); mapBackIter = subTomoMeta.currentTomoCPR; @@ -227,7 +238,6 @@ [ useGPU ] = BH_multi_checkGPU( -1 ); - gDev = gpuDevice(useGPU); @@ -249,11 +259,11 @@ cpuVols = struct; [ preSizeWindow, preSizeCalc, preSizeMask, prePadWindow, prePadCalc ] = ... - BH_multi_validArea(preMaskSize,preMaskRadius, emc.scale_calc_size ); + BH_multi_validArea(preMaskSize,preMaskRadius, emc.scale_calc_size ) [ sizeWindow, sizeCalc, sizeMask, padWindow, padCalc ] = ... - BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ); + BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) if (test_multi_ref_diffmap) @@ -265,10 +275,17 @@ % If emc.classification is negative combine the data for clustering, but don't set % any of the alignment changes to be persistant so that extracted class % averages are still independent half-sets. +if (flgGold) + oddRot = eye(3); +else + iRefPrev = 1; + + aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',iRefPrev)) + oddRot = reshape(aliParams(1,:),3,3)'; + % refine the translation per particle. -aliParams = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Resample%s%d','Ref',1)); -oddRot = reshape(aliParams(1,:),3,3)'; - + clear iRefPrev +end %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -277,15 +294,15 @@ refGroup = cell(2,1); classVector{1} = emc.('Raw_classes_odd')(1,:); classVector{2} = emc.('Raw_classes_eve')(1,:); -refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ]; -refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ]; +refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ] +refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ] for iGold = 1:2 % Sort low to high, because order is rearranged as such unstack refVectorFull{iGold} = sortrows(refVectorFull{iGold}', 1)'; % class id corresponding to membership in ???_refName - refVector{iGold} = refVectorFull{iGold}(1,:); + refVector{iGold} = refVectorFull{iGold}(1,:) % reference id, so multiple classes can be merged into one - refGroup{iGold} = refVectorFull{iGold}(3,:); + refGroup{iGold} = refVectorFull{iGold}(3,:) end @@ -338,35 +355,39 @@ end % IF combining for analysis, resample prior to any possible binning. -if (nReferences(1) ~= nReferences(2)) - error('When combining half sets, the number of references must match') -end -size(averageMotif) -if (nReferences(1) > 1) - for iRef = 1:nReferences(1) - averageMotif{1}{iRef} = averageMotif{2}{iRef} + ... - BH_resample3d(gather(averageMotif{1}{iRef}), ... +if ~(flgGold) + if (nReferences(1) ~= nReferences(2)) + error('When combining half sets, the number of references must match') + end + size(averageMotif) + if (nReferences(1) > 1) + for iRef = 1:nReferences(1) + averageMotif{1}{iRef} = averageMotif{2}{iRef} + ... + BH_resample3d(gather(averageMotif{1}{iRef}), ... + oddRot, ... + aliParams(2,1:3), ... + {'Bah',1,'spline'}, 'cpu', ... + 'forward'); + averageMotif{2}{iRef} = []; + end + else + averageMotif{1} = averageMotif{2} + ... + BH_resample3d(gather(averageMotif{1}), ... oddRot, ... aliParams(2,1:3), ... {'Bah',1,'spline'}, 'cpu', ... 'forward'); - averageMotif{2}{iRef} = []; + averageMotif{2} = []; end -else - averageMotif{1} = averageMotif{2} + ... - BH_resample3d(gather(averageMotif{1}), ... - oddRot, ... - aliParams(2,1:3), ... - {'Bah',1,'spline'}, 'cpu', ... - 'forward'); - averageMotif{2} = []; -end +end %%% incomplete, the idea is to generate an antialiased scaled volume for PCA if ( refSamplingRate ~= samplingRate ) fprintf('Resampling from %d refSampling to %d pcaSampling\n',refSamplingRate,samplingRate); - averageMotif{1} = BH_reScale3d(averageMotif{1},'',sprintf('%f',1/samplingRate),'GPU'); + for iGold = 1:1+flgGold + averageMotif{iGold} = BH_reScale3d(averageMotif{iGold},'',sprintf('%f',1/samplingRate),'GPU'); + end if (flgLoadMask) externalMask = BH_reScale3d(externalMask,'',sprintf('%f',1/samplingRate),'GPU'); @@ -378,29 +399,41 @@ prevVarianceMaps = struct(); if (flgVarianceMap) - halfSet = 'STD'; - - % For randomsubset (PREVIOUS_PCA = 0) the suffix is *_pcaPart.mat) but - % presumably we could have also just done full, so try that first - try - load(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet)) - catch - fprintf('\nDid not find, %s_%s_pcaFUll.mat, trying *_pcaPart.mat\n',outputPrefix,halfSet); - load(sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet)); - end - - % In most cases, this is the number of "features" specified in the - % parameter file, but in some data not even this may non-zero singluar - % values are found, so the number could be different (lower) - for iScale = 1:emc.n_scale_spaces - eigsFound = size(coeffs{iScale},1); - fname = sprintf('%s_varianceMap%d-%s-%d.mrc', ... - outputPrefix, eigsFound, halfSet, iScale); + for iGold = 1:1+flgGold + + if (flgGold) + if iGold == 1; + halfSet = 'ODD'; + else + halfSet = 'EVE'; + end + else + halfSet = 'STD'; + end - prevVarianceMaps.(sprintf('h%d',iGold)).(sprintf('s%d',iScale)) = ... - OPEN_IMG('single', fname).^flgStdDev; + % For randomsubset (PREVIOUS_PCA = 0) the suffix is *_pcaPart.mat) but + % presumably we could have also just done full, so try that first + + try + load(sprintf('%s_%s_pcaFull.mat',outputPrefix,halfSet)) + catch + fprintf('\nDid not find, %s_%s_pcaFUll.mat, trying *_pcaPart.mat\n',outputPrefix,halfSet); + load(sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet)); + end + + % In most cases, this is the number of "features" specified in the + % parameter file, but in some data not even this may non-zero singluar + % values are found, so the number could be different (lower) + for iScale = 1:emc.n_scale_spaces + eigsFound = size(coeffs{iScale},1); + fname = sprintf('%s_varianceMap%d-%s-%d.mrc', ... + outputPrefix, eigsFound, halfSet, iScale); + + prevVarianceMaps.(sprintf('h%d',iGold)).(sprintf('s%d',iScale)) = ... + OPEN_IMG('single', fname).^flgStdDev; + end + clear v coeffs eigsFound idxList end - clear v coeffs eigsFound idxList end @@ -423,7 +456,8 @@ end % when combining the addition is harmless, but is a convenient way to % include when sets are left 100% separate. - volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1}, pixelSize, ... + % volumeMask = volumeMask .* BH_mask3d(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, '',''); + volumeMask = volumeMask .* EMC_maskReference(averageMotif{1}+averageMotif{1+flgGold}, pixelSize, ... {'pca', true; 'lowpass', emc.shape_mask_lowpass; 'threshold', emc.shape_mask_threshold}); end @@ -442,25 +476,27 @@ volMask = struct(); nPixels = zeros(2,emc.n_scale_spaces); for iScale = 1:emc.n_scale_spaces - stHALF = sprintf('h%d',1); - stSCALE = sprintf('s%d',iScale); - if (flgVarianceMap) - if (test_multi_ref_diffmap) - error('test_multi_ref_diffmap is incompatible with flgVarianceMap') + for iGold = 1:1+flgGold + stHALF = sprintf('h%d',iGold); + stSCALE = sprintf('s%d',iScale); + if (flgVarianceMap) + if (test_multi_ref_diffmap) + error('test_multi_ref_diffmap is incompatible with flgVarianceMap') + end + volTMP = gather(volumeMask.*prevVarianceMaps.(stHALF).(stSCALE)); + else + volTMP = gather(volumeMask); end - volTMP = gather(volumeMask.*prevVarianceMaps.(stHALF).(stSCALE)); - else - volTMP = gather(volumeMask); + + masks.('volMask').(stHALF).(stSCALE) = (volTMP); + masks.('binary').(stHALF).(stSCALE) = (volTMP >= bh_global_binary_pcaMask_threshold); + masks.('binary').(stHALF).(stSCALE) = ... + masks.('binary').(stHALF).(stSCALE)(:); + masks.('binaryApply').(stHALF).(stSCALE) = (volTMP >= 0.01); + + nPixels(iGold,iScale) = gather(sum(masks.('binary').(stHALF).(stSCALE))); + clear volTMP stHALF stSCALE end - - masks.('volMask').(stHALF).(stSCALE) = (volTMP); - masks.('binary').(stHALF).(stSCALE) = (volTMP >= bh_global_binary_pcaMask_threshold); - masks.('binary').(stHALF).(stSCALE) = ... - masks.('binary').(stHALF).(stSCALE)(:); - masks.('binaryApply').(stHALF).(stSCALE) = (volTMP >= 0.01); - - nPixels(1,iScale) = gather(sum(masks.('binary').(stHALF).(stSCALE))); - clear volTMP stHALF stSCALE end clear volumeMask @@ -476,7 +512,7 @@ kernelSize = ceil(threeSigma(iScale).*3) + 3; kernelSize = kernelSize + (1-mod(kernelSize,2)); % masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); - masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'cpu', {}); + masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'gpu', {}); % SAVE_IMG( masks.('scaleMask').(sprintf('s%d',iScale)), ... @@ -486,45 +522,43 @@ end end -avgMotif_FT = cell(emc.n_scale_spaces,1); -avgFiltered = cell(emc.n_scale_spaces,1); -for iScale = 1:emc.n_scale_spaces - - if (test_multi_ref_diffmap) - tmp_avg = averageMotif{1}{iScale}; - else - tmp_avg = averageMotif{1}; - end - - - tmp_avg = tmp_avg - mean(tmp_avg(masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale)))); - tmp_avg = tmp_avg ./ rms(tmp_avg(masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale)))); - tmp_avg = tmp_avg .* masks.('volMask').(sprintf('h%d',1)).(sprintf('s%d',iScale)); - % FIXME: ideally we would do both, but for testing I am stealing scaleSpace for iRef - if ~(test_multi_ref_diffmap) - tmp_avg = EMC_convn(single(gpuArray(tmp_avg)) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); - end - avgMotif_FT{iScale} = ... - BH_bandLimitCenterNormalize(tmp_avg,... - BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... - masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale)),... - [0,0,0;0,0,0],'single'); - - avgFiltered{iScale} = real(ifftn(avgMotif_FT{iScale})); - - avgFiltered{iScale} = avgFiltered{iScale} - mean(avgFiltered{iScale}(masks.('binary').(sprintf('h%d',1)).(sprintf('s%d',iScale)))); - avgFiltered{iScale} = gather(avgFiltered{iScale} ./rms(avgFiltered{iScale}(masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale)))) .* ... - masks.('binaryApply').(sprintf('h%d',1)).(sprintf('s%d',iScale))); +avgMotif_FT = cell(1+flgGold,emc.n_scale_spaces); +avgFiltered = cell(1+flgGold,emc.n_scale_spaces); +% Here always read in both, combine if flgGold = 0 +for iGold = 1:1+flgGold + for iScale = 1:emc.n_scale_spaces + + if (test_multi_ref_diffmap) + tmp_avg = averageMotif{iGold}{iScale}; + else + tmp_avg = averageMotif{iGold}; + end + + + tmp_avg = tmp_avg - mean(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); + tmp_avg = tmp_avg ./ rms(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); + tmp_avg = tmp_avg .* masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)); + % FIXME: ideally we would do both, but for testing I am stealing scaleSpace for iRef + if ~(test_multi_ref_diffmap) + tmp_avg = EMC_convn(single(gpuArray(tmp_avg)) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); + end + avgMotif_FT{iGold, iScale} = ... + BH_bandLimitCenterNormalize(tmp_avg,... + BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... + masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)),... + [0,0,0;0,0,0],'single'); - % Gather these so they are not destroyed when assigning gpuDevice - avgMotif_FT{iScale} = gather(avgMotif_FT{iScale}); - avgFiltered{iScale} = gather(avgFiltered{iScale}); + avgFiltered{iGold, iScale} = real(ifftn(avgMotif_FT{iGold, iScale})); + + avgFiltered{iGold, iScale} = avgFiltered{iGold, iScale} - mean(avgFiltered{iGold, iScale}(masks.('binary').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); + avgFiltered{iGold, iScale} = gather(avgFiltered{iGold, iScale} ./rms(avgFiltered{iGold, iScale}(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))) .* ... + masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale))); + end end - -montOUT = BH_montage4d(avgFiltered(:),''); +montOUT = BH_montage4d(avgFiltered(1,:),''); SAVE_IMG(MRCImage(montOUT), sprintf('%s_filt.mrc', outputPrefix),pixelSize); clear montOUT @@ -532,12 +566,23 @@ % If emc.Pca_randSubset is string with a previous matfile use this, without any % decomposition. -for iGold = 1 +for iGold = 1:1+flgGold flgRefIsPadded = 0; - - stHALF = sprintf('h%d',iGold); - halfSet = 'STD'; - randSet = [1,2]; + if (flgGold) + if iGold == 1; + halfSet = 'ODD'; + stHALF = sprintf('h%d',iGold); + randSet =1; + else + stHALF = sprintf('h%d',iGold); + halfSet = 'EVE'; + randSet = 2; + end + else + stHALF = sprintf('h%d',iGold); + halfSet = 'STD'; + randSet = [1,2]; + end if (PREVIOUS_PCA) previousPCA = sprintf('%s_%s_pcaPart.mat',outputPrefix,halfSet); @@ -567,19 +612,17 @@ % Initialize array in main memory for pca clear dataMatrix tempDataMatrix - dataMatrix = cell(emc.n_scale_spaces,1); - % tempDataMatrix = cell(3,1); + dataMatrix = cell(3,1); + tempDataMatrix = cell(3,1); for iScale = 1:emc.n_scale_spaces dataMatrix{iScale} = zeros(nPixels(iGold,iScale), nSUBSET, 'single'); - % FIXME: if reverting parfor, this will be needed, for now, giving each worker its own copy. - % tempDataMatrix{iScale} = zeros(nPixels(iGold,iScale), nParticlesOnGpuBeforeTransfer, 'single', 'gpuArray'); + tempDataMatrix{iScale} = zeros(nPixels(iGold,iScale), nParticlesOnGpuBeforeTransfer, 'single', 'gpuArray'); end % Pull masks onto GPU (which are cleared along with everything else when % the device is reset at the end of each loop.) gpuMasks = struct(); - % FIXME: these gpumasks are no longer needed for iScale = 1:emc.n_scale_spaces stSCALE = sprintf('s%d',iScale); @@ -592,11 +635,10 @@ if ~(test_multi_ref_diffmap) gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); end - - masks.('highPass').(stHALF).(stSCALE) = gather(BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize)); + + + gpuMasks.('highPass').(stSCALE) = BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize); end - - % % % for iGold_inner = 1:1+flgGold % % % for iScale = 1:emc.n_scale_spaces @@ -606,78 +648,27 @@ % % % end - nExtracted = cell(emc.n_scale_spaces,1); - nTemp = cell(emc.n_scale_spaces,1); - nTempPrev = cell(emc.n_scale_spaces,1); - idxList = cell(emc.n_scale_spaces,1); - peakList = cell(emc.n_scale_spaces,1); - is_validData = cell(emc.n_scale_spaces,1); - tomoIDXList = cell(emc.n_scale_spaces,1); - reconCoords = cell(emc.n_scale_spaces,1); - TLT = cell(emc.n_scale_spaces,1); - geom = cell(emc.n_scale_spaces,1); - - for iScale = 1:emc.n_scale_spaces - nExtracted{iScale} = 1; - nTemp{iScale} = 1; - nTempPrev{iScale} = 0; - idxList{iScale} = zeros(1,nSUBSET); - tomoIDXList{iScale} = zeros(1,nSUBSET); - is_validData{iScale} = zeros(1,nSUBSET); - peakList{iScale} = zeros(1,nSUBSET); - reconCoords{iScale} = subTomoMeta.mapBackGeometry.tomoCoords; - geom{iScale} = subTomoMeta.(cycleNumber).(geom_name); - TLT{iScale} = subTomoMeta.('tiltGeometry'); - - end - % nExtracted = 1; - % nTemp = 1; - % nTempPrev = 0; - % idxList = zeros(1,nSUBSET); - % peakList = zeros(1,nSUBSET); + nExtracted = 1; + nTemp = 1; + nTempPrev = 0; + idxList = zeros(1,nSUBSET); + peakList = zeros(1,nSUBSET); firstLoop = true; sI = 1; - - - % FIXME: hardcoded for initial testing - nWorkers = 4+1; - if ~isempty(gcp('nocreate')) - delete(gcp('nocreate')) - EMC_parpool(nWorkers); - else - EMC_parpool(nWorkers); - end - fprintf('init with %d workers\n',nWorkers); - - parVect = 1:emc.n_scale_spaces - nGPUs = emc.nGPUs; - parfor iScaleSpace = parVect - % for iScaleSpace = 1:emc.n_scale_spaces - - gpuIDXList = mod(parVect + nGPUs, nGPUs)+1; - iGPUidx = gpuIDXList(iScaleSpace); - gpuDevice(iGPUidx); - % FIXME: need to check gpu idx that are available as in other parallel code - fprintf('parProc %d/%d assigned to GPU %d\n',iScaleSpace,nWorkers,iScaleSpace); - - volMask = gpuArray(masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScaleSpace))); - binaryMask = gpuArray(masks.('binary').(sprintf('h%d',iGold)).(sprintf('s%d',iScaleSpace))); - if ~(test_multi_ref_diffmap) - scaleMask = gpuArray(masks.('scaleMask').(sprintf('s%d',iScaleSpace))); - end - highPass = gpuArray(masks.('highPass').(sprintf('h%d',iGold)).(sprintf('s%d',iScaleSpace))); - - d_avgMotif_FT = gpuArray(avgMotif_FT{iScaleSpace}); - - % Initialize the tempDataMatrix - tempDataMatrix = zeros(nPixels(iGold,iScaleSpace), nParticlesOnGpuBeforeTransfer, 'single', 'gpuArray'); - + nIgnored = 0; for iTomo = 1:nTomograms + tomoName = tomoList{iTomo}; iGPU = 1; + tomoIdx = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tomoIdx; + tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; + reconCoords = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iTomo}); + TLT = subTomoMeta.('tiltGeometry').(tomoList{iTomo}); + + if (emc.flgCutOutVolumes) volumeData = []; else @@ -691,35 +682,51 @@ end - fprintf('Working on %d/%d volumes %s, iScaleSpace %d\n',iTomo,nTomograms,tomoName, iScaleSpace); + iTiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoName).tiltName; + + tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); + + fprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); - + % Loop over peaks inside each tomo to limit wedge mask xfer positionList = positionList(positionList(:,26) ~= -9999,:); nSubTomos = size(positionList,1); + + + % reset for each tomogram wdgIDX = 0; radialMask = ''; if (flgNorm) - [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(size(d_avgMotif_FT),'Cartesian',... + + + % bins = 1./[1000,800,600,400,300,200,150,100,80,60,50,40,35,30,28,26,24,22,20,18,16,14,12,10,8,6,4,2]; + % bins = [0, bins]; + + [radialGrid,~,~,~,~,~] = BH_multi_gridCoordinates(size(avgMotif_FT{iGold, iScale}),'Cartesian',... 'GPU',{'none'},1,0,1); - - bins = radialGrid(1:floor(size(d_avgMotif_FT,1)/2),1,1); + + bins = radialGrid(1:floor(size(avgMotif_FT{iGold, iScale},1)/2),1,1); bins = bins(bins < 0.5); - + radialMask = cell(length(bins)-1,1); - + for iBin = 1:length(bins)-1 radialMask{iBin} = find(radialGrid >= bins(iBin) & radialGrid < bins(iBin+1)); end - + radialGrid = ''; + end - wdgBP = ifftshift(highPass); + wdgBP = ifftshift(gpuMasks.('highPass').(sprintf('s%d',iScale))); for iSubTomo = 1:nSubTomos + %%%%% %%%%% + + % Check that the given subTomo is not to be ignored - for now, treat % all peaks as included. The assumption is that using this will be % for initializing the project to get a good starting model. "True" @@ -727,6 +734,7 @@ % subset of peaks. FIXME includeParticle = positionList(iSubTomo, 8); particleIDX = positionList(iSubTomo, 4); % Same for all peaks + iPeak=0; % make sure this exists if we are no including the particle if (includeParticle) make_sf3d = true; for iPeak = 0:emc.nPeaks-1 @@ -742,7 +750,7 @@ make_sf3d = false; radialGrid = ''; padWdg = [0,0,0;0,0,0]; - [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, TLT{iScaleSpace}.(tomoList{iTomo}), center, reconCoords{iScaleSpace}.(tomoList{iTomo}), wiener_constant); + [ wedgeMask ] = BH_weightMaskMex(sizeWindow, samplingRate, TLT, center, reconCoords, wiener_constant); end % If flgGold there is no change, otherwise temporarily resample the @@ -763,14 +771,12 @@ BH_isWindowValid([volHeader.nX,volHeader.nY,volHeader.nZ], ... sizeWindow, maskRadius, center); end - - % Skipping particles has created friction several times over the years, so I'm going to make this an error for now. - if ischar(indVAL) - error("invalid windowing, is your mask larger than the one used in emClarity avg? indVAL"); - end - shiftVAL = shiftVAL + aliParams(2,1:3)./samplingRate; + if ~(flgGold) + shiftVAL = shiftVAL + aliParams(2,1:3)./samplingRate; + end + using_this_subtomo = true; if ~ischar(indVAL) % Read in and interpolate at single precision as the local values % in the interpolant suffer from any significant round off errors. @@ -823,66 +829,94 @@ padWindow(1,2)+1 : end - padWindow(2,2), ... padWindow(1,3)+1 : end - padWindow(2,3)); - if (test_multi_ref_diffmap) - iPrt = iTrimParticle; - else - iPrt = EMC_convn(iTrimParticle , scaleMask); - end - - iPrt = BH_bandLimitCenterNormalize( ... - iPrt .* ... - volMask,... - highPass,... - binaryMask,... - [0,0,0;0,0,0],'single'); - [iWmd,~] = BH_diffMap(d_avgMotif_FT,iPrt,ifftshift(iWedge),... - flgNorm,pixelSize,radialMask, padWdg); - - - if (all(isfinite(iWmd(binaryMask)))) - tempDataMatrix(:,nTemp{iScaleSpace}) = single(iWmd(binaryMask)); - - is_validData{iScaleSpace}(1, nExtracted{iScaleSpace}) = 1; - + for iScale = 1:emc.n_scale_spaces + if (test_multi_ref_diffmap) + iPrt = iTrimParticle; + else + iPrt = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); + end + + iPrt = BH_bandLimitCenterNormalize( ... + iPrt .* ... + gpuMasks.('volMask').(sprintf('s%d',iScale)), ... + gpuMasks.('highPass').(sprintf('s%d',iScale)),... + gpuMasks.('binary').(sprintf('s%d',iScale)),... + [0,0,0;0,0,0],'single'); + + [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iPrt,ifftshift(iWedge),... + flgNorm,pixelSize,radialMask, padWdg); + + + % using_this_subtomo is set true for each particle, but if we are not finite for ANY scale space, we want to skip this particle for all + % scale spaces. + if all(isfinite(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale))))) + tempDataMatrix{iScale}(:,nTemp) = single(iWmd(gpuMasks.('binary').(sprintf('s%d',iScale)))); + else + fprintf('inf or nan in subtomo %d scalePace %d',particleIDX, iScale); + using_this_subtomo = false; + end + + end % loop on scale spaces + clear iAvg iWmd iTrimParticle + + + + if (using_this_subtomo) + idxList(1, nExtracted) = particleIDX; + peakList(1,nExtracted) = iPeak+1; + nExtracted = nExtracted + 1; + nTemp = nTemp + 1; % pull data of the gpu every 1000 particls (adjust this to max mem) - if ( nTemp{iScaleSpace} == nParticlesOnGpuBeforeTransfer ) - dataMatrix{iScaleSpace}(:,1+nTempPrev{iScaleSpace}:nTemp{iScaleSpace}+nTempPrev{iScaleSpace}) = gather(tempDataMatrix(:,1:nTemp{iScaleSpace})); + if nTemp - 1 == nParticlesOnGpuBeforeTransfer + for iScale = 1:emc.n_scale_spaces + dataMatrix{iScale}(:,1+nTempPrev:nTemp+nTempPrev-1) = ... + gather(tempDataMatrix{iScale}(:,1:nTemp-1)); + end - nTempPrev{iScaleSpace} = nTempPrev{iScaleSpace} + nTemp{iScaleSpace} ; - nTemp{iScaleSpace} = 1; + nTempPrev = nTempPrev + nTemp - 1; + nTemp = 1; end - end % end of update data matrix or if ignoring due to non-finite processing post diffmap - update metaData - - iWmd = []; - iTrimParticle = []; - - end % end of if condition on valid windowing - idxList{iScaleSpace}(1, nExtracted{iScaleSpace}) = particleIDX; - peakList{iScaleSpace}(1,nExtracted{iScaleSpace}) = iPeak + 1; - nExtracted{iScaleSpace} = nExtracted{iScaleSpace} + 1; - nTemp{iScaleSpace} = nTemp{iScaleSpace} + 1; - tomoIDXList{iScaleSpace}(1,nExtracted{iScaleSpace}) = iTomo; + else + nIgnored = nIgnored + 1; + fprintf('Ignoring subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; + subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; + end + + + else + nIgnored = nIgnored + 1; + fprintf('Ignoring inside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; + subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; + + end % end of update data matrix or if ignoring update metaData + end % end of loop over peaks - - end % end of ignore for partial data set - - if ~rem(iSubTomo,100) - if (iScaleSpace == 1) - fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... - iSubTomo, nSubTomos, iPeak+1, emc.nPeaks, iTomo, nTomograms); - fprintf('Total nExtracted = %d\n', nExtracted{iScaleSpace}-1); + else + nIgnored = nIgnored + 1; + if ~(emc.Pca_randSubset) + fprintf('Ignoring outside subtomo %d from %s\n',particleIDX, tomoList{iTomo}); + eraseIDX = subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(:,4) == particleIDX; + subTomoMeta.(cycleNumber).(geom_name).(tomoList{iTomo})(eraseIDX, 26+iPeak*26) = -9999; end + end % end of ignore if statment from extracted window out of bounds (ischar(indVAL)) + if ~rem(iSubTomo,100) + fprintf('\nworking on %d/%d subTomo peak %d/%d from %d/%d Tomo\n', ... + iSubTomo, nSubTomos,iPeak+1,emc.nPeaks, iTomo,nTomograms); + + fprintf('Total nExtracted = %d\n', nExtracted-1); + fprintf('Total nIgnored = %d\n', nIgnored); + end end % end of the loop over subTomos - volumeData = []; + clear volumeData end % end of the loop over Tomograms, - dataMatrix{iScaleSpace}(:,1+nTempPrev{iScaleSpace}:nTemp{iScaleSpace}-1+nTempPrev{iScaleSpace}) = gather(tempDataMatrix(:,1:nTemp{iScaleSpace}-1)); - end % end of the parallel loop over scale spaces % % % volBinaryMask = reshape(gather(volBinaryMask),sizeMask); for iScale = 1:emc.n_scale_spaces @@ -891,81 +925,34 @@ end - % Get a count of all the valid particles - note if one peak is being ignored, this is not valid FIXME! - is_validData_bool = true(1,nSUBSET); - for iScale = 1:emc.n_scale_spaces - is_validData_bool = is_validData_bool & is_validData{iScale}; - end - - n_validData = sum(is_validData_bool) - - % Sanity check that all the data idx are the same - for iScale = 1:emc.n_scale_spaces - idxList{iScale} = idxList{iScale} .* is_validData_bool; - peakList{iScale} = peakList{iScale} .* is_validData_bool; - if iScale > 1 - if any(idxList{iScale} ~= idxList{1}) - idxList{1} - idxList{iScale} - error('Error: found a difference in valid data idxList between scale spaces'); - end - end - end - - idxList = idxList{1}; - peakList = peakList{1}; - - % Remove any columns corresponding to an ignored particle and also update the geometry - clean_geometry = subTomoMeta.(cycleNumber).(geom_name); - + subTomoMeta.(cycleNumber).('newIgnored_PCA').(halfSet) = gather(nIgnored); + + subTomoMeta = subTomoMeta; + save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); + for iScale = 1:emc.n_scale_spaces - fprintf('Cleaning geometry for scale space %d\n',iScale); - fprintf('nPixels x n_validData = %d x %d\n',nPixels(1,iScale),n_validData); - tempDataMatrix = zeros(nPixels(1,iScale), n_validData, 'single'); - tempDataMatrix(:,1:n_validData) = dataMatrix{iScale}(:,is_validData_bool); - dataMatrix{iScale} = []; - dataMatrix{iScale} = tempDataMatrix; + dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... + gather(tempDataMatrix{iScale}(:,1:nTemp-1)); end - dataMatrix - tempDataMatrix = []; - - % Remove empty columns from the geometry - for iErase = 1:length(idxList) - if ~(is_validData_bool(1,iErase)) - tomo_name = tomoList{tomoIDXList(1,iErase)}; - eraseIDX = clean_geometry.(tomo_name)(:,4) == idxList(1,iErase); - if (sum(eraseIDX) > emc.nPeaks || sum(eraseIDX) == 0) - error('Error: found %d particles with the same index %d',sum(eraseIDX),idxList{iScale}(1,iErase)); - end - clean_geometry.(tomo_name)(eraseIDX(1), 26:26:26*emc.nPeaks) = -9999; - end - end - - % Now update the subtomo meta and save it - subTomoMeta.(cycleNumber).(geom_name) = clean_geometry; - save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); - - % Now shrink the idxList and peakList - idxList = idxList(1,is_validData_bool); - peakList = peakList(1,is_validData_bool); - - % FIXME: delete if removing the parallel loop - % for iScale = 1:emc.n_scale_spaces - % dataMatrix{iScale}(:,1+nTempPrev:nTemp-1+nTempPrev) = ... - % gather(tempDataMatrix{iScale}(:,1:nTemp-1)); - % end - - + clear tempDataMatrix + % Get rid of any zero vals from newly ignored particles which are there due to + % pre-allocation. Assuming no zeros have found their way in anywhere else which + % would be a major problem. + cleanIDX = (idxList~=0)'; + idxList = idxList(cleanIDX); + peakList = peakList(cleanIDX); for iScale = 1:emc.n_scale_spaces + dataMatrix{iScale} = dataMatrix{iScale}(:,1:size(idxList,2)); % Center the rows for row = 1:size(dataMatrix{iScale},1) dataMatrix{iScale}(row,:) = dataMatrix{iScale}(row,:) - mean(double(dataMatrix{iScale}(row,:))); end end - dataMatrix + + %save('preparpoolSave.mat'); try @@ -1012,9 +999,7 @@ U{iScale} = single(U{iScale}); S{iScale} = single(S{iScale}); V{iScale} = single(V{iScale}); - size(U{iScale}) - size(S{iScale}) - size(V{iScale}) + % [U{iScale},S{iScale},V{iScale}] = svd(dataMatrix{iScale}, 0); sDiag{iScale} = diag(S{iScale}); @@ -1025,8 +1010,7 @@ fprintf('Found %d / %d non-zero eigenvalues sum = %4.4f, in set %s.\n All singular values converged is t/f ( %d ) ', ... numNonZero, size(S{iScale}, 1), sum(sDiag{iScale}), halfSet, convergenceFlag); - coeffs{iScale} = S{iScale} * V{iScale}'; - size(coeffs{iScale}) + coeffs{iScale} = S{iScale} * V{iScale}' % Can be GB-TB if calculated full %varianceMap{iScale} = (U{iScale}*S{iScale}.^2*V{iScale} ./ numel(U{iScale}-1)); @@ -1062,7 +1046,7 @@ eigenImage = eigenImage - mean(eigenImage(masks.('binaryApply').(stHALF).(sprintf('s%d',iScale)))); eigenImage = eigenImage ./rms(eigenImage(masks.('binaryApply').(stHALF).(sprintf('s%d',iScale)))).* masks.('binary').(stHALF).(sprintf('s%d',iScale)) ; eigList{iEig,1} = gather(eigenImage); - eigList_SUM{iEig,1} = gather((eigenImage + avgFiltered{iScale} )./2); + eigList_SUM{iEig,1} = gather((eigenImage + avgFiltered{iGold, iScale} )./2); end From 3bb5c581114c993d4ffd354eb30ba5b40435db87 Mon Sep 17 00:00:00 2001 From: himesb Date: Sat, 5 Oct 2024 11:52:09 -0400 Subject: [PATCH 084/151] Fix classIdx -> iRef in alignRaw --- alignment/BH_alignRaw3d_v2.m | 18 ++++++++++++++++-- testScripts/mCompile.sh | 2 +- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 355b3890..7eecafec 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -210,6 +210,7 @@ refWGT = cell(2,1); refWgtROT = cell(2,1); imgCounts = cell(2,1); +ref_to_class_idx = cell(2,1); for iGold = 1:2 if iGold == 1 @@ -224,7 +225,8 @@ weightNAME = sprintf('class_%d_Locations_Ref_%s_Wgt', refName, halfSet); imgCounts{iGold} = subTomoMeta.(cycleNumber).(imgNAME){3}; - + % Get the class index for each reference + ref_to_class_idx{iGold} = subTomoMeta.(cycleNumber).(imgNAME){3}(1,:); [ refTMP ] = BH_unStackMontage4d(1:nReferences(iGold), ... subTomoMeta.(cycleNumber).(imgNAME){1}, ... @@ -567,6 +569,8 @@ particle_symmetry = 'C1'; end parfor iParProc = parVect + % for iParProc = parVect + symmetry = emc.symmetry; bestAngles_tmp = struct(); @@ -957,9 +961,19 @@ case 0 refToAlign = 1; case 1 + % Align this particle against all possible refs refToAlign = 1:max(nReferences(:)); case 2 - refToAlign = classIDX; + ref_to_class_idx_value = find(ref_to_class_idx{iGold} == classIDX); + if isempty(ref_to_class_idx_value) + error("No reference class found for classIDX %d", classIDX); + else + if length(ref_to_class_idx_value) > 1 + error("Multiple reference classes found for classIDX %d", classIDX); + end + end + + refToAlign = ref_to_class_idx_value; otherwise error('emc.multi_reference_alignment is not 0,1,2') end diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index cacd5592..26bbaa91 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=1 -nightly=0 +nightly=1 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 501e1991ad13cc91b102794967797b95b5f66a9b Mon Sep 17 00:00:00 2001 From: himesb Date: Mon, 20 Jan 2025 10:27:56 -0500 Subject: [PATCH 085/151] wip: making eigenvolumes more useful when symmetry contraints are applied by re-symmetrizing the outputs for visualization. Note: When reviewing, it looks like the options for PCA masking using a previous variance map has never actually applied the mask. Fixed that logical condition, but it is untested. --- alignment/BH_templateSearch3d_2.m | 27 ++----- masking/BH_mask3d.m | 19 +++-- metaData/BH_geometryAnalysis.m | 11 +++ metaData/BH_parseParameterFile.m | 12 ++++ statistics/BH_pcaPub.m | 116 +++++++++++++++++++----------- synthetic/BH_to_cisTEM_mapBack.m | 9 ++- testScripts/emClarity.m | 11 ++- testScripts/interpolator.m | 21 +++++- testScripts/mCompile.sh | 4 +- 9 files changed, 152 insertions(+), 78 deletions(-) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index be97c19a..47063ced 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -111,30 +111,11 @@ % For testing print_warning=false; -try - wantedCut = emc.('lowResCut'); - fprintf('lowResCut is deprecated and will be removed in future versions.\n') - fprintf('please switch to Tmp_bandpass\n\n'); - bp_vals = [1e-3,600,wantedCut]; - print_warning = true; -catch - bp_vals = [1e-3,600,28]; -end -try - bp_vals = emc.('Tmp_bandpass'); - if numel(bp_vals) ~= 3 - error('Tmp_bandpass is [filter at zero freq, res high-pass cutoff, res low-pass cutoff]'); - end - if print_warning - fprintf('WARNING, you specified lowResCut (deprecated) and Tmp_bandpass!\n'); - end - fprintf('You specified a bandpass with values [%2.2e,%3.2f,%3.2f]\n',bp_vals); -catch - bp_vals = [1e-3,600,28]; - fprintf('Using default bandpass with values [%2.2e,%3.2f,%3.2f]\n',bp_vals); - -end + + +bp_vals = emc.('Tmp_bandpass'); + try stats_diameter_fraction = emc.('diameter_fraction_for_local_stats') catch diff --git a/masking/BH_mask3d.m b/masking/BH_mask3d.m index 651ad775..9ff14a8f 100755 --- a/masking/BH_mask3d.m +++ b/masking/BH_mask3d.m @@ -207,14 +207,21 @@ 0, 1, 0 ); end - sectorMax = 2*pi/asymmetricRestriction * 1.025; - angles = (angles > (2*pi-sectorMax/2) | angles < sectorMax/2); + taper_kernel = EMC_gaussianKernel([1,9], 3, 'gpu', {}); + dilate_kernel = EMC_gaussianKernel([1,5], 3, 'gpu', {}); + sectorMax = 2*pi/asymmetricRestriction; + + % Dilate the selection a bit. It is better to have some of the neighboring asym unit than to have some of our unit missing. + angles = (angles >= (2*pi-sectorMax/2) | angles <= sectorMax/2); + angles = EMC_convn(single(angles), dilate_kernel); + angles = angles > 0.0; % % % gc = BH_multi_gaussian3d(-1.*size(angles),1.5); % % % mWindow = real(ifftn(fftn(angles.*fullMask).*gc)); - KERNEL = EMC_gaussianKernel([1,5], 1.5, 'gpu', {}); - mWindow = EMC_convn(single(angles.*fullMask), KERNEL); - clear KERNEL - mWindow = mWindow ./ max(angles(:)); + + mWindow = EMC_convn(single(angles.*fullMask), taper_kernel); + + clear KERNEL taper_kernel dilate_kernel + mWindow = mWindow ./ max(mWindow(:)); else diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 7c33afa4..c16ac758 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -268,6 +268,17 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... clear blankClassMont % montage is always square, and the first row is always full, so use this to % determine the size. + + % if it is raw alignment, assume we are still deleting classes - hacked in to test, note cycle number is bogus + % if strcmpi(STAGEofALIGNMENT, 'RawAlignment') + % imgNAME = sprintf('class_%d_Locations_%s_%s_NoWgt', className, 'Cls', halfSet); + % imgNAMEStd = sprintf('class_%d_Locations_%s_%s_NoWgt', className, 'Cls', 'ODD'); + % try + % locations= masterTM.('cycle005').(imgNAME){2}; + % catch + % locations= masterTM.('cycle005').(imgNAMEStd){2}; + % end + % end sizeMontage = max(locations{end}(2),locations{end}(4)) blankClassMont(sizeMontage, sizeMontage) = single(0); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 801ea26b..16b01f99 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -641,4 +641,16 @@ EMC_assert_numeric(emc.tmp_scan, 3, [-1, 1]); else emc.tmp_scan = [1,1,0]; +end + +if isfield(emc, 'Tmp_bandpass') + EMC_assert_numeric(emc.Tmp_bandpass, 3); +else + emc.Tmp_bandpass = [0.001, 1200, 28]; +end + +if isfield(emc, 'Pca_bandpass') + EMC_assert_numeric(emc.Pca_bandpass, 3); +else + emc.Pca_bandpass = [0.001, 1200, 28]; end \ No newline at end of file diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index a425bdab..f5f3ba34 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -104,7 +104,7 @@ global bh_global_binary_pcaMask_threshold; -use_new_interpolator = true; + % Previous_pca has two functions, when true and > 0 use the decomposition % calculated from a random subset of the data to project the full data set % onto each of the selected principle components. when true and < 0 run @@ -348,6 +348,8 @@ if ~(test_multi_ref_diffmap) averageMotif{iGold} = averageMotif{iGold}{1}; end + + % The same external mask is applied to all references, so only load once. if (flgLoadMask) && (iGold == 1) fprintf('\n\nLoading external mask\n'); externalMask = OPEN_IMG('single',sprintf('%s-pcaMask',subTomoMeta.(cycleNumber).(imgNAME){1})); @@ -437,20 +439,26 @@ end -if (PREVIOUS_PCA) +if (PREVIOUS_PCA == 1) volumeMask = gpuArray(OPEN_IMG('single', sprintf('%s_pcaVolMask.mrc',outputPrefix))); + if (emc.Pca_constrain_symmetry) + volumeMask_symmetryConstraint = gpuArray(OPEN_IMG('single', sprintf('%s_pcaVolMask_symmetryConstraint.mrc',outputPrefix))); + end else + [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); + + % Saving the symmetry mask separately to we can reconstitute the variance maps and eigen images for better + % visual interp. if (emc.Pca_constrain_symmetry) gridSearch = eulerSearch(emc.symmetry,180,5,360,5,0.0,1,true); - [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter, ... + [ volumeMask_symmetryConstraint ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter, ... '3d', gridSearch.number_of_asymmetric_units); - else - [ volumeMask ] = BH_mask3d(maskType, sizeMask, maskRadius, maskCenter); + volumeMask = volumeMask .* volumeMask_symmetryConstraint; end if ( flgPcaShapeMask ) - % For testing we won't handle this block + % For testing we won't handle this block as it would require a volume mask for every reference (which is fine, but not implemented) if (test_multi_ref_diffmap) error('test_multi_ref_diffmap is incompatible with flgPcaShapeMask') end @@ -463,32 +471,31 @@ end if (flgLoadMask) - % For testing we won't handle this block - if (test_multi_ref_diffmap) - error('test_multi_ref_diffmap is incompatible with flgLoadMask') - end volumeMask = volumeMask .* externalMask; end - SAVE_IMG(MRCImage(gather(volumeMask)),sprintf('%s_pcaVolMask.mrc',outputPrefix),pixelSize); + SAVE_IMG(volumeMask,sprintf('%s_pcaVolMask.mrc',outputPrefix),pixelSize); + if (emc.Pca_constrain_symmetry) + SAVE_IMG(volumeMask_symmetryConstraint,sprintf('%s_pcaVolMask_symmetryConstraint.mrc',outputPrefix),pixelSize); + end end volMask = struct(); nPixels = zeros(2,emc.n_scale_spaces); +if (emc.Pca_constrain_symmetry) + masks.('volMask_symmetryConstraint') = gather(volumeMask_symmetryConstraint); +end for iScale = 1:emc.n_scale_spaces for iGold = 1:1+flgGold stHALF = sprintf('h%d',iGold); stSCALE = sprintf('s%d',iScale); if (flgVarianceMap) - if (test_multi_ref_diffmap) - error('test_multi_ref_diffmap is incompatible with flgVarianceMap') - end volTMP = gather(volumeMask.*prevVarianceMaps.(stHALF).(stSCALE)); else volTMP = gather(volumeMask); end - masks.('volMask').(stHALF).(stSCALE) = (volTMP); + masks.('binary').(stHALF).(stSCALE) = (volTMP >= bh_global_binary_pcaMask_threshold); masks.('binary').(stHALF).(stSCALE) = ... masks.('binary').(stHALF).(stSCALE)(:); @@ -559,7 +566,7 @@ montOUT = BH_montage4d(avgFiltered(1,:),''); -SAVE_IMG(MRCImage(montOUT), sprintf('%s_filt.mrc', outputPrefix),pixelSize); +SAVE_IMG(montOUT, sprintf('%s_filt.mrc', outputPrefix),pixelSize); clear montOUT @@ -636,8 +643,8 @@ gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); end - - gpuMasks.('highPass').(stSCALE) = BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize); + bp_vals = emc.Pca_bandpass; + gpuMasks.('highPass').(stSCALE) = BH_bandpass3d(sizeMask,bp_vals(1), bp_vals(2), bp_vals(3),'GPU',pixelSize); end % % % for iGold_inner = 1:1+flgGold @@ -804,24 +811,15 @@ padVAL(2,1:3), 'GPU', 'single'); end - if ( use_new_interpolator ) - % Pulling in the newer interpolater from alignRaw3d_v2. There, I instantiate a new interpolator every subtomo, but it is generally - % being used many times, over the angle loop. It may be more efficient to do this outside the for subtomo loop here, but - % to start, just do it the same way. - use_only_once = true; - [ ~, iParticle ] = interpolator(gpuArray(iParticle),angles, shiftVAL, 'Bah', 'inv', emc.symmetry, use_only_once); - - [ ~, iWedge ] = interpolator(gpuArray(wedgeMask),angles,[0,0,0], 'Bah', 'inv', emc.symmetry, use_only_once); - - else - % Transform the particle, and then trim to motif size - - [ iParticle ] = BH_resample3d(iParticle, angles, shiftVAL, ... - 'Bah', 'GPU', 'inv'); + % Pulling in the newer interpolater from alignRaw3d_v2. There, I instantiate a new interpolator every subtomo, but it is generally + % being used many times, over the angle loop. It may be more efficient to do this outside the for subtomo loop here, but + % to start, just do it the same way. + use_only_once = true; + [ ~, iParticle ] = interpolator(gpuArray(iParticle),angles, shiftVAL, 'Bah', 'inv', emc.symmetry, use_only_once); + + [ ~, iWedge ] = interpolator(gpuArray(wedgeMask),angles,[0,0,0], 'Bah', 'inv', emc.symmetry, use_only_once); - [ iWedge ] = BH_resample3d(wedgeMask, angles, [0,0,0], ... - 'Bah', 'GPU', 'inv'); - end + @@ -962,6 +960,14 @@ EMC_parpool(nCores); end + fprintf('PCA decomposition complete\n'); + + rotConvention = 'Bah'; + use_interpolator_once = true; + normalize_asymmetric_count = false; + symmetry_weight = []; + + if (previousPCA) % Read the matrix of eigenvectors from the prior PCA. oldPca = load(previousPCA); @@ -1030,9 +1036,27 @@ tmpReshape = zeros(prod(sizeMask),1); tmpReshape(masks.('binary').(stHALF).(sprintf('s%d',iScale)) ) = varianceMap(:); + tmpReshape = reshape(single(tmpReshape), sizeMask); + symmetry_weight = []; + if (emc.Pca_constrain_symmetry) + + % resample the tmpReshape to apply the symmetry + % Assuming we are not using helical here + SAVE_IMG(tmpReshape,sprintf('%s_reshape1_weight%d-%s-%d.mrc',outputPrefix, eigsFound, halfSet, iScale),pixelSize); + + [~, tmpReshape] = interpolator(gpuArray(tmpReshape), [0,0,0], [0,0,0], rotConvention , 'inv', emc.symmetry, use_interpolator_once, normalize_asymmetric_count); + [~, symmetry_weight ] = interpolator(gpuArray(single(masks.('binary').(stHALF).(sprintf('s%d',iScale)))), [0,0,0], [0,0,0], rotConvention , 'inv', emc.symmetry, use_interpolator_once, normalize_asymmetric_count); + symmetry_weight(symmetry_weight < .01) = 1; + + + SAVE_IMG(symmetry_weight,sprintf('%s_symmetry_weight%d-%s-%d.mrc',outputPrefix, eigsFound, halfSet, iScale),pixelSize); + SAVE_IMG(tmpReshape,sprintf('%s_reshape2_weight%d-%s-%d.mrc',outputPrefix, eigsFound, halfSet, iScale),pixelSize); + + tmpReshape = tmpReshape ./ symmetry_weight; + end fname = sprintf('%s_varianceMap%d-%s-%d.mrc',outputPrefix, eigsFound, halfSet, iScale); - SAVE_IMG(MRCImage(single(gather(reshape(tmpReshape, sizeMask)))), fname,pixelSize); + SAVE_IMG(tmpReshape, fname,pixelSize); @@ -1042,11 +1066,23 @@ for iEig = 1:eigsFound tmpReshape = zeros(prod(sizeMask),1); tmpReshape(masks.('binary').(stHALF).(sprintf('s%d',iScale)) ) = U{iScale}(:, iEig); - eigenImage = reshape(tmpReshape, sizeMask); + eigenImage = reshape(single(tmpReshape), sizeMask); eigenImage = eigenImage - mean(eigenImage(masks.('binaryApply').(stHALF).(sprintf('s%d',iScale)))); eigenImage = eigenImage ./rms(eigenImage(masks.('binaryApply').(stHALF).(sprintf('s%d',iScale)))).* masks.('binary').(stHALF).(sprintf('s%d',iScale)) ; + if (emc.Pca_constrain_symmetry) + % resample the tmpReshape to apply the symmetry + % Assuming we are not using helical here + + [~, eigenImage] = interpolator(gpuArray(eigenImage), [0,0,0], [0,0,0], rotConvention , 'inv', emc.symmetry, use_interpolator_once,normalize_asymmetric_count); + [~, eigenImage_sum ] = interpolator(gpuArray(avgFiltered{iGold, iScale}), [0,0,0], [0,0,0], rotConvention , 'inv', emc.symmetry, use_interpolator_once,normalize_asymmetric_count); + eigenImage_sum = (eigenImage_sum + eigenImage) ./ (2.*symmetry_weight); + eigenImage = eigenImage ./ symmetry_weight; + + else + eigenImage_sum = (eigenImage + avgFiltered{iGold, iScale} )./2; + end eigList{iEig,1} = gather(eigenImage); - eigList_SUM{iEig,1} = gather((eigenImage + avgFiltered{iGold, iScale} )./2); + eigList_SUM{iEig,1} = gather(gather(eigenImage_sum)); end @@ -1054,8 +1090,8 @@ [ eigMont_SUM ] = BH_montage4d(eigList_SUM, 'eigMont_SUM'); fname = sprintf('%s_eigenImage%d-%s-mont_%d.mrc',outputPrefix, eigsFound, halfSet, iScale); fname_SUM = sprintf('%s_eigenImage%d-SUM-%s-mont_%d.mrc',outputPrefix, eigsFound, halfSet, iScale); - SAVE_IMG(MRCImage(single(gather(eigMont))), fname,pixelSize); - SAVE_IMG(MRCImage(single(gather(eigMont_SUM))), fname_SUM,pixelSize); + SAVE_IMG(eigMont, fname,pixelSize); + SAVE_IMG(eigMont_SUM, fname_SUM,pixelSize); % If requested, limit the number of principal components and coeffs saved diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index bdbe0e65..a46e5f7a 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -1,4 +1,4 @@ -function [ ] = BH_to_cisTEM_mapBack(PARAMETER_FILE, CYCLE, output_prefix, symmetry, MAX_EXPOSURE, varargin) +function [ ] = BH_to_cisTEM_mapBack(PARAMETER_FILE, CYCLE, output_prefix, symmetry, MAX_EXPOSURE, mapBackIter) % Map back and align using the subtomograms as fiducial markers. @@ -16,6 +16,7 @@ skip_to_end = false; emc = BH_parseParameterFile(PARAMETER_FILE); +mapBackIter = EMC_str2double(mapBackIter); MAX_EXPOSURE = EMC_str2double(MAX_EXPOSURE) if isnan(MAX_EXPOSURE) error('MAX_EXPOSURE is nan - if running from an interactive matlab session, did you enter as a string?'); @@ -90,7 +91,10 @@ load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); -mapBackIter = subTomoMeta.currentTomoCPR; +if (mapBackIter == -1) + mapBackIter = subTomoMeta.currentTomoCPR; +end + % TODO: use these to add an optional defocus fitting step % So translational, optional defocus, angles @@ -274,6 +278,7 @@ maxZ = 0; % The + tilt_filepath tiltHeader = getHeader(MRCImage(tilt_filepath, 0)); tiltName = subTomoMeta.mapBackGeometry.tomoName.(tomoList{1}).tiltName; [ maxZ ] = emc_get_max_specimen_NZ( ... diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 271e34b8..05b7fe6c 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -624,14 +624,19 @@ case 'reconstruct' if emcProgramHelp || ... - length(varargin) ~= 6 + length(varargin) ~= 6 && length(varargin) ~= 7 fprintf(['paramterfile\n',... 'cycle #\n',... 'output prefix\n', ... 'symmetry (C1)\n',... - 'max exposure (e/A^2)\n']); + 'max exposure (e/A^2)\n', ... + 'mapBackIter']); else - BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}); + if (length(varargin) == 7) + BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}, varargin{7}); + else + BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}, -1); + end end otherwise error('command --%s-- not recognized. Try "help" for a list.', varargin{1}) diff --git a/testScripts/interpolator.m b/testScripts/interpolator.m index c7307b3e..02f8d250 100644 --- a/testScripts/interpolator.m +++ b/testScripts/interpolator.m @@ -30,12 +30,28 @@ %UNTITLED Construct an instance of this classlt % Detailed explanation goes here + + % Check that the input is single and a gpuArray and cast if needed otherwise the mex call will fail silently + if ~isa(inputVol, 'single') + inputVol = single(inputVol); + end + if ~isa(inputVol, 'gpuArray') + inputVol = gpuArray(inputVol); + end + + - if (nargin == 7) + if (nargin >= 7) useOnlyOnce = varargin{1}; else useOnlyOnce = false; end + + if (nargin == 8) + dividByAsymmetricCount = varargin{2}; + else + dividByAsymmetricCount = true; + end check_symmetry(obj, symmetry, convention); [angles, shifts] = check_anglesAndShifts(obj,angles, shifts, convention, direction); @@ -62,7 +78,8 @@ end - if (obj.nSymMats > 1) + % There are cases where we just want the naive average, eg. re-symmetrizing a symmetry constraint mask for normalization + if (obj.nSymMats > 1 && dividByAsymmetricCount) resampledVol = resampledVol ./ obj.nSymMats; end diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 26bbaa91..1cc08c01 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -44,8 +44,8 @@ outName="$(basename ${mFile} .m)${post}" # bugs line. e.g. buggs=5testingFeature major=1 minor=8 -bugs=1 -nightly=1 +bugs=2 +nightly=0 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 1d81d4e8e5c09c26209ba5e456fceb6c94d9ddce Mon Sep 17 00:00:00 2001 From: himesb Date: Sat, 25 Jan 2025 07:13:12 -0500 Subject: [PATCH 086/151] Wip: fixes bug where CTF/sf3d missing in -z presumably related to uint int conversion collapse --- alignment/BH_alignRaw3d_v2.m | 27 +++++++++++++++----------- masking/BH_mask3d.m | 7 ++++++- masking/BH_weightMaskMex.m | 1 + mexFiles/include/core_headers.cuh | 4 ++-- mexFiles/mexCTF.cu | 10 +++++----- mexFiles/mexSF3D.cu | 32 +++++++++++++++++-------------- mexFiles/mexXform2d.cu | 6 +++--- mexFiles/utils/ctf.cu | 4 ++-- statistics/BH_pcaPub.m | 28 +++++++++++++++++---------- testScripts/emClarity.m | 2 +- testScripts/interpolator.m | 18 ++++++++--------- transformations/BH_average3d.m | 4 ++-- 12 files changed, 83 insertions(+), 60 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 7eecafec..43d044ab 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -103,11 +103,11 @@ rotConvention = 'Helical'; end -if (emc.classification) - refName = emc.('Ref_className'); -else +% if (emc.classification) +% refName = emc.('Ref_className'); +% else refName = emc.('Raw_className'); -end +% end outputPrefix = sprintf('%s_%s', cycleNumber, emc.('subTomoMeta')); @@ -120,11 +120,12 @@ % % % % if (emc.classification || emc.multi_reference_alignment) -if (emc.classification) - geometry = subTomoMeta.(cycleNumber).ClassAlignment; - refVectorFull{1}= [emc.('Ref_references_odd');1] - refVectorFull{2}= [emc.('Ref_references_eve');1] -elseif (emc.multi_reference_alignment) +% if (emc.classification) +% geometry = subTomoMeta.(cycleNumber).ClassAlignment; +% refVectorFull{1}= [emc.('Ref_references_odd');1] +% refVectorFull{2}= [emc.('Ref_references_eve');1] +% else if +if (emc.multi_reference_alignment) geometry = subTomoMeta.(cycleNumber).ClusterRefGeom; refVectorFull{1}= [emc.('Raw_classes_odd');classVector{1} ] refVectorFull{2}= [emc.('Raw_classes_eve');classVector{2} ] @@ -340,7 +341,9 @@ for iWccc = 1:length(nReferences(1)); wCCC{iWccc} = 0; end -if (emc.classification || emc.multi_reference_alignment) +% if (emc.classification || emc.multi_reference_alignment) +if ( emc.multi_reference_alignment) + for iRef = 1:nReferences(1) fscINFO = subTomoMeta.(cycleNumber).('fitFSC').(sprintf('Ref%d',iRef)); @@ -1006,7 +1009,7 @@ - % maybe I should be rotating peak mask here in case it has + % maybe I should be rotating peak mask here in case it hastrack_stats % an odd shape, since we are leaving the proper frame iRotRef = BH_bandLimitCenterNormalize(... @@ -1395,6 +1398,8 @@ % It is probably more useful see the shifts in the particle % reference frame vs. the avg which was the original + % Note: the final translation is with the ref rotated to the particles orientation, so the + % shift is in the tomogram (lab) reference frame. if (emc.printShiftsInParticleBasis) printShifts = zeros(3,3); printShifts(1,:) = RotMat * reshape(cccInitial(1,end-2:end),3,1); diff --git a/masking/BH_mask3d.m b/masking/BH_mask3d.m index 9ff14a8f..c5a86a3a 100755 --- a/masking/BH_mask3d.m +++ b/masking/BH_mask3d.m @@ -186,7 +186,12 @@ ellipsoid = (G1./mRadius(1)).^2 + (G2./mRadius(2)).^2; if (flg3d) - fullMask = (ellipsoid <= 1) & (G3 <= mRadius(3)); + if (asymmetricRestriction) + % Only deal with Z, letting the sector extend to the edge of XY and multiply that by the vol mask + fullMask = (G3 <= mRadius(3)); + else + fullMask = (ellipsoid <= 1) & (G3 <= mRadius(3)); + end else fullMask = (ellipsoid <= 1); end diff --git a/masking/BH_weightMaskMex.m b/masking/BH_weightMaskMex.m index 6db4344a..d477c593 100644 --- a/masking/BH_weightMaskMex.m +++ b/masking/BH_weightMaskMex.m @@ -49,6 +49,7 @@ fractionOfElastics = fractionOfElastics ./ max(fractionOfElastics(:)); + [SF3D] = mexSF3D(doHalfMask,doSqCTF,SIZE,pixelSize_angstrom * SAMPLING,iWavelength,iCs, ... gather(single(iDefocus + iddF)), ... gather(single(iDefocus - iddF)), ... diff --git a/mexFiles/include/core_headers.cuh b/mexFiles/include/core_headers.cuh index 47c50dac..3cb49c81 100644 --- a/mexFiles/include/core_headers.cuh +++ b/mexFiles/include/core_headers.cuh @@ -86,5 +86,5 @@ struct ctfParams { // Kernel defs -__global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered); -__global__ void ctf(cufftReal* ctf, uint2 dims, uint2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered, float radial_weight, float total_exposure, float wiener_constant = 0.f); +__global__ void ctf(cufftReal* a, int2 dims, int2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered); +__global__ void ctf(cufftReal* ctf, int2 dims, int2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered, float radial_weight, float total_exposure, float wiener_constant = 0.f); diff --git a/mexFiles/mexCTF.cu b/mexFiles/mexCTF.cu index 9e128dc6..3be0d664 100644 --- a/mexFiles/mexCTF.cu +++ b/mexFiles/mexCTF.cu @@ -48,8 +48,8 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) *defocus1, *defocus2, *defocusAst); // mexPrintf("%f %f %f %f\n",*defocus1,*defocus2,b_ctf.defocus1,b_ctf.defocus2); - uint2 dims; - uint2 o_dims; + int2 dims; + int2 o_dims; @@ -64,17 +64,17 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) mwSize const input_dims = 2; mxComplexity output_type = mxREAL; mxClassID output_class = mxSINGLE_CLASS; - dims = make_uint2(*nX,*nY); + dims = make_int2(*nX,*nY); mwSize output_size[input_dims]; if (*doHalfGrid ) { - o_dims = make_uint2(0, *nY/2); + o_dims = make_int2(0, *nY/2); dims.x = dims.x/2 + 1; } else { - o_dims = make_uint2(*nX/2, *nY/2); + o_dims = make_int2(*nX/2, *nY/2); } diff --git a/mexFiles/mexSF3D.cu b/mexFiles/mexSF3D.cu index b1620241..59604f5e 100644 --- a/mexFiles/mexSF3D.cu +++ b/mexFiles/mexSF3D.cu @@ -25,7 +25,7 @@ const float cosine_edge_norm = 1.0f;// / 3.0f; __global__ void sf3dKernel(const cudaTextureObject_t tex_obj, float *outputData, float3 size_shift, - uint3 dims, + int3 dims, float2 sinAcosA ) { @@ -44,9 +44,9 @@ __global__ void sf3dKernel(const cudaTextureObject_t tex_obj, // The centered coordinate in the 3d volume - float x_centered = (float)x - (float)dims.x/2; + float x_centered = float(x - dims.x/2); tw_pre = -x_centered * sinAcosA.x + size_shift.z; - float y_centered_normalized = ((float)y - (float)(dims.y/2) + size_shift.y) / (float)dims.y + 0.5f; + float y_centered_normalized = (float(y - dims.y/2) + size_shift.y) / float(dims.y) + 0.5f; for (int z = 0; z < dims.z; z++) { @@ -54,15 +54,15 @@ __global__ void sf3dKernel(const cudaTextureObject_t tex_obj, tw = tw_pre + z_centered * sinAcosA.y; if (tw < -slice_thickness_pixel_radius || tw > slice_thickness_pixel_radius) { - continue; + continue; } // FIXME this should approximate a sinc - zWeight = (0.5 + 0.5*cosf(tw * cosine_edge_arg )) * cosine_edge_norm; - + zWeight = (0.5 + 0.5*cosf(tw * cosine_edge_arg )); + tu = x_centered*sinAcosA.y + z_centered*sinAcosA.x + size_shift.x; tu /= (float)dims.x; // Normalized coords - tw /= (float)dims.z; + // tw /= (float)dims.z; @@ -112,9 +112,9 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) float * d_output_img = NULL; float * d_ctf_img = NULL; - uint3 dims; - uint2 ctf_dims; - uint2 o_ctf_dims; + int3 dims; + int2 ctf_dims; + int2 o_ctf_dims; float2 sinAcosA[*nTilts]; @@ -133,19 +133,19 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) } - dims = make_uint3(wantedSize[0],wantedSize[1],wantedSize[2]); - ctf_dims = make_uint2(wantedSize[0],wantedSize[1]); + dims = make_int3(wantedSize[0],wantedSize[1],wantedSize[2]); + ctf_dims = make_int2(wantedSize[0],wantedSize[1]); if (*doHalfGrid ) { - o_ctf_dims = make_uint2(0, ctf_dims.y/2); + o_ctf_dims = make_int2(0, ctf_dims.y/2); ctf_dims.x = ctf_dims.x/2 + 1; } else { - o_ctf_dims = make_uint2(ctf_dims.x/2, ctf_dims.y/2); + o_ctf_dims = make_int2(ctf_dims.x/2, ctf_dims.y/2); } @@ -251,7 +251,11 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) is_initialized = true; } // Call the sf3d kernel + checkCudaErrors(cudaGetLastError()); sf3dKernel<<>>(tex_obj, d_output_img, size_shift, dims, sinAcosA[iAng]); + checkCudaErrors(cudaPeekAtLastError()); + cudaError_t error = cudaStreamSynchronize(cudaStreamPerThread); + checkCudaErrors(error); } diff --git a/mexFiles/mexXform2d.cu b/mexFiles/mexXform2d.cu index ddf85227..5cf39745 100644 --- a/mexFiles/mexXform2d.cu +++ b/mexFiles/mexXform2d.cu @@ -344,7 +344,7 @@ const float EXTRAPVAL = 0.0f; //! @param outputData output data in global memory //////////////////////////////////////////////////////////////////////////////// __global__ void transformKernel_FWD(float *outputData, - uint2 dims, + int2 dims, float2 rm_1, float2 rm_2, float2 shifts, @@ -401,7 +401,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) int iSpot = 1; float * d_output_img = NULL; float * d_input_img = NULL; - uint2 dims; + int2 dims; float2 shifts; float2 rm_1; float2 rm_2; @@ -450,7 +450,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) mwSize const numel_input = mxGPUGetNumberOfElements(inputArray); mxComplexity input_data_type = mxGPUGetComplexity(inputArray); - dims = make_uint2(input_size[0],input_size[1]); + dims = make_int2(input_size[0],input_size[1]); size = dims.x * dims.y * sizeof(float); diff --git a/mexFiles/utils/ctf.cu b/mexFiles/utils/ctf.cu index 0f0ecc82..21b2098e 100644 --- a/mexFiles/utils/ctf.cu +++ b/mexFiles/utils/ctf.cu @@ -9,7 +9,7 @@ const float expC = 2.81410f; const float kvScale = 1.0f; //FIXME for other voltages (0.8 for 200) // createb_ctf -__global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, +__global__ void ctf(cufftReal* a, int2 dims, int2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered) { @@ -65,7 +65,7 @@ __global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, flo } // createb_ctf -__global__ void ctf(cufftReal* a, uint2 dims, uint2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, +__global__ void ctf(cufftReal* a, int2 dims, int2 o_dims, ctfParams b_ctf, float2 fourierVoxelSize, bool calc_centered, float radial_weight, float total_exposure, float wiener_constant) { diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index f5f3ba34..64e195fd 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -266,11 +266,8 @@ BH_multi_validArea( maskSize, maskRadius, emc.scale_calc_size ) -if (test_multi_ref_diffmap) - refName = emc.('Raw_className'); -else - refName = 0; -end +refName = emc.('Raw_className'); + % If emc.classification is negative combine the data for clustering, but don't set % any of the alignment changes to be persistant so that extracted class @@ -519,7 +516,7 @@ kernelSize = ceil(threeSigma(iScale).*3) + 3; kernelSize = kernelSize + (1-mod(kernelSize,2)); % masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); - masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'gpu', {}); + masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'cpu', {}); % SAVE_IMG( masks.('scaleMask').(sprintf('s%d',iScale)), ... @@ -529,6 +526,7 @@ end end +default_highpass = [1e-6,400,2.2*pixelSize]; avgMotif_FT = cell(1+flgGold,emc.n_scale_spaces); avgFiltered = cell(1+flgGold,emc.n_scale_spaces); % Here always read in both, combine if flgGold = 0 @@ -542,6 +540,16 @@ end + if (test_multi_ref_diffmap) + % We don't want to apply this on top of the length scale blurring, so only use the default or user supplied bandpass + % when we have multiple references, and hence, no length scale blurring. + bp_vals = emc.Pca_bandpass; + else + bp_vals = default_highpass; + end + + masks.('highPass').(sprintf('s%d',iScale)) = gather(BH_bandpass3d(sizeMask,bp_vals(1), bp_vals(2), bp_vals(3),'GPU',pixelSize)); + tmp_avg = tmp_avg - mean(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); tmp_avg = tmp_avg ./ rms(tmp_avg(masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)))); tmp_avg = tmp_avg .* masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)); @@ -551,7 +559,7 @@ end avgMotif_FT{iGold, iScale} = ... BH_bandLimitCenterNormalize(tmp_avg,... - BH_bandpass3d(sizeMask,1e-6,400,2.2*pixelSize,'GPU',pixelSize), ... + BH_bandpass3d(sizeMask,default_highpass(1),default_highpass(2),default_highpass(3),'GPU',pixelSize), ... masks.('binaryApply').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)),... [0,0,0;0,0,0],'single'); @@ -642,11 +650,11 @@ if ~(test_multi_ref_diffmap) gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); end + gpuMasks.('highPass').(stSCALE) = gpuArray(masks.('highPass').(stSCALE)); - bp_vals = emc.Pca_bandpass; - gpuMasks.('highPass').(stSCALE) = BH_bandpass3d(sizeMask,bp_vals(1), bp_vals(2), bp_vals(3),'GPU',pixelSize); + end - + % % % for iGold_inner = 1:1+flgGold % % % for iScale = 1:emc.n_scale_spaces % % % avgMotif_FT{iGold_inner, iScale} = ... diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 05b7fe6c..1a942e9b 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -635,7 +635,7 @@ if (length(varargin) == 7) BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}, varargin{7}); else - BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}, -1); + BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}, "-1"); end end otherwise diff --git a/testScripts/interpolator.m b/testScripts/interpolator.m index 02f8d250..624b2797 100644 --- a/testScripts/interpolator.m +++ b/testScripts/interpolator.m @@ -6,7 +6,7 @@ % Pointers to underlying cuda objects texObject = ''; cuArray = ''; - % Symmetry matrices + % Symmetry matrices nSymMats; symmetry_matrices; symmetry_type = ''; @@ -22,7 +22,7 @@ % keep it alive for symmetry ops, but delete after interp, and store % the input volume here. input_volume = ''; - make_tex_persistent = false; + make_tex_persistent = false; end methods @@ -57,7 +57,7 @@ [angles, shifts] = check_anglesAndShifts(obj,angles, shifts, convention, direction); boolDirection = check_inputs(obj,direction); - % Set the input size +% Set the input size obj.input_size = uint64((size(inputVol))); obj.dummy_vol = inputVol(1:2,1:2,1:2); @@ -75,8 +75,8 @@ shifts, ... boolDirection, ... obj.texObject); - - end + + end % There are cases where we just want the naive average, eg. re-symmetrizing a symmetry constraint mask for normalization if (obj.nSymMats > 1 && dividByAsymmetricCount) @@ -189,7 +189,7 @@ function [ ] = check_symmetry(obj, symmetry, convention) - if isempty(obj.symmetry_type | ~strcmpi(symmetry, obj.symmetry_type)) + if isempty(obj.symmetry_type | ~strcmpi(symmetry, obj.symmetry_type)) if (strcmpi(symmetry, obj.symmetry_type)) return; @@ -394,8 +394,8 @@ error('Only CX, DX, O, I(2) symmetry is implemented'); end - - + + @@ -422,7 +422,7 @@ obj.texObject = ''; obj.cuArray = ''; end - end + end end end diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index a3bcb88d..17e2a08a 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -848,8 +848,8 @@ [ iSF3D ] = BH_weightMaskMex(sizeCalc, samplingRate, TLT, center.*samplingRate, reconGeometry, emc.wiener_constant); make_sf3d = false; end - - + + if (emc.flgQualityWeight) iCCC = positionList(iSubTomo,[1]+26*(iPeak-1)); From 837c76c50785636d08841701f34c46e6af548da1 Mon Sep 17 00:00:00 2001 From: himesb Date: Fri, 28 Feb 2025 15:09:50 -0500 Subject: [PATCH 087/151] wip: working out a better handedness check --- alignment/BH_alignRaw3d_v2.m | 54 +- alignment/BH_runAutoAlign.m | 34 +- alignment/BH_templateSearch3d_2.m | 5 +- alignment/emC_autoAlign | 1 + coordinates/BH_multi_iterator.m | 20 +- ctf/BH_ctf_Correct3d.m | 53 +- ctf/BH_ctf_Estimate.m | 111 ++-- ctf/BH_ctf_Refine2.m | 554 +++++++++++------- ctf/BH_ctf_Updatefft.m | 12 +- ctf/BH_runCtfFind.m | 144 +++-- logicals/EMC_assert_deprecated_substitution.m | 12 +- masking/BH_bandLimitCenterNormalize.m | 6 +- masking/BH_bandpass3d.m | 2 +- masking/BH_weightMaskMex.m | 2 + metaData/BH_geometryAnalysis.m | 28 +- metaData/BH_parseParameterFile.m | 210 +++++-- mexFiles/mexFP16.cu | 4 +- mexFiles/mexSF3D.cu | 2 +- statistics/BH_pcaPub.m | 59 +- synthetic/BH_synthetic_mapBack.m | 105 +++- synthetic/BH_to_cisTEM_mapBack.m | 470 ++++++++++++--- testScripts/BH_benchmark.m | 2 +- testScripts/fourierTransformer.m | 19 +- testScripts/mCompile.sh | 2 +- transformations/BH_average3d.m | 30 +- 25 files changed, 1331 insertions(+), 610 deletions(-) diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index 43d044ab..b56d5b06 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -367,6 +367,7 @@ for iRef = 1 + fscINFO = subTomoMeta.(cycleNumber).('fitFSC').('Ref1'); [radialGrid,~,~,~,~,~ ] = BH_multi_gridCoordinates(sizeCalc, 'Cartesian', ... 'GPU', {'none'}, 1, 0, 1 ); @@ -433,8 +434,8 @@ % and rely on the normalization during the CCC calc. TODO ref_FT1{iGold}{iRef} = gather(conj(BH_bandLimitCenterNormalize(... refTMP.*volMask, bandpassFiltREF{iRef}, (volMask>0.01), padCalc, flgPrecision))); - - + + ref_FT2{iGold}{iRef} = gather(refTMP_2); @@ -488,8 +489,8 @@ gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, true); nAngles = sum(gridSearch.number_of_angles_at_each_theta); - inPlaneSearch = gridSearch.parameter_map.psi - + inPlaneSearch = gridSearch.parameter_map.psi + try symmetry_constrained_search = emc.('symmetry_constrained_search'); fprintf('Using symmetry constrained search\n'); @@ -509,14 +510,13 @@ end end - flgRefine=false; - - for i = 1:length(gridSearch.parameter_map.phi) - if gridSearch.parameter_map.phi{i} > 0 - flgRefine=true; - break; - end - end + flgRefine= gridSearch.number_of_out_of_plane_angles > 1; + % for i = 1:gridSearch.number_of_out_of_plane_angles + % if gridSearch.parameter_map.phi{i} > 0 + % flgRefine=true; + % break; + % end + % end angleStep = []; @@ -541,7 +541,6 @@ nCount = 1; - firstLoop = true; nIgnored = 0; @@ -574,6 +573,25 @@ parfor iParProc = parVect % for iParProc = parVect + % To avoid unitialized temporary warnings + iMaxWedgeIfft = []; + imgWdgInterpolator = ''; + refInterpolator = ''; + refWdgInterpolator = ''; + particleInterpolator = ''; + phiInc = []; + thetaInc = []; + psiInc = []; + estPeakCoords = []; + iTrimParticle = []; + iTrimInitial = []; + iWedgeInitial = []; + iWedgeMask = []; + refToAlign = ''; + iRotRef = []; + iRotWdg = []; + iRotWdgMask = []; + symmetry = emc.symmetry; bestAngles_tmp = struct(); @@ -629,9 +647,12 @@ peakMaskInterpolator = ''; peakMaskInterpolator = interpolator(gpuArray(peakMask),[0,0,0],[0,0,0], rotConvention , 'forward', 'C1', false); + mip = struct(); if (emc.track_stats) - mip = struct(); mip.('mask') = gpuArray(stat_mask); + else + % to avoid uninitialized temporaries warnings + mip.('mask') = []; end @@ -742,6 +763,8 @@ mip.('x') = {}; mip.('x2') = {}; mip.('N') = 0; + else + measure_noise = false; end if (breakPeak) continue; @@ -907,8 +930,7 @@ for iInPlane = inPlaneSearch psi = iInPlane; %[phi,theta,psi-phi]; - - + RotMat = BH_defineMatrix([phi, theta, psi - phi],rotConvention, 'inv'); RotMat = reshape(angles,3,3) * RotMat; diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index 28bde588..a813cbd3 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -163,21 +163,25 @@ % switch_axes = false; % abs(abs(imgRotation) - 180) -a = ones(nX,nY,'single','gpuArray'); -p = BH_multi_padVal([nX,nY],max([nX,nY]).*[2,2]); -pad = BH_padZeros3d(a,'fwd',p,'GPU','single'); - -b = BH_resample2d(pad,[imgRotation,0,0],[0,0],'Bah','GPU','inv',1,size(pad)); -s = pad+b; -score_1 = sum(sum(s==2))./sum(b(:)); - -pad = rot90(pad); -b = BH_resample2d(pad,[90-imgRotation,0,0],[0,0],'Bah','GPU','forward',1,size(pad)); -s = pad+b; -score_2 = sum(sum(s==2))./sum(b(:)); - -if score_2 > score_1 - switch_axes = true; +if (emc.autoAli_switchAxes) + a = ones(nX,nY,'single','gpuArray'); + p = BH_multi_padVal([nX,nY],max([nX,nY]).*[2,2]); + pad = BH_padZeros3d(a,'fwd',p,'GPU','single'); + + b = BH_resample2d(pad,[imgRotation,0,0],[0,0],'Bah','GPU','inv',1,size(pad)); + s = pad+b; + score_1 = sum(sum(s==2))./sum(b(:)); + + pad = rot90(pad); + b = BH_resample2d(pad,[90-imgRotation,0,0],[0,0],'Bah','GPU','forward',1,size(pad)); + s = pad+b; + score_2 = sum(sum(s==2))./sum(b(:)); + + if score_2 > score_1 + switch_axes = true; + else + switch_axes = false; + end else switch_axes = false; end diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 47063ced..a257df8b 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -483,8 +483,7 @@ tomoChunk = gather(tomoChunk .*validCalcMask); - tmp_sum = sum(tomoChunk(validCalcMask > 0.1)); % REVERT - % tmp_sum = sum(tomoChunk(:)); + tmp_sum = sum(tomoChunk(:)); fullX = fullX + gather(tmp_sum); fullX2 = fullX2 + gather(tmp_sum.^2); @@ -500,11 +499,11 @@ tomoCoords(tomoIDX,:) = [cutX,cutY,cutZ]; tomoIDX = tomoIDX + 1; - end % end of loop over Z chunks end % end of loop over Y chunks end % end of loop over X chunks + % Normalize the global variance globalVariance = (fullX2/fullnX) - (fullX/fullnX)^2; diff --git a/alignment/emC_autoAlign b/alignment/emC_autoAlign index 7237b54d..053028a6 100755 --- a/alignment/emC_autoAlign +++ b/alignment/emC_autoAlign @@ -306,6 +306,7 @@ echo $iEcho && iEcho=$(($iEcho+1)) -AngleOffset ${tiltAngleOffset} > tiltAlign.log else # run without local until an intial stable global solution is found + echo "${EMC_TILTALIGN} -ModelFile ${pName}.fid -ImageFile ${pName}.preali -ImagesAreBinned ${iBin} -OutputModelFile ${pName}.3dmod -OutputResidualFile ${pName}.resid -OutputFidXYZFile ${pName}fid.xyz -OutputTiltFile ${pName}.tlt -OutputTransformFile ${pName}.tltxf_nonScaled -RotationAngle 0.0 -TiltFile ${pName}.rawtlt -AngleOffset ${tiltAngleOffset} -RotOption 1 -RotDefaultGrouping 3 -TiltOption ${TILT_OPTION} -TiltDefaultGrouping 3 -MagOption ${MAG_OPTION} -MagDefaultGrouping 3 -BeamTiltOption 0 -ResidualReportCriterion 1.0 -SurfacesToAnalyze 1 -MetroFactor 0.25 -MaximumCycles 1000 -KFactorScaling 1.0 -NoSeparateTiltGroups 2 -AxisZShift 1000 > tiltAlign.log" > ./.${inp}_tilalign.sh tiltalign \ -ModelFile ${pName}.fid \ -ImageFile ${pName}.preali \ diff --git a/coordinates/BH_multi_iterator.m b/coordinates/BH_multi_iterator.m index 467c46e7..249ee995 100755 --- a/coordinates/BH_multi_iterator.m +++ b/coordinates/BH_multi_iterator.m @@ -92,18 +92,18 @@ % needs more memory, so a smaller size here means more transfers, but this % should be balanced by the finer angles (more comp) if all(SIZES(1,:) == 256) - nextBest = [128,144,160,168,192,216,224,256]; + nextBest = [128,144,160,168,192,216,224,256]; elseif all(SIZES(1,:) == 384) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384]; + nextBest = [128,144,160,168,192,216,224,256,... + 288,300,320,336,360,384]; elseif all(SIZES(1,:) == 432) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432]; + nextBest = [128,144,160,168,192,216,224,256,... + 288,300,320,336,360,384,400,432]; elseif all(SIZES(1,:) == 512) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432,480,512]; + nextBest = [128,144,160,168,192,216,224,256,... + 288,300,320,336,360,384,400,432,480,512]; elseif all(SIZES(1,:) > 512) - nextBest = [128,144,160,168,192,216,224,256,... + nextBest = [128,144,160,168,192,216,224,256,... 288,300,320,336,360,384,400,432,480,512,... 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]; end @@ -115,7 +115,7 @@ borderSizeCalc = floor((sizeTemplate + APODIZATION)./2); borderSizeKeep = borderSizeCalc + 2.*sizeParticle; - abs(sum(sizeImage - sizeTemplate)) +abs(sum(sizeImage - sizeTemplate)) sum(0.1.*sizeImage) if abs(sum(sizeImage - sizeTemplate)) < sum(0.1.*sizeImage) @@ -137,7 +137,7 @@ % searched is the same size as the reference score(:,2:4) = repmat(nextBest',1,3) ./ postPad .* (minIter >= 0) - + [~, cX] = max(score(:,2)) ; [~, cY] = max(score(:,3)) ; [~, cZ] = max(score(:,4)) ; diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 649f2b08..3a53b3d1 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -21,7 +21,7 @@ % TODO remove thise params tiltWeight = [0.2,0]; -shiftDefocusOrigin = 1; +shiftDefocusOrigin = emc.set_defocus_origin_using_subtomos; tiltStart = 1; try @@ -48,6 +48,10 @@ end + +flip_defocus_offset = emc.test_flip_defocus_offset +flip_tilt_offset = emc.test_flip_tilt_offset + expand_lines = emc.('expand_lines'); if isempty(super_sample) || expand_lines == false expand_lines = ''; @@ -128,11 +132,8 @@ % This will be set false if the reconstruction is for template matching or % for tomoCPR -try - useSurfaceFit = emc.('useSurfaceFit') -catch - useSurfaceFit = 1; -end +useSurfaceFit = emc.('useSurfaceFit') + try % Not for normal use, pass the total dose less first frame to flip values. @@ -288,10 +289,7 @@ % If there is only one tilt, things break in a weird way nGPUs = min(nGPUs, nTilts); -% For now, limit the number of processes to avoid memory issues -% TODO: determine something more precise -n_cores_wanted = min(emc.nCpuCores, floor(emc.pixel_size_angstroms*0.7)*nGPUs); -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, n_cores_wanted); +[ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores); try EMC_parpool(nParProcesses) @@ -301,14 +299,14 @@ end - parfor iParProc = 1:nParProcesses -% for iParProc = 1:nParProcesses %%revert +% for iParProc = 1:nParProcesses % revert % iGPU = mod(iParProc,nGPUs); for iTilt = iterList{iParProc} nTomos = 0; alreadyMade = 0; + % For now, since the tilt geometry is not necessarily updated (it is manual) % in the subTomoMeta, check that newer (possible perTilt refined) data is % not present. @@ -352,9 +350,12 @@ if alreadyMade == nTomos fprintf('All tomos 1-%d found to exist for tilt-series %s\n',nTomos,tiltList{iTilt}); + % remove the value form the iter list + iterList{iParProc} = iterList{iParProc}(iterList{iParProc} ~= iTilt); continue end end + preBinStacks(TLT, ... tiltList{iTilt}, ... @@ -550,7 +551,9 @@ useSurfaceFit,invertDose,... bh_global_turn_on_phase_plate,... filterProjectionsForTomoCPRBackground,... - emc.whitenPS); + emc.whitenPS, ... + flip_defocus_offset, ... + flip_tilt_offset); end % Write out the stack to the cache directory as a tmp file @@ -962,7 +965,9 @@ useSurfaceFit,invertDose, ... phakePhasePlate, ... filterProjectionsForTomoCPRBackground,... - flgWhitenPS) + flgWhitenPS, ... + flip_defocus_offset, ... + flip_tilt_offset) % Correct in strips which is more expensive but (hopefully) more accurate. @@ -992,6 +997,8 @@ if (useSurfaceFit) defocusOffset = 0; else + % FIXME: this should probably be ctf3dDepth/2 + % FIXME: this should be renamed as it is an offset in Z (opposite sign to defocus) defocusOffset = (((n_slabs_to_reconstruct-1)/-2+(iSection-1))*ctf3dDepth); fprintf('Not using surface fit, so using offset %3.3e nm for section %d with COM offset %3.3e nm with ctf3dDepth %3.3e\n', defocusOffset*10^9, iSection, avgZ*10^9, ctf3dDepth*10^9); % Assuming the majority of the fit defocus came from the subtomograms, then the estimated defocus value needs to be moved from @@ -999,6 +1006,9 @@ defocusOffset = (defocusOffset + avgZ); % The average height of the particles is factored into the surface fit end +if (flip_defocus_offset) + defocusOffset = -defocusOffset; +end if ( flgDampenAliasedFrequencies ) @@ -1086,23 +1096,27 @@ rZ = zeros([d1,d2],'single','gpuArray'); end - defocus_adj = D0 - (defocusOffset.*cosd(TLT(iPrj,4))); + if (flip_tilt_offset) + defocus_adj = D0 + (defocusOffset.*cosd(TLT(iPrj,4))); + else + defocus_adj = D0 - (defocusOffset.*cosd(TLT(iPrj,4))); + end + % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) rA = BH_defineMatrix(TLT(iPrj,4),'TILT','fwdVector') ; % Transform the specimen plane tX = round(rA(1).*rX + rA(4).*rY + rA(7).*rZ + oX); tY = round(rA(2).*rX + rA(5).*rY + rA(8).*rZ + oY); - % undefocus is positive, but we have stored the negative value (so we just add this to the positional offset) + % undefocus is positive, so we subtract the offset in Z tZ = defocus_adj - (pixel_size_angstroms*10^-10).*(rA(3).*rX + rA(6).*rY + rA(9).*rZ); % Some edge pixels can be out of bounds depending on the orientation of - % the plan fit. Setting to zero will will ignore them (assuming defocus - % is always < 0) + % the plan fit. Setting to zero will will ignore them tZ( tX < 1 | tY < 1 | tX > d1 | tY > d2) = 1; minDefocus = min(tZ(:)); - maxDefocus = max(tZ(tZ < 1)); + maxDefocus = max(tZ(tZ < 1)); %tZ is in angstrom so always << 1 % To track sampling in case I put in overlap samplingMask = zeros([d1,d2],'single','gpuArray'); @@ -1140,6 +1154,7 @@ tmpCorrection = BH_padZeros3d(real(ifftn(iProjectionFT.*Hqz)),trimVal(1,:),trimVal(2,:),'GPU','single'); end + % Each loop we increment by ctf3dDepth tmpMask = (tZ > iDefocus - ctf3dDepth/2 & tZ <= iDefocus + ctf3dDepth/2); linearIDX = unique(sub2ind([d1,d2],tX(tmpMask),tY(tmpMask))); diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 5978948c..62efad8d 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -125,60 +125,33 @@ % Assuming that the first CTF zero is always less than this value FIXED_FIRSTZERO = emc.pixel_size_si / (70*10^-10) ; highCutoff = emc.pixel_size_si/emc.('defCutOff'); -% I still use the def for underfocus < 0 as this places the origin at the -% focal plan in the microscope rather than on the specimen. Which makes -% more sense to me. + defEST = emc.('defEstimate').*10^6 defWIN = emc.('defWindow').*10^6 tiltRange = [-1]; backGroundBuffer = 0.9985; -try - deltaZTolerance = emc.('deltaZTolerance'); -catch - deltaZTolerance = 100e-9; -end -try - zShift = abs(emc.('zShift')); -catch - zShift = 150e-9; -end -if abs(zShift) > 100e-7 - error('make sure your zShift values are of reasonable amounts (50-200nm)'); -end -try - maxNumberOfTiles = emc.('ctfMaxNumberOfTiles'); -catch - - maxNumberOfTiles = 10000; - -end % Starting at +/- 100nm -deltaZTolerance = deltaZTolerance / emc.pixel_size_si; +emc.deltaZTolerance = emc.deltaZTolerance / emc.pixel_size_si; % Use to check for proper gradient. -zShift = zShift / emc.pixel_size_si; +emc.zShift = emc.zShift / emc.pixel_size_si; % Tile size & overlap -try - tileSize = emc.('ctfTileSize'); -catch - tileSize = floor(680e-10 / emc.pixel_size_si); -end + tileOverlap = 2; -tileSize = tileSize + mod(tileSize,2); -% tileSize = max(tileSize, 384); -if (tileSize > 512) +% emc.ctf_tile_size = max(emc.ctf_tile_size, 384); +if (emc.ctf_tile_size > 512) tileOverlap = tileOverlap * 2; end -fprintf('Using a tile size of %d\n',tileSize); +fprintf('Using a tile size of %d\n',emc.ctf_tile_size); -overlap = floor(tileSize ./ tileOverlap); +overlap = floor(emc.ctf_tile_size ./ tileOverlap); % Size to padTile to should be even, large, and preferably a power of 2 try @@ -187,7 +160,7 @@ paddedSize = 768; end -padVAL = BH_multi_padVal([tileSize,tileSize], [paddedSize,paddedSize]); +padVAL = BH_multi_padVal([emc.ctf_tile_size,emc.ctf_tile_size], [paddedSize,paddedSize]); @@ -470,19 +443,24 @@ iEvalMask = BH_multi_gridCoordinates([d1C,1,1],'Cartesian','GPU',{'none'},0,1,0); % Convert to the z-height in the projection + % A positive tilt angle (looking down Y at the origin) is CCW and rotates the positive X axis down in Z farther from focus + % that is the reason for the negative sign iEvalMask = iEvalMask.*(-1.*tand(TLT(tiltIDX,4))); - iEvalPos = iEvalMask; - iEvalNeg = iEvalMask; + iEval_farther_from_focus = iEvalMask; + iEval_closer_to_focus = iEvalMask; + + % zShift is in SI in parameter file, but converted to pixels here. - % Shift by any amount wanted - iEvalPos = iEvalPos - zShift; - iEvalNeg = iEvalNeg + zShift; + % Select region that is at a smaller Z coordinate (farther from focus) + iEval_farther_from_focus = iEval_farther_from_focus - emc.zShift; + % Select region that is at a larger Z coordinate (closer to focus) + iEval_closer_to_focus = iEval_closer_to_focus + emc.zShift; - % Select region limited by tolerance - iEvalPos = ( iEvalPos > gpuArray(-deltaZTolerance) & iEvalPos < gpuArray(deltaZTolerance)); - iEvalNeg = ( iEvalNeg > gpuArray(-deltaZTolerance) & iEvalNeg < gpuArray(deltaZTolerance)); - iEvalMask = ( iEvalMask > gpuArray(-deltaZTolerance) & iEvalMask < gpuArray(deltaZTolerance)); + % Select region limited by defocus tolerance + iEvalMask = ( iEvalMask > gpuArray(-emc.deltaZTolerance) & iEvalMask < gpuArray(emc.deltaZTolerance)); + iEval_farther_from_focus = ( iEval_farther_from_focus > gpuArray(-emc.deltaZTolerance) & iEval_farther_from_focus < gpuArray(emc.deltaZTolerance)); + iEval_closer_to_focus = ( iEval_closer_to_focus > gpuArray(-emc.deltaZTolerance) & iEval_closer_to_focus < gpuArray(emc.deltaZTolerance)); tmpTile = zeros([halfX,paddedSize,3],'single','gpuArray'); @@ -496,21 +474,34 @@ end iProjection = iProjection - ... - BH_movingAverage(iProjection,[tileSize,tileSize]); + BH_movingAverage(iProjection,[emc.ctf_tile_size,emc.ctf_tile_size]); iProjection = iProjection ./ ... - BH_movingRMS(iProjection,[tileSize,tileSize]); + BH_movingRMS(iProjection,[emc.ctf_tile_size,emc.ctf_tile_size]); - for i = 1+tileSize/2:overlap:d1C-tileSize/2 - if min([nT,nT2,nT3])< maxNumberOfTiles && (iEvalMask(i) || iEvalPos(i) || iEvalNeg(i)) - for j = 1+tileSize/2:overlap:d2C-tileSize/2 + reduced_x = floor(emc.ctf_tile_size*cosd(TLT(k,4))); + % ---------------+--------------- + % 000000---------+---------000000 + tile_origin_x = emc_get_origin_index(emc.ctf_tile_size); + reduced_origin_x = emc_get_origin_index(reduced_x); + zeroed_coords = [1:1+(tile_origin_x-reduced_origin_x),(tile_origin_x+reduced_origin_x):emc.ctf_tile_size]; + + + for i = 1+emc.ctf_tile_size/2:overlap:d1C-emc.ctf_tile_size/2 + if min([nT,nT2,nT3])< emc.ctfMaxNumberOfTiles && (iEvalMask(i) || iEval_farther_from_focus(i) || iEval_closer_to_focus(i)) + for j = 1+emc.ctf_tile_size/2:overlap:d2C-emc.ctf_tile_size/2 + thisTile = iProjection( i-emc.ctf_tile_size/2+1:i+emc.ctf_tile_size/2,... + j-emc.ctf_tile_size/2+1:j+emc.ctf_tile_size/2); + + thisTile = thisTile - mean(thisTile(:)); + thisTile = thisTile ./ rms(thisTile(:)); + thisTile(zeroed_coords,:) = 0; + thisTile = abs(bhF2.fwdFFT(BH_padZeros3d(... - (iProjection( ... - i-tileSize/2+1:i+tileSize/2,... - j-tileSize/2+1:j+tileSize/2)),... - padVAL(1,:),padVAL(2,:),... - 'GPU','singleTaper'))); + thisTile,... + padVAL(1,:),padVAL(2,:),... + 'GPU','singleTaper'))); tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; @@ -518,11 +509,11 @@ nT = nT+1; tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; end - if ( iEvalPos(i) ) + if ( iEval_farther_from_focus(i) ) nT2 = nT2+1; tmpTile(:,:,2) = tmpTile(:,:,2) + thisTile; end - if (iEvalNeg(i) ) + if (iEval_closer_to_focus(i) ) nT3 = nT3+1; tmpTile(:,:,3) = tmpTile(:,:,3) + thisTile; end @@ -542,7 +533,7 @@ rotAvgPowerSpec = zeros([paddedSize,paddedSize,3],'single','gpuArray'); for iTile = 1:3 - tmp = bhF2.swapIndexFWD(psTile(:,:,iTile)); + % tmp = bhF2.swapIndexFWD(psTile(:,:,iTile)); psTile(:,:,iTile) = bhF2.swapIndexFWD(psTile(:,:,iTile)); rotAvgPowerSpec(:,:,iTile) = BH_multi_makeHermitian(psTile(:,:,iTile),[paddedSize,paddedSize],1); end @@ -820,7 +811,7 @@ radialForCTF = {fftshift(radialForCTF{1}),1,fftshift(radialForCTF{3})}; currentDefocusEst = maxDef; currentDefocusWin = (defWIN*.25); - measuredVsExpected(1,:) = [maxDef + zShift*emc.pixel_size_si*10^6, maxDef, maxDef - zShift*emc.pixel_size_si*10^6]; + measuredVsExpected(1,:) = [maxDef + emc.zShift*emc.pixel_size_si*10^6, maxDef, maxDef - emc.zShift*emc.pixel_size_si*10^6]; measuredVsExpected(2,2) = maxDef; % Add the determined defocus, and write out with mic paramters as well. TLT(:,15) = repmat(maxDef*10^-6,size(TLT,1),1); @@ -860,7 +851,7 @@ end fprintf('\n******************************************************\n\n'); - fprintf('\nCloser to focus |\tAt focus |\tFarther from focus\n\n'); + fprintf('\Farther from focus |\tAt focus |\Closer to focus\n\n'); fprintf('Expected defocus %3.2f %3.2f %3.2f\n\n', abs(measuredVsExpected(1,:))); fprintf('Measured defocus %3.2f %3.2f %3.2f\n\n' ,abs(measuredVsExpected(2,:))); if ( warnInvertedHand ) diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index ac84601a..e062e4b2 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -73,14 +73,14 @@ % Tile size & overlap tileOverlap = 4; -tileSize = floor(680e-10 / emc.pixel_size_si); -tileSize = BH_multi_iterator(tileSize.*[1,1],'fourier2d'); -tileSize = tileSize(1); + +emc.ctf_tile_size = BH_multi_iterator(emc.ctf_tile_size.*[1,1],'fourier2d'); +emc.ctf_tile_size = emc.ctf_tile_size(1); % if (tileSize > 512) % tileOverlap = tileOverlap * 2; % end -fprintf('Using a tile size of %d\n',tileSize); -overlap = floor(tileSize ./ tileOverlap); +fprintf('Using a tile size of %d\n',emc.ctf_tile_size); +overlap = floor(emc.ctf_tile_size ./ tileOverlap); inc = (0.5 - FIXED_FIRSTZERO) / (paddedSize/2); @@ -114,162 +114,189 @@ end end + + for iStack = 1%stacksFound - - STACK = OPEN_IMG('single',INPUT_CELL{iStack,2}); - % The pixel size should be previously set correctly, but if it is not, then we - % must maintain whatever is there in case beads are to be erased. The model - % used for this process depends on the pixel size in the header when it was - % created in IMod alignment. - [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); - iPixelHeader = EMC_str2double(iPixelHeader); - [d1,d2,d3] = size(STACK); - - - - TLT = INPUT_CELL{iStack,1}; - pathName = INPUT_CELL{iStack,3}; - fileName = INPUT_CELL{iStack,4}; - extension = INPUT_CELL{iStack,5}; - - - SIZEOUT = [d1,d2]; - - [radialForCTF,phi,~,~,~,~] = ... - BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU',{'none'},1,1,0); - - - radialForCTF = {radialForCTF./emc.pixel_size_si,1,phi} ; - - clear phi - - clear sumVector radialAvg - sumVector(length(freqVector)) = gpuArray(double(0)); - radialAvg(length(freqVector)) = gpuArray(double(0)); - - tic - - psTile = zeros([paddedSize,paddedSize,d3],'single'); - - flgReplaceStack = 0; - for iPrj = 1:d3 - iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); - iProjection = iProjection - ... - BH_movingAverage(iProjection,[tileSize,tileSize]); - iProjection = iProjection ./ ... - BH_movingRMS(iProjection,[tileSize,tileSize]); - % Taking a cue from Alexis - maxPixelSizeWanted = 2.0e-10; - if TLT(iPrj,16) < maxPixelSizeWanted - %fprintf(ftmp,'Resampling pixel size\n'); - % Resample to 2Ang/pix - padSq = BH_multi_padVal(size(iProjection),max(size(iProjection)).*[1,1]); + + STACK = OPEN_IMG('single',INPUT_CELL{iStack,2}); + % The pixel size should be previously set correctly, but if it is not, then we + % must maintain whatever is there in case beads are to be erased. The model + % used for this process depends on the pixel size in the header when it was + % created in IMod alignment. + [~,iPixelHeader] = system(sprintf('header -pixel %s',INPUT_CELL{iStack,2})); + iPixelHeader = EMC_str2double(iPixelHeader); + [d1,d2,d3] = size(STACK); + + + + TLT = INPUT_CELL{iStack,1}; + pathName = INPUT_CELL{iStack,3}; + fileName = INPUT_CELL{iStack,4}; + extension = INPUT_CELL{iStack,5}; + + - iProjection = BH_padZeros3d(iProjection,padSq(1,:),padSq(2,:),'GPU','singleTaper'); - sizeIN = size(iProjection,1); - % Replace with BH_fftShift if this works - iProjection = fftshift(fftn(iProjection)); - trimVal = BH_multi_padVal(size(iProjection), floor(size(iProjection).*(TLT(iPrj,16)./maxPixelSizeWanted))); - iProjection = real(ifftn(ifftshift(BH_padZeros3d(iProjection,trimVal(1,:),trimVal(2,:),'GPU','single')))); - sizeOUT = size(iProjection); - if iPrj == 1 - flgReplaceStack = 1; - newSTACK = zeros([sizeOUT,d3],'single'); + SIZEOUT = [d1,d2]; + + [radialForCTF,phi,~,~,~,~] = ... + BH_multi_gridCoordinates([paddedSize,paddedSize,1],'Cylindrical','GPU',{'none'},1,1,0); + + + radialForCTF = {radialForCTF./emc.pixel_size_si,1,phi} ; + + clear phi + + clear sumVector radialAvg + sumVector(length(freqVector)) = gpuArray(double(0)); + radialAvg(length(freqVector)) = gpuArray(double(0)); + + tic + + psTile = zeros([paddedSize,paddedSize,d3],'single'); + psTile_inv = zeros([paddedSize,paddedSize,d3],'single'); + + flgReplaceStack = 0; + for iPrj = 1:d3 + iProjection = gpuArray(STACK(:,:,TLT(iPrj,1))); + iProjection = iProjection - ... + BH_movingAverage(iProjection,[emc.ctf_tile_size,emc.ctf_tile_size]); + iProjection = iProjection ./ ... + BH_movingRMS(iProjection,[emc.ctf_tile_size,emc.ctf_tile_size]); + % Taking a cue from Alexis + maxPixelSizeWanted = 2.0e-10; + if TLT(iPrj,16) < maxPixelSizeWanted + %fprintf(ftmp,'Resampling pixel size\n'); + % Resample to 2Ang/pix + padSq = BH_multi_padVal(size(iProjection),max(size(iProjection)).*[1,1]); + + iProjection = BH_padZeros3d(iProjection,padSq(1,:),padSq(2,:),'GPU','singleTaper'); + sizeIN = size(iProjection,1); + % Replace with BH_fftShift if this works + iProjection = fftshift(fftn(iProjection)); + trimVal = BH_multi_padVal(size(iProjection), floor(size(iProjection).*(TLT(iPrj,16)./maxPixelSizeWanted))); + iProjection = real(ifftn(ifftshift(BH_padZeros3d(iProjection,trimVal(1,:),trimVal(2,:),'GPU','single')))); + sizeOUT = size(iProjection); + if iPrj == 1 + flgReplaceStack = 1; + newSTACK = zeros([sizeOUT,d3],'single'); + end + newSTACK(:,:,TLT(iPrj,1)) = gather(iProjection); + iProjection = []; + % Actual new pixel size + pixelSize = sizeIN./sizeOUT(1).*TLT(iPrj,16); + + % Update the CTF params with the new pixelSize + ctfParams(1) = pixelSize.*10^10; + + %fprintf(ftmp,'%d %d %d %d %d %d\n',trimVal); + %fprintf(ftmp,'pixelOld %3.3e, pixelNew %3.3e\n',TLT(iPrj,16),pixelSize); + + + else + pixelSize = TLT(iPrj,16); + end + end % iPrj 1:d3 + + if ( flgReplaceStack ) + STACK = newSTACK ; clear newSTACK; + end + [d1,d2,d3] = size(STACK) + + + + debug_without_parallel = false; + if (debug_without_parallel) + for iPrj = 1:d3 + fprintf('Calculating stretched tiles on prj %d/ %d in serial debug mode\n',iPrj,d3); + + [psTile(:,:,TLT(iPrj,1)),psTile_inv(:,:,TLT(iPrj,1)),pixelSize] = runAvgTiles(TLT, paddedSize, emc.ctf_tile_size, ... + d1,d2, iPrj, overlap, ... + STACK(:,:,TLT(iPrj,1)), ... + 1, ... + 1, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); + + end + else + try + ppool = EMC_parpool(nWorkers); + catch + delete(gcp('nocreate')); + ppool = EMC_parpool(nWorkers); end - newSTACK(:,:,TLT(iPrj,1)) = gather(iProjection); - iProjection = []; - % Actual new pixel size - pixelSize = sizeIN./sizeOUT(1).*TLT(iPrj,16); - - % Update the CTF params with the new pixelSize - ctfParams(1) = pixelSize.*10^10; - %fprintf(ftmp,'%d %d %d %d %d %d\n',trimVal); - %fprintf(ftmp,'pixelOld %3.3e, pixelNew %3.3e\n',TLT(iPrj,16),pixelSize); + for iPrj = 1:d3 + + pFuture(iPrj) = parfeval(ppool,@runAvgTiles,3, TLT, paddedSize, emc.ctf_tile_size, ... + d1,d2, iPrj, overlap, ... + STACK(:,:,TLT(iPrj,1)), ... + 1, ... + 1, ... + reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); + + + end + for iWorker = 1:d3 + fprintf('Calculating stretched tiles on prj %d/ %d\n',iWorker,d3); + [iPrj, ctfCorr,ctfCorr_inv, pixelSize] = fetchNext(pFuture); + + psTile(:,:,TLT(iPrj,1)) = ctfCorr; + psTile_inv(:,:,TLT(iPrj,1)) = ctfCorr_inv; + end + end % debug without parallel + + pixelSize = pixelSize*10^10; + SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS.mrc',fileName),pixelSize); + bpLog = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); + bpLog = bpLog > 0.99; + bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); + bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); + + for iPrj = 1:d3 + iTile = gpuArray(psTile(:,:,iPrj)); + iTile = iTile.*bp.*bp2; + iTile(~bpLog) = mean(iTile(bpLog)); + psTile(:,:,iPrj) = gather(iTile); - else - pixelSize = TLT(iPrj,16); end - end % iPrj 1:d3 - - if ( flgReplaceStack ) - STACK = newSTACK ; clear newSTACK; - end - [d1,d2,d3] = size(STACK) - - - debug_without_parallel = false; - if (debug_without_parallel) - for iPrj = 1:d3 - fprintf('Calculating stretched tiles on prj %d/ %d in serial debug mode\n',iPrj,d3); + SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName),pixelSize); - [psTile(:,:,TLT(iPrj,1)),pixelSize] = runAvgTiles(TLT, paddedSize, tileSize, ... - d1,d2, iPrj, overlap, ... - STACK(:,:,TLT(iPrj,1)), ... - 1, ... - 1, ... - reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); - end - else - try - ppool = EMC_parpool(nWorkers); - catch - delete(gcp('nocreate')); - ppool = EMC_parpool(nWorkers); - end + SAVE_IMG(MRCImage(gather(psTile_inv)),sprintf('fixedStacks/ctf/%s-PS_inv.mrc',fileName),pixelSize); + bpLog = fftshift(BH_bandpass3d([size(psTile_inv(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); + bpLog = bpLog > 0.99; + bp = fftshift(BH_bandpass3d([size(psTile_inv(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); + bp2 = fftshift(BH_bandpass3d([size(psTile_inv(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); for iPrj = 1:d3 - - pFuture(iPrj) = parfeval(ppool,@runAvgTiles,2, TLT, paddedSize, tileSize, ... - d1,d2, iPrj, overlap, ... - STACK(:,:,TLT(iPrj,1)), ... - 1, ... - 1, ... - reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine); - + iTile = gpuArray(psTile_inv(:,:,iPrj)); + iTile = iTile.*bp.*bp2; + iTile(~bpLog) = mean(iTile(bpLog)); + psTile_inv(:,:,iPrj) = gather(iTile); end + SAVE_IMG(MRCImage(gather(psTile_inv)),sprintf('fixedStacks/ctf/%s-PS2_inv.mrc',fileName),pixelSize); + + delete(ppool); + delete(gcp('nocreate')) + + + BH_runCtfFind(sprintf('fixedStacks/ctf/%s-PS2',fileName), ... + sprintf('%s_ctf',fileName), ctfParams,TLT) + + end % do_make_tiles + % exit an fit the PS using CTFFIND4 + - for iWorker = 1:d3 - fprintf('Calculating stretched tiles on prj %d/ %d\n',iWorker,d3); - [iPrj, ctfCorr,pixelSize] = fetchNext(pFuture); - - psTile(:,:,TLT(iPrj,1)) = ctfCorr; - end - end % debug without parallel - - pixelSize = pixelSize*10^10; - SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS.mrc',fileName),pixelSize); - bpLog = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); - bpLog = bpLog > 0.99; - bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); - bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); - - for iPrj = 1:d3 - iTile = gpuArray(psTile(:,:,iPrj)); - iTile = iTile.*bp.*bp2; - iTile(~bpLog) = mean(iTile(bpLog)); - psTile(:,:,iPrj) = gather(iTile); - end - - SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName),pixelSize); - delete(ppool); - delete(gcp('nocreate')) - - % exit an fit the PS using CTFFIND4 - BH_runCtfFind(sprintf('fixedStacks/ctf/%s-PS2.mrc',fileName), ... - sprintf('%s_ctf.tlt',fileName), ctfParams,TLT) - - -end end -function [psTile,pixelSize] = runAvgTiles(TLT, paddedSize, tileSize, d1,d2, iPrj, overlap, ... + + +function [psTile_out,psTile_inv_out,pixelSize] = runAvgTiles(TLT, paddedSize, tileSize, d1,d2, iPrj, overlap, ... iProjection, evalMask, ... ddZ, ... reScaleRealSpace,pixelSize,fraction_of_extra_tilt_data,testNoRefine) @@ -282,110 +309,183 @@ oXprj = ceil((size(iProjection,1)+1)./2); % Don't worry about extending the edges for thickness -half_width = (size(iProjection,1)/2); +halfX = emc_get_origin_index(paddedSize); maxEval = (fraction_of_extra_tilt_data + ... - cosd(TLT(iPrj,4)).*(1-fraction_of_extra_tilt_data)) .* half_width; + cosd(TLT(iPrj,4)).*(1-fraction_of_extra_tilt_data)) .* halfX; iEvalMask = floor(oXprj-maxEval):ceil(oXprj+maxEval); +% iEvalMask = BH_multi_gridCoordinates([size(iProjection,1),1,1],'Cartesian','GPU',{'none'},0,1,0); + + +psTile = zeros([halfX,paddedSize,1], 'single','gpuArray'); +psTile_inv = zeros([halfX,paddedSize,1], 'single','gpuArray'); -psTile = zeros(paddedSize.*[1,1], 'single','gpuArray'); % Since I'm enforcing Y-tilt axis, then this could be dramatically sped up % by resampling strips along the sampling +bhF1 = fourierTransformer(zeros([paddedSize,paddedSize],'single','gpuArray')); -for iOuter = 1+tileSize/2:overlap:d1-tileSize/2 - randSize = randi(floor(overlap/2),1); - if (randi(2,1) == 2) - randSize = -1*randSize; - end - i = iOuter + randSize; - if (i < tileSize/2 || i > d1-tileSize/2) - continue; - end - - % Slightly randomize the step size to avoid a Moire like effect that - % presents particulary strongly with a continuous carbon layer. +for tilt_sign = [-1,1] + for iOuter = 1+tileSize/2:overlap:d1-tileSize/2 - iDeltaZ = (i - tiltOrigin)*pixelSize*-1.*tand(TLT(iPrj,4)); - if any(ismember(i-tileSize/2+1:i+tileSize/2,iEvalMask)) %evalMask(i,paddedSize/2+1) - - mag = (1+iDeltaZ./DFo).^0.5; - if ~isfinite(mag) - error('mag is not finite'); + if (iOuter < tileSize/2 || iOuter > d1-tileSize/2) + continue; end - estSize = tileSize(1); - ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); - ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),iDeltaZ+DFo,estSize,TLT(iPrj,19),-1,1); - ctf1 = ctf1(1:estSize/2); - ctf2 = ctf2(1:estSize/2); - firstZero = find(ctf1 > 0, 1, 'first'); - % secondZero= find(ctf1(firstZero:end) < 0 , 1, 'first') + firstZero - 1; - - %fprintf(ftmp,'firstZero %d %2.2f\n',firstZero,estSize/firstZero*pixelSize); - % This range will depend on the size of the field of view. For now, setting manually for Florian's HIV - % data, but will derive a formula to make sure the search is appropriate. Here we expect at most ~ 300 nm - % deltaZ, the strongest difference is at the lowest defocus which is ~ 1500 nm, which gives an estimated mag - % ~ 1.095 - defRange = mag-.1:.001:mag+.1; - nDef = length(defRange); - scoreDef = zeros(nDef,1,'gpuArray'); - for iDef = 1:nDef - ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); - % Larger scalings will have zeros rather than extroplation, so% - % % don't let this influence the score. - % lastZero = find(abs(ci) > 0 , 1, 'last'); - %fprintf(ftmp,'%d %d %d %d',size(ci),size(ctf1)); - scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - - end - [~,maxCoord] = max(scoreDef); - mag = defRange(maxCoord); - - defRange = mag-.01:.0001:mag+.01; - nDef = length(defRange); - scoreDef = zeros(nDef,1,'gpuArray'); - for iDef = 1:nDef - ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); - scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); - end - [~,maxCoord] = max(scoreDef); - mag = defRange(maxCoord); - - if (testNoRefine) - mag = 1; - end - - scaledStrip = iProjection(i-tileSize/2+1:i+tileSize/2,:); + % Slightly randomize the step size to avoid a Moire like effect that + % presents particulary strongly with a continuous carbon layer. - for j = 1+tileSize/2:overlap:d2-tileSize/2 - iTile = fftshift(fftn(scaledStrip(:,j-tileSize/2+1:j+tileSize/2)));%.*coordShift; - - % Slightly randomize scaling - if (randi(2,1) == 2) - scaledSize = ceil(size(iTile) .* mag) + randi(2,1) -1; - else - scaledSize = floor(size(iTile) .* mag)+ randi(2,1) -1; + iDeltaZ = (iOuter - tiltOrigin)*pixelSize*tilt_sign.*tand(TLT(iPrj,4)); + if any(ismember(iOuter-tileSize/2+1:iOuter+tileSize/2,iEvalMask)) %evalMask(iOuter,paddedSize/2+1) + + % The formulat is 1 + deltaDefocus / defocus, and deltaZ = - deltaDefocus + mag = (1-iDeltaZ./DFo).^0.5; + if ~isfinite(mag) + error('mag is not finite'); end - - tile_padVal = BH_multi_padVal(size(iTile),scaledSize); - iTile = real(ifftn(ifftshift(BH_padZeros3d(iTile,'fwd',tile_padVal,'GPU','singleTaper', 0)))); - - iPadVal = BH_multi_padVal(scaledSize,paddedSize.*[1,1]); - - iTile = fftshift(abs(fftn(BH_padZeros3d(iTile, 'fwd', iPadVal, ... - 'GPU','singleTaper', mean(iTile(:)))))); + estSize = tileSize(1); + ctf1 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo,estSize,TLT(iPrj,19),-1,1); + ctf2 = BH_ctfCalc(pixelSize,TLT(iPrj,17),TLT(iPrj,18),DFo-iDeltaZ,estSize,TLT(iPrj,19),-1,1); + ctf1 = ctf1(1:estSize/2); + ctf2 = ctf2(1:estSize/2); + firstZero = find(ctf1 > 0, 1, 'first'); + % secondZero= find(ctf1(firstZero:end) < 0 , 1, 'first') + firstZero - 1; + + %fprintf(ftmp,'firstZero %d %2.2f\n',firstZero,estSize/firstZero*pixelSize); + % This range will depend on the size of the field of view. For now, setting manually for Florian's HIV + % data, but will derive a formula to make sure the search is appropriate. Here we expect at most ~ 300 nm + % deltaZ, the strongest difference is at the lowest defocus which is ~ 1500 nm, which gives an estimated mag + % ~ 1.095 + defRange = mag-.1:.001:mag+.1; + nDef = length(defRange); + scoreDef = zeros(nDef,1,'gpuArray'); + for iDef = 1:nDef + ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); + % Larger scalings will have zeros rather than extroplation, so% + % % don't let this influence the score. + % lastZero = find(abs(ci) > 0 , 1, 'last'); + %fprintf(ftmp,'%d %d %d %d',size(ci),size(ctf1)); + scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); + + end + [~,maxCoord] = max(scoreDef); + mag = defRange(maxCoord); - psTile = psTile + iTile; + defRange = mag-.01:.0001:mag+.01; + nDef = length(defRange); + scoreDef = zeros(nDef,1,'gpuArray'); + for iDef = 1:nDef + ci = interpn([1:estSize/2]',ctf2(1:estSize/2),[1:estSize/2]'./defRange(iDef),'linear',0); + scoreDef(iDef) = sum(ci(firstZero:end).*ctf1(firstZero:end))./sqrt(sum(ci(firstZero:end).^2).*sum(ctf1(firstZero:end).^2)); + end + [~,maxCoord] = max(scoreDef); + mag = defRange(maxCoord); - end % loop over j - end % if over eval mask -end % over tiles + if (testNoRefine) + mag = 1; + end + + reduced_x = floor(tileSize*cosd(TLT(iPrj,4))); + % ---------------+--------------- + % 000000---------+---------000000 + tile_origin_x = emc_get_origin_index(tileSize); + reduced_origin_x = emc_get_origin_index(reduced_x); + zeroed_coords = [1:1+(tile_origin_x-reduced_origin_x),(tile_origin_x+reduced_origin_x):tileSize]; + -psTile = gather(psTile); + + scaled_size = floor([paddedSize,paddedSize] .* mag); + scaled_size = scaled_size + mod(scaled_size,2); + cut_out_padVal = BH_multi_padVal([tileSize,tileSize],paddedSize.*[1,1]); + tile_padVal_RealSpace = BH_multi_padVal([(paddedSize),paddedSize],[(scaled_size(1)), scaled_size(2)]); + scaled_halfX = emc_get_origin_index(scaled_size(1)) + scaled_paddedX = emc_get_origin_index(paddedSize) + tile_padVal_FourierSpace = ([scaled_halfX, scaled_size(2)] - [scaled_paddedX,paddedSize]) ./ 2; + real_resize_increase = all(tile_padVal_RealSpace >= 0); + fourier_resize_increase = all(tile_padVal_FourierSpace >= 0); + padded_fft = zeros([scaled_halfX, scaled_size(2)], 'single','gpuArray'); + [bhF2] = fourierTransformer(zeros(scaled_size,'single','gpuArray')); + + for y = 1+tileSize/2:overlap:d2-tileSize/2 + + + iTile = gpuArray(iProjection(iOuter-tileSize/2+1:iOuter+tileSize/2,y-tileSize/2+1:y+tileSize/2)); + % iTile(zeroed_coords,:) = 0; + iTile = iTile - mean(iTile(:)); + iTile = iTile ./ rms(iTile(:)); + + iTile = BH_padZeros3d(iTile,'fwd',cut_out_padVal,'GPU','singleTaper'); + + % iTile = fftshift(fftn(scaledStrip(:,index_into)));%.*coordShift; + % iTile = gpuArray(scaledStrip(:,y-tileSize/2+1:y+tileSize/2));%.*coordShift; + + iTile = bhF1.swapIndexFWD(bhF1.fwdFFT(iTile)); + + + + if (fourier_resize_increase) + padded_fft = padded_fft .* 0; + padded_fft(1:size(iTile,1),1 + tile_padVal_FourierSpace(2):size(iTile,2) + tile_padVal_FourierSpace(2)) = iTile; + else + padded_fft = iTile(1:scaled_halfX,1 - tile_padVal_FourierSpace(2):size(iTile,2) + tile_padVal_FourierSpace(2)); + end + + iTile = real(bhF2.invFFT(bhF2.swapIndexINV(padded_fft))); + + % % Slightly randomize scaling + % if (randi(2,1) == 2) + % scaledSize = ceil(size(iTile) .* mag) + randi(2,1) -1; + % else + % scaledSize = floor(size(iTile) .* mag)+ randi(2,1) -1; + % end + + % tile_padVal = BH_multi_padVal(size(iTile),scaledSize); + % iTile = real(ifftn(ifftshift(BH_padZeros3d(iTile,'fwd',tile_padVal,'GPU','singleTaper', 0)))); + + + + % Do the final forwardSwap at the end + iTile = bhF1.fwdFFT(BH_padZeros3d(iTile, 'inv', tile_padVal_RealSpace, 'GPU','singleTaper', mean(iTile(:)))); + + % iTile = fftshift(abs(fftn(BH_padZeros3d(iTile, 'fwd', iPadVal, ... + % 'GPU','singleTaper', mean(iTile(:)))))); + + if (tilt_sign == -1) + psTile = psTile + abs(iTile); + else + psTile_inv = psTile_inv + abs(iTile); + end + + + end % loop over y + end % if over eval mask + end % over tiles +end % loop over tilt_sign + +psTile = bhF1.swapIndexFWD(psTile); +psTile_inv = bhF1.swapIndexFWD(psTile_inv); + +psTile = psTile - mean(psTile(:)); +psTile = psTile ./ rms(psTile(:)); +oob = psTile > 3; +psTile(oob) = 3 + rand([sum(oob(:)),1],'single','gpuArray')./3; + +psTile_inv = psTile_inv - mean(psTile_inv(:)); +psTile_inv = psTile_inv ./ rms(psTile_inv(:)); +oob = psTile_inv > 3; +psTile_inv(oob) = 3 + rand([sum(oob(:)),1],'single','gpuArray')./3; + + +psTile_out = gather(BH_multi_makeHermitian(psTile, [paddedSize,paddedSize], 1)); +psTile_inv_out = gather(BH_multi_makeHermitian(psTile_inv, [paddedSize,paddedSize], 1)); + + + +% psTile = gather(psTile); +% psTile_inv = gather(psTile_inv); clear tmpTile iProjection ddZ evalMask Xnew Ynew x1 y1 end diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index ba9c3a05..99751875 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -88,6 +88,14 @@ for iTilt = 1:length(ITER_LIST{iGPU}) STACK_PRFX = ITER_LIST{iGPU}{iTilt}; + + try + l = load(sprintf('fixedStacks/ctf/%s_ali3_ctf.tlt',STACK_PRFX)); + fprintf('FOUND geometry for %s\n',STACK_PRFX); + continue; + catch + fprintf('reloading geometry for %s\n',STACK_PRFX); + end if (mapBackIter) mapBackPrfx = sprintf('mapBack%d/%s_ali%d_ctf',mapBackIter,STACK_PRFX,mapBackIter) @@ -230,6 +238,7 @@ if (mapBackIter) fprintf('Combining tranformations\n\n'); % Load in the mapBack alignment + skip = false; try mbEST = load(sprintf('%s.tltxf',mapBackPrfx)); catch @@ -237,6 +246,7 @@ system(sprintf('cp fixedStacks/ctf/%s_ali1_ctf.tlt fixedStacks/ctf/%s_ali%d_ctf.tlt',STACK_PRFX,STACK_PRFX,mapBackIter+1)); continue; end + mbTLT = load(sprintf('%s.tlt',mapBackPrfx)); defShifts = sprintf('%s.defShifts',mapBackPrfx); if exist(defShifts,'file') @@ -325,7 +335,7 @@ end end else - error('Why would we get here?') + error('ctf update should only be called after tomoCPR (mapBackIter > 0), you can change this with emClarity geometry paramX.m X SwitchCurrentTomoCPR [mapBackIter,0,0] STD'); end diff --git a/ctf/BH_runCtfFind.m b/ctf/BH_runCtfFind.m index 48be404f..7fbe5edd 100644 --- a/ctf/BH_runCtfFind.m +++ b/ctf/BH_runCtfFind.m @@ -1,4 +1,4 @@ -function [ ] = BH_runCtfFind(stackName, tltName, ctfParams, tiltAngles) +function [ ] = BH_runCtfFind(stackNameBaseName, tltNameBaseName, ctfParams, tiltAngles) %Fit the ctf to a background subtracted PS using ctffind4 % CTF params % PixelSize (Ang) @@ -9,83 +9,129 @@ system('mkdir -p fixedStacks/ctf/forCtfFind'); rng('shuffle'); -randPrfx = sprintf('%s_%d',tltName,randi(1e6,[1,1])); +randPrfx = sprintf('%s_%d',tltNameBaseName,randi(1e6,[1,1])); +randPrfx_inv = sprintf('%s_inv',randPrfx); ctfFindPath = getenv('EMC_CTFFIND'); fprintf('%s\n',ctfFindPath);% split the stack up -fullStack = OPEN_IMG('single', stackName); +fullStack = OPEN_IMG('single', sprintf('%s.mrc',stackNameBaseName)); +fullStack_inv = OPEN_IMG('single', sprintf('%s_inv.mrc',stackNameBaseName)); [d1,d2,d3] = size(fullStack); % FIXME d1 assumed to equal d2 Add check in saving for iPrj = 1:d3 SAVE_IMG(MRCImage(fullStack(:,:,iPrj)),sprintf('fixedStacks/ctf/forCtfFind/%s_%d.mrc',randPrfx,iPrj)); + SAVE_IMG(MRCImage(fullStack_inv(:,:,iPrj)),sprintf('fixedStacks/ctf/forCtfFind/%s_%d.mrc',randPrfx_inv,iPrj)); end % % Check to make sure this hasn't alread been done -% if ~exist(sprintf('fixedStacks/ctf/%s_orig',tltName), 'file') -system(sprintf('mv fixedStacks/ctf/%s fixedStacks/ctf/%s_orig',tltName,tltName)); +% if ~exist(sprintf('fixedStacks/ctf/%s_orig',tltNameBaseName), 'file') +system(sprintf('mv fixedStacks/ctf/%s.tlt fixedStacks/ctf/%s.tlt_orig',tltNameBaseName,tltNameBaseName)); % end -tmpTLT = load(sprintf('fixedStacks/ctf/%s_orig',tltName)); +tmpTLT = load(sprintf('fixedStacks/ctf/%s.tlt_orig',tltNameBaseName)); meanDefocus = mean(abs(tmpTLT(:,15)))*10^10; fprintf('Searching around an estimated mean defocus of %3.6f Angstrom\n'); % write the run script, this should link to a distributed version with % special name, but for testing use the beta. -scriptName = sprintf('.%s.sh',randPrfx); -fID = fopen(scriptName,'w'); - -fprintf(fID,'#!/bin/bash\n\n'); -for iPrj = 1:d3 - % I want to fit to lower resolution at higher tilts - tltIDX = find(tiltAngles(:,1) == iPrj); - - % put in a line to limit number of cores, or use the threaded version - fprintf(fID,'\n%s --amplitude-spectrum-input << eof &',ctfFindPath); - fprintf(fID,'\nfixedStacks/ctf/forCtfFind/%s_%d.mrc\n',randPrfx,iPrj); - fprintf(fID,'fixedStacks/ctf/forCtfFind/%s_diagnostic_%d.mrc\n',randPrfx,iPrj); - fprintf(fID,'%f\n%f\n%f\n%f\n%d\n%f\n%f\n%d\n%d\n%d\n', ... - ctfParams(1:4), ... - d1, ... - 30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)).^0.4,... - 0.75*meanDefocus,... - 1.25*meanDefocus,... - 25.0); - fprintf(fID,'no\nno\nyes\n500.0\nno\nno\nno\neof\n\n'); -end -fprintf(fID,'wait\n'); -fclose(fID); +score = 0; +score_inv = 0; +for i_run = [1:2] + if (i_run == 1) + % regular + using_prfx = randPrfx; + else + using_prfx = randPrfx_inv; + % inverse + end -system(sprintf('chmod a=wrx %s',scriptName)); + scriptName = sprintf('.%s.sh',using_prfx); + + + fID = fopen(scriptName,'w'); + + fprintf(fID,'#!/bin/bash\n\n'); + for iPrj = 1:d3 + % I want to fit to lower resolution at higher tilts + tltIDX = find(tiltAngles(:,1) == iPrj); + + % put in a line to limit number of cores, or use the threaded version + fprintf(fID,'\n%s --amplitude-spectrum-input << eof &',ctfFindPath); + fprintf(fID,'\nfixedStacks/ctf/forCtfFind/%s_%d.mrc\n',using_prfx,iPrj); + fprintf(fID,'fixedStacks/ctf/forCtfFind/%s_diagnostic_%d.mrc\n',using_prfx,iPrj); + fprintf(fID,'%f\n%f\n%f\n%f\n%d\n%f\n%f\n%d\n%d\n%d\n', ... + ctfParams(1:4), ... + d1, ... + 30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)).^0.4,... + 0.75*meanDefocus,... + 1.25*meanDefocus,... + 25.0); + fprintf(fID,'no\nno\nyes\n500.0\nno\nno\nno\neof\n\n'); + end + fprintf(fID,'wait\n'); + fclose(fID); -[runFail] = system(sprintf('./%s',scriptName)); + system(sprintf('chmod a=wrx %s',scriptName)); -if (runFail) - system(sprintf('cp ./%s tmpFail',scriptName)); - system(sprintf('mv tmpFail ./%s',scriptName)); [runFail] = system(sprintf('./%s',scriptName)); + if (runFail) - error('Tried to run %s twice and failed\n',scriptName); + system(sprintf('cp ./%s tmpFail',scriptName)); + system(sprintf('mv tmpFail ./%s',scriptName)); + [runFail] = system(sprintf('./%s',scriptName)); + if (runFail) + error('Tried to run %s twice and failed\n',scriptName); + end end -end -% will this wait for return? -baseName = sprintf('fixedStacks/ctf/forCtfFind/%s_diagnostic_',randPrfx); -tmpName = sprintf('fixedStacks/ctf/forCtfFind/%s_tmp',randPrfx); -system(sprintf('newstack %s?.mrc %s??.mrc %s_full.st',baseName,baseName,baseName)); -system(sprintf('rm %s?.mrc %s??.mrc',baseName,baseName)); -system(sprintf('rm -f %s',tmpName)); + % will this wait for return? -for iPrj = 1:d3 - % 2024 Jan, finally make switch to record positive for underfocus as is used internally. - system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10, 3.1415926535/180.0*$4, 1*(($2+$3)/2)*10^-10 }'' >> %s', baseName,iPrj,tmpName)); - -end + baseName = sprintf('fixedStacks/ctf/forCtfFind/%s_diagnostic_',using_prfx); + tmpName = sprintf('fixedStacks/ctf/forCtfFind/%s_tmp',using_prfx); + if (i_run == 1) + using_tltName = sprintf('%s.tlt',tltNameBaseName); + else + using_tltName = sprintf('%s_inv.tlt',tltNameBaseName); + end -% TODO ground truth to confirm orientation of astigmatism + system(sprintf('newstack %s?.mrc %s??.mrc %sfull.st',baseName,baseName,baseName)); + system(sprintf('rm %s?.mrc %s??.mrc',baseName,baseName)); + % system(sprintf('rm -f %s?.mrc %s??.mrc',using_prfx,using_prfx)); + system(sprintf('rm -f %s',tmpName)); + for iPrj = 1:d3 + % 2024 Jan, finally make switch to record positive for underfocus as is used internally. + system(sprintf('tail -n -1 %s%d.txt | awk ''{print (($2-$3)/2)*10^-10, 3.1415926535/180.0*$4, 1*(($2+$3)/2)*10^-10, $6 }'' >> %s', baseName,iPrj,tmpName)); + + end + % TODO ground truth to confirm orientation of astigmatism + + system(sprintf('awk ''FNR==NR{a[FNR]=$1;b[FNR]=$2;c[FNR]=$3 ;next}{ print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,a[$1],b[$1],$14,c[$1],$16,$17,$18,$19,$20,$21,$22,$23}'' %s fixedStacks/ctf/%s.tlt_orig > fixedStacks/ctf/%s',tmpName,tltNameBaseName,using_tltName)); + + a = importdata(tmpName); + if (i_run == 1) + % regular + score = mean(a(:,4)); + else + % inverse + score_inv = mean(a(:,4)); + end +end % loop on reg/inv + +% Save the scores in +fprintf('Found an average score: %3.6f and an average inverted hand score: %3.6f for tilt %s\n',score,score_inv, tltNameBaseName); +if (score_inv > score) + fprintf('It looks like your handedness is inverted based on tiles.\n'); +end + +% Clean up the input slices (the stacks are still at fixedStacks/ctf/...PS-2.mrc) +for iPrj = 1:d3 + system(sprintf('rm -f fixedStacks/ctf/forCtfFind/%s_%d.mrc',randPrfx,iPrj)); + system(sprintf('rm -f fixedStacks/ctf/forCtfFind/%s_%d.mrc',randPrfx_inv,iPrj)); +end -system(sprintf('awk ''FNR==NR{a[FNR]=$1;b[FNR]=$2;c[FNR]=$3 ;next}{ print $1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,a[$1],b[$1],$14,c[$1],$16,$17,$18,$19,$20,$21,$22,$23}'' %s fixedStacks/ctf/%s_orig > fixedStacks/ctf/%s',tmpName,tltName,tltName)); +end % function \ No newline at end of file diff --git a/logicals/EMC_assert_deprecated_substitution.m b/logicals/EMC_assert_deprecated_substitution.m index 3ffa635d..d0d89b8f 100644 --- a/logicals/EMC_assert_deprecated_substitution.m +++ b/logicals/EMC_assert_deprecated_substitution.m @@ -1,15 +1,13 @@ -function [parameter_struct] = EMC_assert_deprecated_substitution(parameter_struct, default_value, current_field, deprecated_field) +function [parameter_struct] = EMC_assert_deprecated_substitution(parameter_struct, current_field, deprecated_field) % Copying this stuct around is probaby not the most efficient way to do this % but accuracy is more important than speed here. % Handle type checks and default settings outside. - if isfield(parameter_struct, deprecated_field) - parameter_struct.(current_field) = parameter_struct.(deprecated_field); - parameter_struct = rmfield(parameter_struct, deprecated_field); - else - if ~isfield(parameter_struct, current_field) - parameter_struct.(current_field) = default_value; + if ~isfield(parameter_struct, current_field) + if isfield(parameter_struct, deprecated_field) + parameter_struct.(current_field) = parameter_struct.(deprecated_field); + parameter_struct = rmfield(parameter_struct, deprecated_field); end end diff --git a/masking/BH_bandLimitCenterNormalize.m b/masking/BH_bandLimitCenterNormalize.m index fab524fd..5a6077b6 100755 --- a/masking/BH_bandLimitCenterNormalize.m +++ b/masking/BH_bandLimitCenterNormalize.m @@ -24,10 +24,12 @@ % % use double precision to calculate the mean, and after masking. -applyBandpass = 0; +applyBandpass = false; applyShift = 0; if isnumeric(BANDPASS) - applyBandpass = 1; + if numel(BANDPASS) > 1 + applyBandpass = true; + end elseif strcmpi(BANDPASS, 'shift') % use the mask to shift the image prior to fft as needed for fourier % interp. diff --git a/masking/BH_bandpass3d.m b/masking/BH_bandpass3d.m index b5d50573..fa0cc64b 100755 --- a/masking/BH_bandpass3d.m +++ b/masking/BH_bandpass3d.m @@ -91,7 +91,7 @@ % initialize nd grids of appropriate size -[ radius,~,~,~,~,~] = BH_multi_gridCoordinates( bSize, 'Cartesian', METHOD, ... + [ radius,~,~,~,~,~] = BH_multi_gridCoordinates( bSize, 'Cartesian', METHOD, ... {'single',... [1,0,0;0,1,0;0,0,1],... [0,0,0]','forward',1,1}, ... diff --git a/masking/BH_weightMaskMex.m b/masking/BH_weightMaskMex.m index d477c593..ee3ec366 100644 --- a/masking/BH_weightMaskMex.m +++ b/masking/BH_weightMaskMex.m @@ -20,6 +20,8 @@ reconGeometry.NY, ... reconGeometry.NZ]); +% Using the nomenclature that in frame means from that frames lower left +% wrt origin means from that frames center, a bit confusing, subtomo_origin_in_specimen_frame = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_tilt_origin; iCs = single(TLT(:,17).*10^3); diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index c16ac758..0f50347c 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -24,7 +24,8 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... try % should be a text file of only x,y,z (model2points imodModel classes.txt) % or - % a list of names of tomograms to remove + % a list of names of tomograms to remove + % FIXME if any path this will not work. if ~(strcmp(VECTOR_OP,'tomoList.txt')) [~,modNAME,~] = fileparts(VECTOR_OP); system(sprintf('model2point %s %s.txt > /dev/null',VECTOR_OP,modNAME)); @@ -228,22 +229,21 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... system('rm cache/*.wgt*'); system('rm cache/*.rec*'); end - for iTomo = 1:nTomograms - STACK_PRFX = ... - masterTM.mapBackGeometry.tomoName.(tomoList{iTomo}).tiltName; - - - newTLT = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', ... - STACK_PRFX,mapBackIter+1); + [STACK_LIST, nTiltSeries] = BH_returnIncludedTilts( masterTM.mapBackGeometry ); - - geometry.(tomoList{iTomo}) = load(newTLT); - fprintf('Updating TLT %s\n', newTLT); - + for iStack = 1:length(STACK_LIST) + STACK_PRFX = STACK_LIST{iStack}; + tomo_names = masterTM.mapBackGeometry.(STACK_PRFX).tomoList; + for iTomo = 1:length(tomo_names) - clear newTLT STACK_PRFX + newTLT = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', STACK_PRFX,mapBackIter+1); + geometry.(tomo_names{iTomo}) = load(newTLT); + fprintf('Updating TLT %s\n', newTLT); + end end + + case 'WriteCsv' !mkdir -p csv @@ -479,6 +479,8 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... end cccCutOff = sortCCC(cutIDX) + sortCCC(1) + sortCCC(end) nRemoved = 0; for iTomo = 1:nTomograms diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 16b01f99..e4db7fcf 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -182,6 +182,17 @@ emc.filterDefocus = [0.0, 0.0]; end +if isfield(emc, 'refine_defocus_cisTEM') + EMC_assert_boolean(emc.refine_defocus_cisTEM) +else + emc.refine_defocus_cisTEM = false; +end +if isfield(emc, 'rerun_refinement_cisTEM') + EMC_assert_boolean(emc.rerun_refinement_cisTEM) +else + emc.rerun_refinement_cisTEM = false; +end + if isfield(emc,'flgCutOutVolumes') EMC_assert_boolean(emc.flgCutOutVolumes) else @@ -220,29 +231,52 @@ % TODO: these should maybe be two different orthogonal parameters % if > 1 keep this many subtomos % if < 1 keep this fraction -emc = EMC_assert_deprecated_substitution(emc, 0.0, 'ccc_cutoff', 'flgCCCcutoff'); -EMC_assert_numeric(emc.ccc_cutoff,1) - +emc = EMC_assert_deprecated_substitution(emc, 'ccc_cutoff', 'flgCCCcutoff'); +if isfield(emc, 'ccc_cutoff') + EMC_assert_numeric(emc.ccc_cutoff, 1) +else + emc.('ccc_cutoff') = 0.0; +end % TOOD: DOC -emc = EMC_assert_deprecated_substitution(emc, false, 'projectVolumes', 'flgProjectVolumes'); -EMC_assert_boolean(emc.projectVolumes); +emc = EMC_assert_deprecated_substitution(emc, 'projectVolumes', 'flgProjectVolumes'); +if isfield(emc, 'projectVolumes') + EMC_assert_boolean(emc.projectVolumes); +else + emc.('projectVolumes') = false; +end % Whether the cycle is expected to be used for classification or alignment. % Eventually, the distinction should not matter. -emc = EMC_assert_deprecated_substitution(emc, false, 'classification', 'flgClassify'); -EMC_assert_boolean(emc.classification); +emc = EMC_assert_deprecated_substitution(emc, 'classification', 'flgClassify'); +if isfield(emc, 'classification') + EMC_assert_boolean(emc.classification); +else + emc.('classification') = false; +end -emc = EMC_assert_deprecated_substitution(emc, 0, 'multi_reference_alignment', 'flgMultiRefAlignment'); -EMC_assert_numeric(emc.multi_reference_alignment, 1, [0, 2]); +emc = EMC_assert_deprecated_substitution(emc, 'multi_reference_alignment', 'flgMultiRefAlignment'); +if isfield(emc, 'multi_reference_alignment') + EMC_assert_numeric(emc.multi_reference_alignment, 1, [0, 2]); +else + emc.('multi_reference_alignment') = 0; +end % Zero padding of the volumes before alignment/other FFT ops -emc = EMC_assert_deprecated_substitution(emc, 1.5, 'scale_calc_size', 'scaleCalcSize'); -EMC_assert_numeric(emc.scale_calc_size, 1, [1.0, 2.0]); +emc = EMC_assert_deprecated_substitution(emc, 'scale_calc_size', 'scaleCalcSize'); +if isfield(emc, 'scale_calc_size') + EMC_assert_numeric(emc.scale_calc_size, 1, [1.0, 2.0]); +else + emc.('scale_calc_size') = 1.5; +end -emc = EMC_assert_deprecated_substitution(emc, false, 'limit_to_one_core', 'flgLimitToOneProcess'); -EMC_assert_boolean(emc.limit_to_one_core); +emc = EMC_assert_deprecated_substitution(emc, 'limit_to_one_core', 'flgLimitToOneProcess'); +if isfield(emc, 'limit_to_one_core') + EMC_assert_boolean(emc.limit_to_one_core); +else + emc.('limit_to_one_core') = false; +end if (emc.limit_to_one_core) emc.nCpuCores = 1; @@ -267,14 +301,26 @@ emc.Pca_constrain_symmetry = false; end -emc = EMC_assert_deprecated_substitution(emc, false, 'fsc_with_chimera', 'fscWithChimera'); -EMC_assert_boolean(emc.fsc_with_chimera); +emc = EMC_assert_deprecated_substitution(emc, 'fsc_with_chimera', 'fscWithChimera'); +if isfield(emc, 'fsc_with_chimera') + EMC_assert_boolean(emc.fsc_with_chimera); +else + emc.fsc_with_chimera = false; +end -emc = EMC_assert_deprecated_substitution(emc, 0.1, 'minimum_particle_for_fsc_weighting', 'minimumparticleVolume'); -EMC_assert_numeric(emc.minimum_particle_for_fsc_weighting, 1, [0.01, 1.0]); +emc = EMC_assert_deprecated_substitution(emc, 'minimum_particle_for_fsc_weighting', 'minimumparticleVolume'); +if isfield(emc, 'minimum_particle_for_fsc_weighting') + EMC_assert_numeric(emc.minimum_particle_for_fsc_weighting, 1, [0.01, 1.0]); +else + emc.('minimum_particle_for_fsc_weighting') = 0.1; +end -emc = EMC_assert_deprecated_substitution(emc, 1.0, 'fsc_shape_mask', 'flgFscShapeMask'); -EMC_assert_numeric(emc.fsc_shape_mask, 1, [0.0, 2.0]); +emc = EMC_assert_deprecated_substitution(emc, 'fsc_shape_mask', 'flgFscShapeMask'); +if isfield(emc, 'fsc_shape_mask') + EMC_assert_numeric(emc.fsc_shape_mask, 1, [0.0, 2.0]); +else + emc.fsc_shape_mask = 1.0; +end if isfield(emc, 'shape_mask_lowpass') EMC_assert_numeric(emc.shape_mask_lowpass, 1, [10, 100]); @@ -294,8 +340,12 @@ emc.shape_mask_test = false; end -emc = EMC_assert_deprecated_substitution(emc, 22.0, 'pca_scale_spaces', 'pcaScaleSpace'); -EMC_assert_numeric(emc.pca_scale_spaces); +emc = EMC_assert_deprecated_substitution(emc, 'pca_scale_spaces', 'pcaScaleSpace'); +if isfield(emc, 'pca_scale_spaces') + EMC_assert_numeric(emc.pca_scale_spaces); +else + emc.pca_scale_spaces = 22.0; +end emc.('n_scale_spaces') = numel(emc.pca_scale_spaces); if isfield(emc, 'Pca_maxEigs') @@ -337,6 +387,11 @@ emc.Pca_flattenEigs = true; end +if isfield(emc, 'Pca_use_real_space_conv') + EMC_assert_boolean(emc.Pca_use_real_space_conv); +else + emc.Pca_use_real_space_conv = false; +end if isfield(emc, 'Pca_som_coverSteps') EMC_assert_numeric(emc.Pca_som_coverSteps, 1, [1, 1000]); @@ -362,15 +417,24 @@ end -emc = EMC_assert_deprecated_substitution(emc, false, 'update_class_by_ccc', 'updateClassByBestReferenceScore'); -EMC_assert_boolean(emc.update_class_by_ccc); +emc = EMC_assert_deprecated_substitution(emc, 'update_class_by_ccc', 'updateClassByBestReferenceScore'); +if isfield(emc, 'update_class_by_ccc') + EMC_assert_boolean(emc.update_class_by_ccc); +else + emc.('update_class_by_ccc') = true; +end + if (~emc.multi_reference_alignment) % update by ccc only makes sense for multi reference alignment emc.update_class_by_ccc = false; end -emc = EMC_assert_deprecated_substitution(emc, true, 'move_reference_by_com', 'flgCenterRefCOM'); -EMC_assert_boolean(emc.move_reference_by_com); +emc = EMC_assert_deprecated_substitution(emc, 'move_reference_by_com', 'flgCenterRefCOM'); +if isfield(emc, 'move_reference_by_com') + EMC_assert_boolean(emc.move_reference_by_com); +else + emc.('move_reference_by_com') = true; +end if isfield(emc, 'use_new_grid_search') EMC_assert_boolean(emc.use_new_grid_search); @@ -381,8 +445,12 @@ %%%%%%%%%%%%%%%%%%%%%%%%%% tomoCPR params, mostly experimental -emc = EMC_assert_deprecated_substitution(emc, false, 'save_mapback_classes', 'flgColorMap'); -EMC_assert_boolean(emc.save_mapback_classes); +emc = EMC_assert_deprecated_substitution(emc, 'save_mapback_classes', 'flgColorMap'); +if isfield(emc, 'save_mapback_classes') + EMC_assert_boolean(emc.save_mapback_classes); +else + emc.('save_mapback_classes') = false; +end if isfield(emc, 'only_use_reference_classes') EMC_assert_boolean(emc.only_use_reference_classes); @@ -443,7 +511,7 @@ if isfield(emc, 'tomoCPR_target_n_patches_x_y') EMC_assert_numeric(emc.tomoCPR_target_n_patches_x_y, 2, [0, 100]); else - emc.tomoCPR_target_n_patches_x_y = [0,0]; + emc.tomoCPR_target_n_patches_x_y = [2,2]; end % I think this has been removed if isfield(emc, 'probabilityPeakiness') @@ -557,14 +625,26 @@ emc.shift_z_to_to_centroid = true; end -emc = EMC_assert_deprecated_substitution(emc, 500e-9, 'tomo_cpr_defocus_range', 'tomoCprDefocusRange'); -EMC_assert_numeric(emc.tomo_cpr_defocus_range, 1, [0.0, 10000e-9]); +emc = EMC_assert_deprecated_substitution(emc, 'tomo_cpr_defocus_range', 'tomoCprDefocusRange'); +if isfield(emc, 'tomo_cpr_defocus_range') + EMC_assert_numeric(emc.tomo_cpr_defocus_range, 1, [0.0, 10000e-9]); +else + emc.tomo_cpr_defocus_range = 500e-9; +end -emc = EMC_assert_deprecated_substitution(emc, 100e-9, 'tomo_cpr_defocus_step', 'tomoCprDefocusStep'); -EMC_assert_numeric(emc.tomo_cpr_defocus_step, 1, [1.0e-9, 10000e-9]); +emc = EMC_assert_deprecated_substitution(emc, 'tomo_cpr_defocus_step', 'tomoCprDefocusStep'); +if isfield(emc, 'tomo_cpr_defocus_step') + EMC_assert_numeric(emc.tomo_cpr_defocus_step, 1, [1.0e-9, 10000e-9]); +else + emc.tomo_cpr_defocus_step = 100e-9; +end -emc = EMC_assert_deprecated_substitution(emc, false, 'tomo_cpr_defocus_refine', 'calcCTF'); -EMC_assert_boolean(emc.tomo_cpr_defocus_refine); +emc = EMC_assert_deprecated_substitution(emc, 'tomo_cpr_defocus_refine', 'calcCTF'); +if isfield(emc, 'tomo_cpr_defocus_refine') + EMC_assert_boolean(emc.tomo_cpr_defocus_refine); +else + emc.tomo_cpr_defocus_refine = false; +end if isfield(emc, 'print_alignment_stats') EMC_assert_boolean(emc.print_alignment_stats); @@ -603,6 +683,25 @@ emc.n_tilt_workers = 4; end +if isfield(emc,'useSurfaceFit') + EMC_assert_boolean(emc.useSurfaceFit) +else + emc.useSurfaceFit = false; +end + +if isfield(emc, 'test_flip_defocus_offset') + EMC_assert_boolean(emc.test_flip_defocus_offset) +else + emc.test_flip_defocus_offset = false; +end + +if isfield(emc, 'test_flip_tilt_offset') + EMC_assert_boolean(emc.test_flip_tilt_offset) +else + emc.test_flip_tilt_offset = false; +end + + % Number of tiltalign processes to run in parallel in tomoCPR % For now, default to zero and manually re-run while sorting out the % optimization process @@ -612,6 +711,15 @@ emc.run_tomocpr_alignments = 0; end +% Generally useful for single particle like projects. If there is substantial density not related to the specimen, +% this will not be so useful as the measure defocus will come largely from those. +% Since most people are using other software for high-res in vitro work, default this to false now. +if isfield(emc, 'set_defocus_origin_using_subtomos') + EMC_assert_boolean(emc.set_defocus_origin_using_subtomos) +else + emc.set_defocus_origin_using_subtomos = false; +end + if isfield(emc, 'max_ctf3dDepth') EMC_assert_numeric(emc.max_ctf3dDepth, 1, [1 * 10^-9, 1000 * 10^-9]); else @@ -653,4 +761,36 @@ EMC_assert_numeric(emc.Pca_bandpass, 3); else emc.Pca_bandpass = [0.001, 1200, 28]; -end \ No newline at end of file +end + +if isfield(emc, 'autoAli_switchAxes') + EMC_assert_boolean(emc.autoAli_switchAxes); +else + emc.autoAli_switchAxes = true; +end + +if isfield(emc, 'ctf_tile_size') + EMC_assert_numeric(emc.ctf_tile_size, 1); +else + emc.ctf_tile_size = floor(680e-10 / emc.pixel_size_si); +end +emc.ctf_tile_size = emc.ctf_tile_size + mod(emc.ctf_tile_size,2); + + +if isfield(emc, 'deltaZTolerance') + EMC_assert_numeric(emc.deltaZTolerance, 1, [10e-9, 300e-9]); +else + emc.deltaZTolerance = 100e-9; +end + +if isfield(emc, 'zShift') + EMC_assert_numeric(emc.zShift, 1, [100e-9, 300e-9]); +else + emc.zShift = 150e-9; +end + +if isfield(emc, 'ctfMaxNumberOfTiles') + EMC_assert_numeric(emc.ctfMaxNumberOfTiles, 1); +else + emc.('ctfMaxNumberOfTiles') = 10000; +end diff --git a/mexFiles/mexFP16.cu b/mexFiles/mexFP16.cu index 741420a8..034a7dac 100644 --- a/mexFiles/mexFP16.cu +++ b/mexFiles/mexFP16.cu @@ -136,7 +136,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { mexFP16_DEBUG_PRINT("Copying single to host\n"); checkCudaErrors(cudaMallocHost(&temporary_single, *n_elements * sizeof(float))); checkCudaErrors(cudaMemcpy(temporary_single, input_single, *n_elements * sizeof(float), cudaMemcpyDeviceToHost)); - checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); +checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); } else temporary_single = input_single; @@ -170,7 +170,7 @@ void mexFunction(int nlhs, mxArray *plhs[], int nrhs, mxArray const *prhs[]) { if (!half_array_is_on_gpu) checkCudaErrors(cudaFreeAsync(temporary_uint16, cudaStreamPerThread)); - checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); +checkCudaErrors(cudaStreamSynchronize(cudaStreamPerThread)); } else { mexFP16_DEBUG_PRINT("Casting to single that is on host already\n"); diff --git a/mexFiles/mexSF3D.cu b/mexFiles/mexSF3D.cu index 59604f5e..848b0805 100644 --- a/mexFiles/mexSF3D.cu +++ b/mexFiles/mexSF3D.cu @@ -45,7 +45,7 @@ __global__ void sf3dKernel(const cudaTextureObject_t tex_obj, // The centered coordinate in the 3d volume float x_centered = float(x - dims.x/2); - tw_pre = -x_centered * sinAcosA.x + size_shift.z; + tw_pre = x_centered * -sinAcosA.x + size_shift.z; float y_centered_normalized = (float(y - dims.y/2) + size_shift.y) / float(dims.y) + 0.5f; for (int z = 0; z < dims.z; z++) { diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 64e195fd..839d78d7 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -150,6 +150,10 @@ %%% Put this in the param file later - the input values should be in angstrom %%% and are the relevant scale spaces for classification. +use_notch_filter = true; +if (emc.Pca_use_real_space_conv) + use_notch_filter = false; +end samplingRate = emc.('Cls_samplingRate'); refSamplingRate= emc.('Ali_samplingRate'); @@ -247,12 +251,14 @@ [ maskType, maskSize, maskRadius, maskCenter ] = ... - BH_multi_maskCheck(emc, 'Cls', pixelSize); + BH_multi_maskCheck(emc, 'Cls', pixelSize) + [ preMaskType, preMaskSize, preMaskRadius, preMaskCenter ] = ... BH_multi_maskCheck(emc, 'Ali', refPixelSize); -% This is prob not a good way to make sure the mask size matches::w +% The size changes based on the radius, but we want the size to match the size from averaging, +% so use the aliMask size with the clsMask radius maskSize=preMaskSize; % Make sure everthing matches the extracted average and wedge @@ -509,14 +515,19 @@ % radius, convert Ang to pix , denom = equiv stdv from normal to include, e.g. % for 95% use 1/sig = 1/2 %stdDev = 1/2 .* (emc.pca_scale_spaces ./ pixelSize - 1) .* 3.0./log(emc.pca_scale_spaces) + if ~(test_multi_ref_diffmap) threeSigma = 1/3 .* (emc.pca_scale_spaces ./ pixelSize); for iScale = 1:emc.n_scale_spaces - kernelSize = ceil(threeSigma(iScale).*3) + 3; - kernelSize = kernelSize + (1-mod(kernelSize,2)); - % masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); - masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'cpu', {}); + if (use_notch_filter) + masks.('scaleMask').(sprintf('s%d',iScale)) = gather(BH_bandpass3d(sizeMask,0.1, emc.pca_scale_spaces(iScale)*1.1, emc.pca_scale_spaces(iScale)*0.9,'GPU',pixelSize)); + else + kernelSize = ceil(threeSigma(iScale).*3) + 3; + kernelSize = kernelSize + (1-mod(kernelSize,2)); + % masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,kernelSize], threeSigma(iScale), 'cpu', {}); + masks.('scaleMask').(sprintf('s%d',iScale)) = EMC_gaussianKernel([1,1,1].*kernelSize, 2*threeSigma(iScale), 'cpu', {}); + end % SAVE_IMG( masks.('scaleMask').(sprintf('s%d',iScale)), ... @@ -555,7 +566,11 @@ tmp_avg = tmp_avg .* masks.('volMask').(sprintf('h%d',iGold)).(sprintf('s%d',iScale)); % FIXME: ideally we would do both, but for testing I am stealing scaleSpace for iRef if ~(test_multi_ref_diffmap) - tmp_avg = EMC_convn(single(gpuArray(tmp_avg)) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); + if (use_notch_filter) + tmp_avg = real(ifftn(fftn(single(gpuArray(tmp_avg))) .* single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) ))); + else + tmp_avg = EMC_convn(single(gpuArray(tmp_avg)) , single(gpuArray(masks.('scaleMask').(sprintf('s%d',iScale))) )); + end end avgMotif_FT{iGold, iScale} = ... BH_bandLimitCenterNormalize(tmp_avg,... @@ -578,6 +593,7 @@ clear montOUT + % If emc.Pca_randSubset is string with a previous matfile use this, without any % decomposition. @@ -651,6 +667,9 @@ gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); end gpuMasks.('highPass').(stSCALE) = gpuArray(masks.('highPass').(stSCALE)); + if (use_notch_filter) + gpuMasks.('highPass').(stSCALE) = gpuMasks.('highPass').(stSCALE) .* gpuArray(masks.('scaleMask').(stSCALE)); + end end @@ -702,6 +721,9 @@ tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); fprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName); + if (iTomo ~= 3) + continue; %revertH68_1_label_101_2H68_1_label_101_2 + end % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); @@ -831,28 +853,27 @@ - iTrimParticle = iParticle(padWindow(1,1)+1 : end - padWindow(2,1), ... - padWindow(1,2)+1 : end - padWindow(2,2), ... - padWindow(1,3)+1 : end - padWindow(2,3)); + iParticle = iParticle(padWindow(1,1)+1 : end - padWindow(2,1), ... + padWindow(1,2)+1 : end - padWindow(2,2), ... + padWindow(1,3)+1 : end - padWindow(2,3)); for iScale = 1:emc.n_scale_spaces - if (test_multi_ref_diffmap) - iPrt = iTrimParticle; - else - iPrt = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); + iTrimParticle = iParticle; + if ~(test_multi_ref_diffmap) && ~(use_notch_filter) + iTrimParticle = EMC_convn(iTrimParticle , gpuMasks.('scaleMask').(sprintf('s%d',iScale))); end - - iPrt = BH_bandLimitCenterNormalize( ... - iPrt .* ... + % if using the notch filter, it is integrated with the highPass at this point + iTrimParticle = BH_bandLimitCenterNormalize( ... + iTrimParticle .* ... gpuMasks.('volMask').(sprintf('s%d',iScale)), ... gpuMasks.('highPass').(sprintf('s%d',iScale)),... gpuMasks.('binary').(sprintf('s%d',iScale)),... [0,0,0;0,0,0],'single'); - [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iPrt,ifftshift(iWedge),... + [iWmd,~] = BH_diffMap(avgMotif_FT{iGold, iScale},iTrimParticle,ifftshift(iWedge),... flgNorm,pixelSize,radialMask, padWdg); @@ -866,7 +887,7 @@ end end % loop on scale spaces - clear iAvg iWmd iTrimParticle + clear iAvg iWmd diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index ccbf7f5f..ea813c07 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -142,7 +142,9 @@ calcCTF = emc.('tomo_cpr_defocus_refine'); -[tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); +[tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ) + + % if (multi_node_run) % [ nParProcesses, iterList] = BH_multi_parallelJobs(nTiltSeries, nGPUs, sizeCalc(1), emc.nCpuCores, [cycle_numerator,cycle_denominator]); @@ -237,14 +239,17 @@ mbOUT = {[tmpCache],[mapBackIter+1],'dummy'}; for iTiltSeries = tiltStart:nTiltSeries + skip_this_tilt_series_bc_alignments_are_bonkers = false; if (skip_to_the_end_and_run) continue; end + + - mapBackRePrjSize = min(64,subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize')) - + % mapBackRePrjSize = min(64,subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize')) + mapBackRePrjSize = 512 % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos if nTomograms == 0 @@ -360,7 +365,8 @@ EMC_parpool(nWorkers); else EMC_parpool(nWorkers); - end + end + fprintf('init with %d workers\n',nWorkers); @@ -475,6 +481,32 @@ % Track the number of fiducials in order to scale the K-factor to more or less % aggressivley downweight outliers in the alignment + + % In-case we want to restrict the number of fiducials, we need to look at all the tomograms ahead of time + positionList= cell(nTomograms,1); + n_possible_particles = 0; + for iTomo = 1:nTomograms + positionList{iTomo} = geometry.(tomoList{iTomo}); + positionList{iTomo} = positionList{iTomo}(positionList{iTomo}(:,26) ~= -9999,:); + n_possible_particles = n_possible_particles + size(positionList{iTomo},1); + end + + % Now check to see if we need to limit the number of fiducials + if (emc.tomoCPR_random_subset == -1 || emc.tomoCPR_random_subset > n_possible_particles) + fprintf('Using all of the %d available fiducials\n',size(positionList{iTomo},1)); + else + fprintf('Using a random subset of %d fiducials from the %d available\n',... + emc.tomoCPR_random_subset, n_possible_particles); + + for iTomo = 1:nTomograms + % Limit the number in each tomo proportional to the number of + n_to_keep = floor(size(positionList{iTomo},1) * emc.tomoCPR_random_subset ./ n_possible_particles) + 1; + keepFids = datasample(1:size(positionList{iTomo},1),n_to_keep,'Replace',false); + positionList{iTomo} = positionList{iTomo}(keepFids,:); + end + end + + nFidsTotal = 0; fidIDX = 0; for iTomo = 1:nTomograms @@ -518,10 +550,11 @@ fprintf(iXF,'%f %f %f %f %f %f\n',xfTLT'); fclose(iXF); - positionList = geometry.(tomoList{iTomo}); + - positionList = positionList(positionList(:,26) ~= -9999,:); - nFidsTotal = nFidsTotal + size(positionList,1); + + + nFidsTotal = nFidsTotal + size(positionList{iTomo},1); % Need to store tilt name/path explicity in meta deta tiltName = tilt_binned_filename; @@ -552,7 +585,7 @@ reconGeometry.NY, ... reconGeometry.NZ]); nPrjs = size(TLT,1); - nSubTomos = size(positionList,1); + nSubTomos = size(positionList{iTomo},1) if (nSubTomos == 0) % No points were saved after template matching so skip this tilt series @@ -594,12 +627,12 @@ for iSubTomo = 1:nSubTomos - subtomo_rot_matrix = reshape(positionList(iSubTomo,17:25),3,3); - subtomo_origin_in_tomo_frame = positionList(iSubTomo,11:13); + subtomo_rot_matrix = reshape(positionList{iTomo}(iSubTomo,17:25),3,3); + subtomo_origin_in_tomo_frame = positionList{iTomo}(iSubTomo,11:13); subtomo_origin_wrt_tilt_origin = subtomo_origin_in_tomo_frame - tomo_origin_in_tomo_frame + tomo_origin_wrt_tilt_origin; iRefIDX = 1; - iClassIDX = positionList(iSubTomo,26); + iClassIDX = positionList{iTomo}(iSubTomo,26); use_this_class = true; if (nRefs > 1) % Assuming generally there are fewer classes seleceted as references than there are total classes @@ -641,12 +674,12 @@ fprintf('ignoring subTomo %d for class %d because it is not one of the references.\n', iSubTomo, iClassIDX); end else - if positionList(iSubTomo,7) == 1 + if positionList{iTomo}(iSubTomo,7) == 1 iAvgResamp = BH_resample3d(refVol{1}{iRefIDX},subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'); - elseif positionList(iSubTomo,7) ==2 + elseif positionList{iTomo}(iSubTomo,7) ==2 iAvgResamp = BH_resample3d(refVol{2}{iRefIDX},subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'); else - error('positionList iSubtomo %d col 7 is %d',iSubTomo,positionList(iSubTomo,7)); + error('positionList iSubtomo %d col 7 is %d',iSubTomo,positionList{iTomo}(iSubTomo,7)); end iMaskResamp = BH_resample3d(particleMask{iRefIDX},subtomo_rot_matrix',shiftVAL,'Bah','GPU','forward'); @@ -729,7 +762,7 @@ d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n', ... - fidIDX, tomoIdx,positionList(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(subtomo_rot_matrix,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList(iSubTomo,7)); + fidIDX, tomoIdx,positionList{iTomo}(iSubTomo,4),d1,d2,180./pi.*TLT(iPrj_nat,13),reshape(subtomo_rot_matrix,1,9) , preExposure(iPrj_nat), postExposure(iPrj_nat),positionList{iTomo}(iSubTomo,7)); % These shifts are a record of transformation from the raw data, but here % we are comparing with [CTF] corrected data, from which the @@ -866,6 +899,10 @@ outputStackName = sprintf('%smapBack%d/%s_%d_mapBack.st',mbOUT{1:3},iSave); while (keepItRunning) + + if (skip_this_tilt_series_bc_alignments_are_bonkers) + break; + end inc = 0:mapBackRePrjSize:sTY-1; if inc(end) < sTY-1 @@ -984,10 +1021,13 @@ case 8 mapBackRePrjSize = 4; case 4 - system(sprintf('%s > failReProj.log',rePrjFileName)) - error(['mapBackRePrjSize = 4 is still too much for the'],... - ['GPU which probably means your local alignments'],... - ['are too large']) + system(sprintf('%s >> failReProj.log',rePrjFileName)) + system(sprintf('echo "reprojection for %s failed, which probably means the local alignments are bonkers" >> failReProj_name.log %s\n ', tilt_binned_filename)); + skip_this_tilt_series_bc_alignments_are_bonkers = true; + end + + if (skip_this_tilt_series_bc_alignments_are_bonkers) + break; end % Update the stored size subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize') = mapBackRePrjSize; @@ -997,8 +1037,8 @@ system(sprintf('rm %s.tmp %s',rePrjFileName,rePrjFileName)); system(sprintf('%s',rePrjFileName)); - Break out to next iter of while loop, recalculating the - chunk size at reduced depth. + % Break out to next iter of while loop, recalculating the + % chunk size at reduced depth. break @@ -1014,7 +1054,10 @@ end % end of while loop end % loop over error and masked tomo - + if (skip_this_tilt_series_bc_alignments_are_bonkers) + continue; + end + fprintf(reModFile,['#!/bin/bash\n\n',... 'tilt -StandardInput << EOF\n',... 'input %s\n', ... @@ -1169,16 +1212,16 @@ nUniqueFids = numel(unique(fidList(:,2))); % I think the max val of this column should also be okay (+1) nFidsTotal = nUniqueFids; % nFidsTotal = sum(fidList(:,5) == 1 ); - if emc.tomoCPR_random_subset == -1 || emc.tomoCPR_random_subset > nUniqueFids - fprintf('Using all of the %d available fiducials\n',nUniqueFids); - else - fprintf('Using a random subset of %d fiducials from the %d available\n',... - emc.tomoCPR_random_subset, nUniqueFids); + % if emc.tomoCPR_random_subset == -1 || emc.tomoCPR_random_subset > nUniqueFids + % fprintf('Using all of the %d available fiducials\n',nUniqueFids); + % else + % fprintf('Using a random subset of %d fiducials from the %d available\n',... + % emc.tomoCPR_random_subset, nUniqueFids); - keepFids = datasample(0:nUniqueFids-1,emc.tomoCPR_random_subset,'Replace',false); - fidList(~ismember(fidList(:,2),keepFids),2) = -9999; - nFidsTotal = emc.tomoCPR_random_subset; - end + % keepFids = datasample(0:nUniqueFids-1,emc.tomoCPR_random_subset,'Replace',false); + % fidList(~ismember(fidList(:,2),keepFids),2) = -9999; + % nFidsTotal = emc.tomoCPR_random_subset; + % end for iPrj = 1:nPrjs diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index a46e5f7a..fdbf2a0f 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -14,7 +14,7 @@ % directlyCT % by the users (private methods-ish) -skip_to_end = false; + emc = BH_parseParameterFile(PARAMETER_FILE); mapBackIter = EMC_str2double(mapBackIter); MAX_EXPOSURE = EMC_str2double(MAX_EXPOSURE) @@ -22,6 +22,10 @@ error('MAX_EXPOSURE is nan - if running from an interactive matlab session, did you enter as a string?'); end +% to re-run using the existing intial star and particle stack. To avoid an overwrite, provide a new basename to the call +% to emClarity reconstruct and then link the star/stack +skip_to_end = emc.('rerun_refinement_cisTEM'); + % For trouble shooting on tilted images. MIN_EXPOSURE = 0; % Ideally, we would transform fully and go back to the non-rotated stack. I think with the apoferritin test set, @@ -434,28 +438,27 @@ nPrjsIncluded = 0; for iPrj = 1:nPrjs - iPrj_nat = find(TLT(:,1) == iPrj); - if (MIN_EXPOSURE < abs(TLT(iPrj_nat,11)) && abs(TLT(iPrj_nat,11)) <= MAX_EXPOSURE) + iPrj_index_in_TLT = find(TLT(:,1) == iPrj); + if (MIN_EXPOSURE < abs(TLT(iPrj_index_in_TLT,11)) && abs(TLT(iPrj_index_in_TLT,11)) <= MAX_EXPOSURE) nPrjsIncluded = nPrjsIncluded + 1; - % imod is indexing from zero - zCoord = iPrj_nat; + % For a positive angle, this will rotate the positive X axis farther from the focal plane (more underfocus) - % rTilt = BH_defineMatrix([0,TLT(iPrj_nat,4),0],'SPIDER','inv'); - rTilt = BH_defineMatrix(TLT(iPrj_nat,4),'TILT','fwdVector') ; + % rTilt = BH_defineMatrix([0,TLT(iPrj_index_in_TLT,4),0],'SPIDER','inv'); + rTilt = BH_defineMatrix(TLT(iPrj_index_in_TLT,4),'TILT','fwdVector') ; prjCoords = rTilt*subtomo_origin_wrt_specimen_origin'; % I think this is for comparison with the values obtained from projecting using IMOD: FIXME - fprintf(defOUT,'%d %d %6.6e\n', fidIDX, zCoord, abs(TLT(iPrj_nat,15)) - prjCoords(3).*pixel_size.*10^-10); + fprintf(defOUT,'%d %d %6.6e\n', fidIDX, iPrj, abs(TLT(iPrj_index_in_TLT,15)) - prjCoords(3).*pixel_size.*10^-10); % Defocus value adjusted for Z coordinate in the tomogram. nm - d1 = (abs(TLT(iPrj_nat,15)) - subtomo_origin_wrt_specimen_origin(3).*pixel_size.*10^-10) * 10^9; - d2 = TLT(iPrj_nat,12)*10^9; % half astigmatism value + d1 = (abs(TLT(iPrj_index_in_TLT,15)) - subtomo_origin_wrt_specimen_origin(3).*pixel_size.*10^-10) * 10^9; + d2 = TLT(iPrj_index_in_TLT,12)*10^9; % half astigmatism value fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n', ... - fidIDX, tomoIdx, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_nat,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_nat), postExposure(iPrj_nat), positionList(iSubTomo,7)); + fidIDX, tomoIdx, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_index_in_TLT,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_index_in_TLT), postExposure(iPrj_index_in_TLT), positionList(iSubTomo,7)); else fprintf(coordSTART,'%d %d %d %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %3.3f %d\n',-9999, -9999,-9999,1.0,1.0,1.0,1,1,1,1,1,1,1,1,1,0,0,1); end @@ -760,7 +763,7 @@ % df1 = ( wrkPar(iFid,4) + wrkPar(iFid,5)) * 10; % df2 = ( wrkPar(iFid,4) - wrkPar(iFid,5)) * 10; % dfA = wrkPar(iFid,6) - % fidIDX, tomoIdx, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_nat,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_nat), postExposure(iPrj_nat), positionList(iSubTomo,7)); + % fidIDX, tomoIdx, positionList(iSubTomo,4), d1, d2, 180./pi.*TLT(iPrj_index_in_TLT,13), reshape(subtomo_rot_matrix,1,9), preExposure(iPrj_index_in_TLT), postExposure(iPrj_index_in_TLT), positionList(iSubTomo,7)); df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; @@ -806,7 +809,7 @@ system(sprintf('cat %s >> %s', newstack_file, newstack_file_with_n_stacks)); system(sprintf('newstack -FileOfInputs %s %s.mrc > /dev/null', newstack_file_with_n_stacks, output_prefix)); -end % skip to here +end % skip to here skip_to_end % SAVE_IMG(cat(3,output_cell{:}),sprintf('%s.mrc',output_prefix),pixelSize); outputHeader = getHeader(MRCImage(sprintf('%s.mrc',output_prefix),0)); @@ -839,7 +842,6 @@ end - % %%%%%%%%%%%%%%%%%%%%%%%%% % Initial reconstruction % %%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -879,12 +881,12 @@ 'No\n', ...Apply likelihood blurring [No] : 'No\n', ...Threshold input reconstruction [No] : 'Yes\n', ...Dump intermediate arrays (merge later) [No] : - '%s/dump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : - '%s/dump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%s/%sdump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : + '%s/%sdump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : '%d\n', ... Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache, iProc,tmpCache,iProc, n_threads_per_proc(iProc)); + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache,output_prefix, iProc,tmpCache,output_prefix,iProc, n_threads_per_proc(iProc)); fprintf(recScript, 'eof\n'); @@ -899,6 +901,28 @@ end +% sometimes we get to merge 3d before the reconstructions are done? +n_pauses = 0; +max_pauses = 6; +while (n_pauses < max_pauses) + all_found = true; + for iProc = 1:n_recon_procs + fname1 = sprintf('%s/%sdump_1_%d.dat',tmpCache,output_prefix,iProc); + fname2 = sprintf('%s/%sdump_2_%d.dat',tmpCache,output_prefix,iProc); + if ~(exist(fname1, 'file') && exist(fname2, 'file')) + all_found = false; + break; + end + end + if all_found + break; + else + fprintf('Waiting for reconstructions to finish...\n'); + pause(10); + end + n_pauses = n_pauses + 1; +end + merge3d_name = sprintf('%s_merge3d.sh',output_prefix); @@ -915,14 +939,14 @@ '%4.4f\n', ... molecularMass' '%3.3f\n', ... inermask ang '%3.3f\n', ... outermas ang - '%s/dump_1_.dat\n', ... - '%s/dump_2_.dat\n', ... + '%s/%sdump_1_.dat\n', ... + '%s/%sdump_2_.dat\n', ... '%d\n'], ... Number of dump files [8] : getenv('EMC_MERGE3D'), ... output_prefix, output_prefix, output_prefix, output_prefix, ... emc.('particleMass')*10^3, ... 0.0, mean(emc.('Ali_mRadius')), ... - tmpCache, tmpCache, n_recon_procs); + tmpCache,output_prefix, tmpCache, output_prefix,n_recon_procs); fprintf(merge3dScript, 'eof\n'); @@ -931,11 +955,11 @@ pause(1) system(sprintf('./%s',merge3d_name)); % clean up dumps -system(sprintf('rm %s/dump_?_*.dat',tmpCache)); +system(sprintf('rm %s/%sdump_?_*.dat',tmpCache,output_prefix)); % Get the FSC cutoff for refinement fsc = importdata(sprintf('%s_stats.txt',output_prefix),' ',12); -fsc_cutoff = fsc.data(find(fsc.data(:,5) < 0.5,1),2) +fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) %%%%%%%%%%%%%%%%%%%%%%%%% % Refine @@ -1047,12 +1071,12 @@ 'No\n', ...Apply likelihood blurring [No] : 'No\n', ...Threshold input reconstruction [No] : 'Yes\n', ...Dump intermediate arrays (merge later) [No] : - '%s/dump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : - '%s/dump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%s/%sdump_1_%d.dat\n', ...Output dump filename for odd particle [%sdump_file_1.dat] : + '%s/%sdump_2_%d.dat\n', ...Output dump filename for even particle [%sdump_file_2.dat] : '%d\n', ... Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')),tmpCache, iProc,tmpCache,iProc, n_threads_per_proc(iProc)); + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')),tmpCache, output_prefix, iProc, tmpCache, output_prefix, iProc, n_threads_per_proc(iProc)); fprintf(recScript, 'eof\n'); @@ -1067,6 +1091,27 @@ end +% sometimes we get to merge 3d before the reconstructions are done? +n_pauses = 0; +max_pauses = 6; +while (n_pauses < max_pauses) + all_found = true; + for iProc = 1:n_recon_procs + fname1 = sprintf('%s/%sdump_1_%d.dat',tmpCache,output_prefix,iProc); + fname2 = sprintf('%s/%sdump_2_%d.dat',tmpCache,output_prefix,iProc); + if ~(exist(fname1, 'file') && exist(fname2, 'file')) + all_found = false; + break; + end + end + if all_found + break; + else + fprintf('Waiting for reconstructions to finish...\n'); + pause(10); + end + n_pauses = n_pauses + 1; +end merge3d_name = sprintf('%s_merge3d.sh',output_prefix); system(sprintf('rm -f %s',merge3d_name)); @@ -1082,14 +1127,14 @@ '%4.4f\n', ... molecularMass' '%3.3f\n', ... inermask ang '%3.3f\n', ... outermas ang - '%s/dump_1_.dat\n', ... - '%s/dump_2_.dat\n', ... + '%s/%sdump_1_.dat\n', ... + '%s/%sdump_2_.dat\n', ... '%d\n'], ... Number of dump files [8] : getenv('EMC_MERGE3D'), ... output_prefix, output_prefix, output_prefix, output_prefix, ... emc.('particleMass')*10^3, ... 0.0, mean(emc.('Ali_mRadius')), ... - tmpCache, tmpCache, n_recon_procs); + tmpCache,output_prefix, tmpCache,output_prefix, n_recon_procs); fprintf(merge3dScript, 'eof\n'); @@ -1098,11 +1143,13 @@ pause(1) system(sprintf('./%s',merge3d_name)); -system(sprintf('rm %s/dump_?_*.dat',tmpCache)); +system(sprintf('rm %s/%sdump_?_*.dat',tmpCache,output_prefix)); % Get the FSC cutoff for refinement fsc = importdata(sprintf('%s_stats_refined.txt',output_prefix),' ',12); -fsc_cutoff = fsc.data(find(fsc.data(:,5) < 0.5,1),2) +fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) +% Use to decide whether to do more angular refinement +fsc_last = fsc_cutoff %%%%%%%%%%%%%%%%%%%%%%%%% % Refine @@ -1214,12 +1261,12 @@ 'No\n', ...Apply likelihood blurring [No] : 'No\n', ...Threshold input reconstruction [No] : 'Yes\n', ...Dump intermediate arrays (merge later) [No] : - '%s/dump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : - '%s/dump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%s/%sdump_1_%d.dat\n', ...Output dump filename for odd particle [%sdump_file_1.dat] : + '%s/%sdump_2_%d.dat\n', ...Output dump filename for even particle [%sdump_file_2.dat] : '%d\n', ... Max. threads to use for calculation [36] : ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache, iProc,tmpCache,iProc, n_threads_per_proc(iProc)); + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache, output_prefix,iProc,tmpCache,output_prefix,iProc, n_threads_per_proc(iProc)); fprintf(recScript, 'eof\n'); @@ -1234,6 +1281,27 @@ end +% sometimes we get to merge 3d before the reconstructions are done? +n_pauses = 0; +max_pauses = 6; +while (n_pauses < max_pauses) + all_found = true; + for iProc = 1:n_recon_procs + fname1 = sprintf('%s/%sdump_1_%d.dat',tmpCache,output_prefix,iProc); + fname2 = sprintf('%s/%sdump_2_%d.dat',tmpCache,output_prefix,iProc); + if ~(exist(fname1, 'file') && exist(fname2, 'file')) + all_found = false; + break; + end + end + if all_found + break; + else + fprintf('Waiting for reconstructions to finish...\n'); + pause(10); + end + n_pauses = n_pauses + 1; +end merge3d_name = sprintf('%s_merge3d.sh',output_prefix); system(sprintf('rm -f %s',merge3d_name)); @@ -1249,14 +1317,14 @@ '%4.4f\n', ... molecularMass' '%3.3f\n', ... inermask ang '%3.3f\n', ... outermas ang - '%s/dump_1_.dat\n', ... - '%s/dump_2_.dat\n', ... + '%s/%sdump_1_.dat\n', ... + '%s/%sdump_2_.dat\n', ... '%d\n'], ... Number of dump files [8] : getenv('EMC_MERGE3D'), ... output_prefix, output_prefix, output_prefix, output_prefix, ... emc.('particleMass')*10^3, ... 0.0, mean(emc.('Ali_mRadius')), ... - tmpCache, tmpCache, n_recon_procs); + tmpCache,output_prefix, tmpCache,output_prefix, n_recon_procs); fprintf(merge3dScript, 'eof\n'); @@ -1265,33 +1333,59 @@ pause(1) system(sprintf('./%s',merge3d_name)); -system(sprintf('rm %s/dump_?_*.dat',tmpCache)); +system(sprintf('rm %s/%sdump_?_*.dat',tmpCache,output_prefix)); + + + % Get the FSC cutoff for refinement fsc = importdata(sprintf('%s_stats_refined2.txt',output_prefix),' ',12); -fsc_cutoff = fsc.data(find(fsc.data(:,5) < 0.5,1),2) +fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) -% TODO dfocus refine if res high enough -% if (fsc_cutoff < 6.0) -if (false) +if ~(do_initial) + fsc_last = fsc_cutoff / 0.94; +end +n_max_refinements = 7; +i_refine = 2; + +do_refine_defocus_loop = emc.('refine_defocus_cisTEM') + +while ( fsc_cutoff / fsc_last < 0.9501 && i_refine < n_max_refinements) + +refine_angles_and_shifts = "yes"; +refine_defocus = "no"; +use_cutoff = fsc_cutoff; +if (do_refine_defocus_loop) + refine_angles_and_shifts = "no"; + refine_defocus = "yes"; + use_cutoff = fsc_res - 0.8 + do_refine_defocus_loop = false; +end + +i_refine = i_refine + 1; +fsc_last = fsc_cutoff; + +fprintf('Refinement %d, fsc cutoff %f\n',i_refine,fsc_cutoff); + + %%%%%%%%%%%%%%%%%%%%%%%%% % Refine %%%%%%%%%%%%%%%%%%%%%%%%%%%%r -refine_def = sprintf('%s_ref_def.sh',output_prefix); -system(sprintf('rm -f %s',refine_def)); -refineScript = fopen(sprintf('%s',refine_def), 'w'); +refine_angles = sprintf('%s_ref_angles_%d.sh',output_prefix, i_refine); +system(sprintf('rm -f %s',refine_angles)); +refineScript = fopen(sprintf('%s',refine_angles), 'w'); fprintf(refineScript,[ ... '#!/bin/bash\n\n', ... '%s << eof\n', ... '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) '%s.star\n', ... sprintf('%s.star',output_prefix) - '%s_recFilt_refined.mrc\n',... - '%s_stats_refined.txt\n',... + '%s_recFilt_refined%d.mrc\n',... + '%s_stats_refined%d.txt\n',... 'yes\n',... Use statistics [Yes] : 'my_projection_stack.mrc\n',... not going to be used : - '%s_refined3.star\n', ... - '%s_changes3.star\n', ...Output parameter changes + '%s_refined%d.star\n', ... + '%s_changes%d.star\n', ...Output parameter changes '%s\n',... Particle symmetry [C1] : '1\n', ...First particle to refine (0 = first in stack) [1] : '0\n', ...Last particle to refine (0 = last in stack) [0] : @@ -1314,36 +1408,40 @@ '100.0\n',...2D mask Y coordinate (A) [100.0] : '100.0\n',...2D mask Z coordinate (A) [100.0] : '100.0\n',...2D mask radius (A) [100.0] : - '5000.0\n',...Defocus search range (A) [500.0] : + '1000.0\n',...Defocus search range (A) [500.0] : '50.0\n',...Defocus step (A) [50.0] : '1.0\n',...Tuning parameters: padding factor [1.0] : 'no\n',...Global search [No] : 'yes\n',... Local refinement [Yes] : - 'no\n',...Refine Psi [no] : - 'no\n',...Refine Theta [no] : - 'no\n',...Refine Phi [no] : - 'no\n',...Refine ShiftX [Yes] : - 'no\n',...Refine ShiftY [Yes] : + '%s\n',...Refine Psi [no] : + '%s\n',...Refine Theta [no] : + '%s\n',...Refine Phi [no] : + '%s\n',...Refine ShiftX [Yes] : + '%s\n',...Refine ShiftY [Yes] : 'no\n',...Calculate matching projections [No] : 'no\n',...Apply 2D masking [No] : - 'yes\n',...Refine defocus [No] : + '%s\n',...Refine defocus [No] : 'yes\n',...Normalize particles [Yes] : 'no\n',...Invert particle contrast [No] : 'yes\n',...Exclude images with blank edges [Yes] : 'yes\n',...Normalize input reconstruction [Yes] : 'no\n',...Threshold input reconstruction [No] : '%2.2d\n', ...Max. threads to use for calculation [36] : - ], getenv('EMC_REFINE3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... - symmetry,emc.pixel_size_angstroms, ... + ], getenv('EMC_REFINE3D'), output_prefix, output_prefix, ... + output_prefix, i_refine - 1, output_prefix, i_refine - 1, ... + output_prefix, i_refine, output_prefix, i_refine, ... + symmetry, emc.pixel_size_angstroms, ... emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... - fsc_cutoff,fsc_cutoff,maxThreads); + use_cutoff,use_cutoff, ... + refine_angles_and_shifts, refine_angles_and_shifts, refine_angles_and_shifts, refine_angles_and_shifts, refine_angles_and_shifts, ... + refine_defocus, maxThreads); fprintf(refineScript, '\neof\n'); fclose(refineScript); pause(1); -system(sprintf('chmod a=wrx %s',refine_def)); +system(sprintf('chmod a=wrx %s',refine_angles)); pause(1); -system(sprintf('./%s',refine_def)); +system(sprintf('./%s',refine_angles)); %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % Reconstruct refined @@ -1356,12 +1454,12 @@ '#!/bin/bash\n\n', ... '%s << eof\n', ... '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) - '%s_refined3.star\n', ... sprintf('%s.star',output_prefix) + '%s_refined%d.star\n', ... sprintf('%s.star',output_prefix) 'none.mrc\n', ... '%s_rec1.mrc\n',... '%s_rec2.mrc\n',... - '%s_recFilt_refined3.mrc\n',... - '%s_stats_refined3.txt\n',... + '%s_recFilt_refined%d.mrc\n',... + '%s_stats_refined%d.txt\n',... '%s\n', ... '%d\n', ... '%d\n', ... @@ -1385,12 +1483,14 @@ 'No\n', ...Apply likelihood blurring [No] : 'No\n', ...Threshold input reconstruction [No] : 'Yes\n', ...Dump intermediate arrays (merge later) [No] : - '%s/dump_1_%d.dat\n', ...Output dump filename for odd particle [dump_file_1.dat] : - '%s/dump_2_%d.dat\n', ...Output dump filename for even particle [dump_file_2.dat] : + '%s/%sdump_1_%d.dat\n', ...Output dump filename for odd particle [%sdump_file_1.dat] : + '%s/%sdump_2_%d.dat\n', ...Output dump filename for even particle [%sdump_file_2.dat] : '%d\n', ... Max. threads to use for calculation [36] : - ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... + ], getenv('EMC_RECONSTRUCT3D'),output_prefix, ... + output_prefix, i_refine, output_prefix, output_prefix, ... + output_prefix, i_refine, output_prefix, i_refine, ... symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... - emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache, iProc,tmpCache,iProc, n_threads_per_proc(iProc)); + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache,output_prefix, iProc,tmpCache,output_prefix,iProc, n_threads_per_proc(iProc)); fprintf(recScript, 'eof\n'); @@ -1405,6 +1505,27 @@ end +% sometimes we get to merge 3d before the reconstructions are done? +n_pauses = 0; +max_pauses = 6; +while (n_pauses < max_pauses) + all_found = true; + for iProc = 1:n_recon_procs + fname1 = sprintf('%s/%sdump_1_%d.dat',tmpCache,output_prefix,iProc); + fname2 = sprintf('%s/%sdump_2_%d.dat',tmpCache,output_prefix,iProc); + if ~(exist(fname1, 'file') && exist(fname2, 'file')) + all_found = false; + break; + end + end + if all_found + break; + else + fprintf('Waiting for reconstructions to finish...\n'); + pause(10); + end + n_pauses = n_pauses + 1; +end merge3d_name = sprintf('%s_merge3d.sh',output_prefix); system(sprintf('rm -f %s',merge3d_name)); @@ -1415,19 +1536,20 @@ '%s << eof\n', ... '%s_rec1.mrc\n',... '%s_rec2.mrc\n',... - '%s_recFilt_refined3.mrc\n',... - '%s_stats_refined3.txt\n',... + '%s_recFilt_refined%d.mrc\n',... + '%s_stats_refined%d.txt\n',... '%4.4f\n', ... molecularMass' '%3.3f\n', ... inermask ang '%3.3f\n', ... outermas ang - '%s/dump_1_.dat\n', ... - '%s/dump_2_.dat\n', ... + '%s/%sdump_1_.dat\n', ... + '%s/%sdump_2_.dat\n', ... '%d\n'], ... Number of dump files [8] : getenv('EMC_MERGE3D'), ... - output_prefix, output_prefix, output_prefix, output_prefix, ... + output_prefix, output_prefix, ... + output_prefix, i_refine, output_prefix, i_refine, ... emc.('particleMass')*10^3, ... 0.0, mean(emc.('Ali_mRadius')), ... - tmpCache, tmpCache, n_recon_procs); + tmpCache,output_prefix, tmpCache,output_prefix, n_recon_procs); fprintf(merge3dScript, 'eof\n'); @@ -1436,12 +1558,208 @@ pause(1) system(sprintf('./%s',merge3d_name)); -system(sprintf('rm %s/dump_?_*.dat',tmpCache)); +system(sprintf('rm %s/%sdump_?_*.dat',tmpCache, output_prefix)); -fsc = importdata(sprintf('%s_stats_refined3.txt',output_prefix),' ',12); -fsc_cutoff = fsc.data(find(fsc.data(:,5) < 0.5,1),2) +% Get the FSC cutoff for refinement +fsc = importdata(sprintf('%s_stats_refined%d.txt',output_prefix, i_refine),' ',12); +fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) + + +end % while loop on extra refinements + + +% TODO dfocus refine if res high enough +% if (fsc_cutoff < 6.0) +if (false) + %%%%%%%%%%%%%%%%%%%%%%%%% + % Refine + %%%%%%%%%%%%%%%%%%%%%%%%%%%%r + refine_def = sprintf('%s_ref_def.sh',output_prefix); + system(sprintf('rm -f %s',refine_def)); + refineScript = fopen(sprintf('%s',refine_def), 'w'); + fprintf(refineScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s.star\n', ... sprintf('%s.star',output_prefix) + '%s_recFilt_refined.mrc\n',... + '%s_stats_refined.txt\n',... + 'yes\n',... Use statistics [Yes] : + 'my_projection_stack.mrc\n',... not going to be used : + '%s_refined3.star\n', ... + '%s_changes3.star\n', ...Output parameter changes + '%s\n',... Particle symmetry [C1] : + '1\n', ...First particle to refine (0 = first in stack) [1] : + '0\n', ...Last particle to refine (0 = last in stack) [0] : + '1.0\n',...Percent of particles to use (1 = all) [1.0] : + '%3.3f\n', ... pixel size + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '300.0\n',...Low resolution limit (A) [300.0] : + '%3.3f\n',...High resolution limit (A) [8.0] : + '0.0\n',...Resolution limit for signed CC (A) (0.0 = max [0.0] : + '0.0\n',...Res limit for classification (A) (0.0 = max) [0.0] : + '0.0\n',...Mask radius for global search (A) (0.0 = max)[100.0] : + '%3.3f\n',...Approx. resolution limit for search (A) [8] : + '0.0\n',...Angular step (0.0 = set automatically) [0.0] : + '20\n',...Number of top hits to refine [20] : + '10\n',...Search range in X (A) (0.0 = 0.5 * mask radius)[12] : + '10\n',...[12] : + '100.0\n',...2D mask X coordinate (A) [100.0] : + '100.0\n',...2D mask Y coordinate (A) [100.0] : + '100.0\n',...2D mask Z coordinate (A) [100.0] : + '100.0\n',...2D mask radius (A) [100.0] : + '5000.0\n',...Defocus search range (A) [500.0] : + '50.0\n',...Defocus step (A) [50.0] : + '1.0\n',...Tuning parameters: padding factor [1.0] : + 'no\n',...Global search [No] : + 'yes\n',... Local refinement [Yes] : + 'no\n',...Refine Psi [no] : + 'no\n',...Refine Theta [no] : + 'no\n',...Refine Phi [no] : + 'no\n',...Refine ShiftX [Yes] : + 'no\n',...Refine ShiftY [Yes] : + 'no\n',...Calculate matching projections [No] : + 'no\n',...Apply 2D masking [No] : + 'yes\n',...Refine defocus [No] : + 'yes\n',...Normalize particles [Yes] : + 'no\n',...Invert particle contrast [No] : + 'yes\n',...Exclude images with blank edges [Yes] : + 'yes\n',...Normalize input reconstruction [Yes] : + 'no\n',...Threshold input reconstruction [No] : + '%2.2d\n', ...Max. threads to use for calculation [36] : + ], getenv('EMC_REFINE3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... + symmetry,emc.pixel_size_angstroms, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), ... + fsc_cutoff,fsc_cutoff,maxThreads); + + fprintf(refineScript, '\neof\n'); + fclose(refineScript); + pause(1); + system(sprintf('chmod a=wrx %s',refine_def)); + pause(1); + system(sprintf('./%s',refine_def)); + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + % Reconstruct refined + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + for iProc = 1:n_recon_procs + system(sprintf('rm -f %s_rec_%d.sh',output_prefix, iProc)); + recScript = fopen(sprintf('%s_rec_%d.sh',output_prefix, iProc), 'w'); + fprintf(recScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s.mrc\n', ... sprintf('%s.mrc',output_prefix) + '%s_refined3.star\n', ... sprintf('%s.star',output_prefix) + 'none.mrc\n', ... + '%s_rec1.mrc\n',... + '%s_rec2.mrc\n',... + '%s_recFilt_refined3.mrc\n',... + '%s_stats_refined3.txt\n',... + '%s\n', ... + '%d\n', ... + '%d\n', ... + '%3.3f\n', ... pixel size + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '0.0\n', ... rec res limit + '0.0\n', ... ref res limit + '5.0\n', ... Particle weighting factor (A^2) [5.0] + '1.0\n', ... Score threshold (<= 1 = percentage) [1.0] + '1.0\n', ...Tuning parameter: smoothing factor [1.0] : + '1.0\n', ...Tuning parameters: padding factor [1.0] : + 'Yes\n', ...Normalize particles [Yes] : + 'No\n', ...Adjust scores for defocus dependence [no] : + 'No\n', ...Invert particle contrast [No] : + 'Yes\n', ...Exclude images with blank edges [yes] : + 'No\n', ...Crop particle images [no] : + 'Yes\n', ...FSC calculation with even/odd particles [Yes] : + 'No\n', ...Center mass [No] : + 'No\n', ...Apply likelihood blurring [No] : + 'No\n', ...Threshold input reconstruction [No] : + 'Yes\n', ...Dump intermediate arrays (merge later) [No] : + '%s/%sdump_1_%d.dat\n', ...Output dump filename for odd particle [%sdump_file_1.dat] : + '%s/%sdump_2_%d.dat\n', ...Output dump filename for even particle [%sdump_file_2.dat] : + '%d\n', ... Max. threads to use for calculation [36] : + ], getenv('EMC_RECONSTRUCT3D'),output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, output_prefix, ... + symmetry,stack_boundaries(iProc),stack_boundaries(iProc+1)-1 ,emc.pixel_size_angstroms, ... + emc.('particleMass')*10^3, 0.0, mean(emc.('Ali_mRadius')), tmpCache,output_prefix, iProc,tmpCache,output_prefix,iProc, n_threads_per_proc(iProc)); + + fprintf(recScript, 'eof\n'); + + fclose(recScript); + system(sprintf('chmod a=wrx %s_rec_%d.sh',output_prefix, iProc)); + pause(1); + if (iProc < n_recon_procs) + system(sprintf('./%s_rec_%d.sh 2>&1 > /dev/null &',output_prefix, iProc)); + else + system(sprintf('./%s_rec_%d.sh && wait',output_prefix, iProc)); + end + + end + +% sometimes we get to merge 3d before the reconstructions are done? +n_pauses = 0; +max_pauses = 6; +while (n_pauses < max_pauses) + all_found = true; + for iProc = 1:n_recon_procs + fname1 = sprintf('%s/%sdump_1_%d.dat',tmpCache,output_prefix,iProc); + fname2 = sprintf('%s/%sdump_2_%d.dat',tmpCache,output_prefix,iProc); + if ~(exist(fname1, 'file') && exist(fname2, 'file')) + all_found = false; + break; + end + end + if all_found + break; + else + fprintf('Waiting for reconstructions to finish...\n'); + pause(10); + end + n_pauses = n_pauses + 1; end + merge3d_name = sprintf('%s_merge3d.sh',output_prefix); + system(sprintf('rm -f %s',merge3d_name)); + merge3dScript = fopen(sprintf('%s',merge3d_name), 'w'); + + fprintf(merge3dScript,[ ... + '#!/bin/bash\n\n', ... + '%s << eof\n', ... + '%s_rec1.mrc\n',... + '%s_rec2.mrc\n',... + '%s_recFilt_refined3.mrc\n',... + '%s_stats_refined3.txt\n',... + '%4.4f\n', ... molecularMass' + '%3.3f\n', ... inermask ang + '%3.3f\n', ... outermas ang + '%s/%sdump_1_.dat\n', ... + '%s/%sdump_2_.dat\n', ... + '%d\n'], ... Number of dump files [8] : + getenv('EMC_MERGE3D'), ... + output_prefix, output_prefix, output_prefix, output_prefix, ... + emc.('particleMass')*10^3, ... + 0.0, mean(emc.('Ali_mRadius')), ... + tmpCache, output_prefix,tmpCache, output_prefix,n_recon_procs); + + + fprintf(merge3dScript, 'eof\n'); + fclose(merge3dScript); + system(sprintf('chmod a=wrx %s',merge3d_name)); + pause(1) + system(sprintf('./%s',merge3d_name)); + + system(sprintf('rm %s/%sdump_?_*.dat',tmpCache,output_prefix)); + + fsc = importdata(sprintf('%s_stats_refined3.txt',output_prefix),' ',12); + fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) + fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) +end % defocus refine loop + end diff --git a/testScripts/BH_benchmark.m b/testScripts/BH_benchmark.m index f49f99d9..466caa1d 100644 --- a/testScripts/BH_benchmark.m +++ b/testScripts/BH_benchmark.m @@ -71,7 +71,7 @@ nTrials = 10; parContainer = cell(nWorkers,1); parfor iProc = 1:nWorkers - + volMem = 0; g = gpuDevice(1); % Create the volume to operate on diff --git a/testScripts/fourierTransformer.m b/testScripts/fourierTransformer.m index 4b6be75c..b9fa4347 100644 --- a/testScripts/fourierTransformer.m +++ b/testScripts/fourierTransformer.m @@ -44,11 +44,11 @@ if nargin > 1 if (ischar(varargin{1})) -if (strcmpi(varargin{1},'OddSizeOversampled')) + if (strcmpi(varargin{1},'OddSizeOversampled')) obj.OddSizeOversampled = 1; else error('Did not recognize the extra argument when intializing the fourierTransformer'); -end + end else if (isnumeric(varargin{1})) if (numel(varargin{1}) == 6) @@ -283,7 +283,6 @@ function delete(obj) obj.indexCenterFWD = EMC_maskIndex('fftshift', obj.inputSize, 'GPU', {'half',true}); end - inputVol = inputVol(obj.indexCenterFWD); end @@ -344,13 +343,13 @@ function makeBandPass(obj, sizeInput, bpValsNew) bpValsNew(3),... 'GPU', ... bpValsNew(4)); - switch ndims(obj.bandpass) - case 3 - obj.bandpass = obj.bandpass(1:obj.halfDimSize,:,:); - case 2 - obj.bandpass = obj.bandpass(1:obj.halfDimSize,:); - case 1 - obj.bandpass = obj.bandpass(1:obj.halfDimSize); + switch ndims(obj.bandpass) + case 3 + obj.bandpass = obj.bandpass(1:obj.halfDimSize,:,:); + case 2 + obj.bandpass = obj.bandpass(1:obj.halfDimSize,:); + case 1 + obj.bandpass = obj.bandpass(1:obj.halfDimSize); end % Update the properties diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 1cc08c01..6a4faeaa 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=2 -nightly=0 +nightly=1 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 17e2a08a..ebaaa200 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -634,7 +634,8 @@ parVect = 1:nParProcesses; parfor iParProc = parVect % for iParProc = parVect % r - + % to avoid unitialized temporaries warnings + iSF3D = []; % Get the gpuIDX assigned to this process gpuIDXList = mod(parVect+emc.nGPUs,emc.nGPUs)+1; iGPUidx = gpuIDXList(iParProc); @@ -667,8 +668,8 @@ {'none'},1,0,1); cccWeight = (cccWeight ./ emc.pixel_size_angstroms).^2; - - + else + cccWeight = 1; end if (eachTomo) @@ -678,6 +679,10 @@ tomoAvgStack{iTomo} = zeros(sizeMask, 'single'); tomoWgtStack{iTomo} = zeros(sizeMask, 'single'); end + else + % To avoid unitialized temporaries warnings + tomoAvgStack = []; + tomoWgtStack = []; end nTomos = 1; @@ -704,6 +709,11 @@ tomoAvg = zeros(sizeMask, 'single', 'gpuArray'); tomoWgt = zeros(sizeCalc , 'single', 'gpuArray'); tomoCount = 0; + else + % To avoid unitialized temporaries warnings + tomoAvg = []; + tomoWgt = []; + tomoCount = 0; end fprintf('gpu %d working on %d/%d volumes\n',iParProc,iTomo,nTomograms); @@ -731,6 +741,7 @@ if (emc.flgCutOutVolumes && ~volumesNeedToBeExtracted) volumeData = []; + volHeader = []; else reconScaling = 1; @@ -797,6 +808,8 @@ for iSubTomo = particleIndex' + % fprintf('gpu %d working on subtomo %d volumes\n',iParProc,iSubTomo); + iParticle = []; iCCCweight = []; iWedgeMask = []; @@ -849,6 +862,7 @@ make_sf3d = false; end + if (emc.flgQualityWeight) @@ -1005,7 +1019,7 @@ iWedgeMask = iWedgeMask .* fftshift(iCCCweight); end - + trimAvg = mean(iParticle(:)); @@ -1245,13 +1259,7 @@ for iClassPos = 1:maxClasses - if (doNotTrim) && (emc.classification) - % % % % % % % m = BH_mask3d('sphere',sizeMask,floor(sizeMask./2-6),pcaMaskCenter); - [ m ] = EMC_maskShape('sphere', sizeMask,floor(sizeMask./2-6), 'gpu', {'shift', pcaMaskCenter}); - else - m = 1; - end - + % Re-weight both halves whether flgGold or not. for iGold = 1:2-flgFinalAvg From 2dbb8390eef2a6e8869ffa01720dc73b24385500 Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 4 Mar 2025 14:34:22 -0500 Subject: [PATCH 088/151] allow ctf 3d in parallel and a bunch of other shit including revamped ctf estimation/hand check --- alignment/BH_runAutoAlign.m | 4 +- alignment/BH_templateSearch3d_2.m | 96 +++++++++++++++++++++++++++++-- coordinates/BH_multi_iterator.m | 86 ++++++++++++++++----------- coordinates/BH_multi_recGeom.m | 5 ++ ctf/BH_ctfCalc.m | 4 +- ctf/BH_ctf_Correct3d.m | 58 ++++++++++--------- ctf/BH_ctf_Estimate.m | 57 +++--------------- ctf/BH_ctf_Refine2.m | 7 ++- metaData/BH_parseParameterFile.m | 5 ++ synthetic/BH_synthetic_mapBack.m | 32 +---------- synthetic/BH_to_cisTEM_mapBack.m | 8 ++- 11 files changed, 211 insertions(+), 151 deletions(-) diff --git a/alignment/BH_runAutoAlign.m b/alignment/BH_runAutoAlign.m index a813cbd3..c42d714c 100644 --- a/alignment/BH_runAutoAlign.m +++ b/alignment/BH_runAutoAlign.m @@ -182,6 +182,7 @@ else switch_axes = false; end + clear a b s p pad else switch_axes = false; end @@ -244,10 +245,11 @@ inputStack(:,:,iPrj) = gather(tmpPrj); end + SAVE_IMG(inputStack,{fixedName,'half'},emc.pixel_size_angstroms); fprintf('finished preprocessing tilt-series\n'); -clear tmpPrj inputStack +clear tmpPrj inputStack gradientAliasFilter cd(wrkDir) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index a257df8b..2e99cc5d 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -132,6 +132,12 @@ scale_mip = false; end +try + measure_noise_variance = emc.('measure_noise_variance'); +catch + measure_noise_variance = false; +end + if pixelSize*2 > bp_vals(3) fprintf('\nLimiting to Nyquist (%f) instead of user requested low pass cutoff %f Angstrom\n',pixelSize*2,bp_vals(3)); @@ -261,9 +267,9 @@ [ OUTPUT ] = BH_multi_iterator( [targetSize; ... - size(tomogram);... - sizeTempBIN; ... - 2.*latticeRadius], 'convolution' ); + size(tomogram);... + sizeTempBIN; ... + 2.*latticeRadius], 'convolution' ); @@ -274,6 +280,7 @@ validCalc = OUTPUT(5,:); nIters = OUTPUT(6,:); + %[ padVal ] = BH_multi_padVal( sizeTemp, sizeChunk ); %tempPre = padVal(1,:); %tempPost = padVal(2,:); @@ -313,7 +320,7 @@ fprintf('valid Calc %d %d %d\n', validCalc); fprintf('# of iterations %d %d %d\n', nIters); fprintf('-----\n'); - +error('asdf') valid_ratio = prod(sizeChunk) ./ prod(validCalc); size(tomogram) @@ -332,13 +339,20 @@ % Currently, this would not produce the correct results for odd size area % % % fftMask = BH_fftShift(validArea,sizeChunk,0); -% Array for storing chunk results +% Array for storing chunk results these could probably be half-precision RESULTS_peak = zeros(sizeTomo, 'single'); RESULTS_angle= zeros(sizeTomo, 'single'); if ( tmpDecoy ) RESULTS_decoy = RESULTS_peak; end +RESULTS_sum = []; +RESULTS_sum_sq = []; +if (measure_noise_variance) + RESULTS_sum = zeros(sizeTomo, 'single'); + RESULTS_sum_sq = zeros(sizeTomo, 'single'); +end + % % % % optimize fft incase a power of two is not used, this will make things run ok. @@ -838,6 +852,16 @@ angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... iCut(2):iCut(2)+sizeChunk(2)-1,... iCut(3):iCut(3)+sizeChunk(3)-1); + + if (measure_noise_variance) + sumTmp = RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + sumSqTmp = RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + end + if ( tmpDecoy ) decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... iCut(2):iCut(2)+sizeChunk(2)-1,... @@ -855,6 +879,18 @@ angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... vA(1,2) + 1:end - vA(2,2), ... vA(1,3) + 1:end - vA(2,3))); + + if (measure_noise_variance) + sumTmp = gpuArray(sumTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3))); + sumSqTmp = gpuArray(sumSqTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3))); + + sumTmp = sumTmp + ccfmap; + sumSqTmp = sumSqTmp + ccfmap.^2; + end firstLoopOverChunk = false; @@ -878,6 +914,11 @@ if ( tmpDecoy ) decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); end + + if (measure_noise_variance) + sumTmp = sumTmp + ccfmap; + sumSqTmp = sumSqTmp + ccfmap.^2; + end intraLoopAngle = intraLoopAngle + 1; @@ -919,6 +960,35 @@ iCut(2):iCut(2)+sizeChunk(2)-1,... iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; clear angStoreTmp + + if (measure_noise_variance) + sumStoreTmp = RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + sumSqStoreTmp = RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + + + sumStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(sumTmp); + sumSqStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(sumSqTmp); + + + RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = sumStoreTmp; + + clear sumStoreTmp + + RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = sumSqStoreTmp; + clear sumSqStoreTmp + end if ( tmpDecoy ) decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... @@ -955,6 +1025,15 @@ 1+tomoPre(2):end-tomoPost(2),... 1+tomoPre(3):end-tomoPost(3)); +if (measure_noise_variance) + RESULTS_sum = RESULTS_sum(1+tomoPre(1):end-tomoPost(1),... + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); + RESULTS_sum_sq = RESULTS_sum_sq(1+tomoPre(1):end-tomoPost(1),... + 1+tomoPre(2):end-tomoPost(2),... + 1+tomoPre(3):end-tomoPost(3)); +end + if ( tmpDecoy ) RESULTS_decoy = RESULTS_decoy(1+tomoPre(1):end-tomoPost(1),... 1+tomoPre(2):end-tomoPost(2),... @@ -985,6 +1064,13 @@ anglesOUT = sprintf('./%s/%s_angles.mrc',convTMPNAME,mapName); angleListOUT = sprintf('./%s/%s_angles.list',convTMPNAME,mapName); SAVE_IMG(mag,{resultsOUT,'half'}); +noiseVarOUT = sprintf('./%s/%s_noise_variance.mrc',convTMPNAME,mapName); + +if (measure_noise_variance) + n_angles_searched = sum(any(ANGLES,2)); + noiseVar = (RESULTS_sum_sq./n_angles_searched - (RESULTS_sum./n_angles_searched).^2); + SAVE_IMG(noiseVar,{noiseVarOUT,'half'}); +end % SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); if ( tmpDecoy ) decoyOUT = sprintf('./%s/%s_decoy.mrc',convTMPNAME,mapName); diff --git a/coordinates/BH_multi_iterator.m b/coordinates/BH_multi_iterator.m index 249ee995..90f360e6 100755 --- a/coordinates/BH_multi_iterator.m +++ b/coordinates/BH_multi_iterator.m @@ -91,61 +91,83 @@ % gpu as possible. With finer angular searches, the number of references % needs more memory, so a smaller size here means more transfers, but this % should be balanced by the finer angles (more comp) - if all(SIZES(1,:) == 256) - nextBest = [128,144,160,168,192,216,224,256]; - elseif all(SIZES(1,:) == 384) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384]; - elseif all(SIZES(1,:) == 432) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432]; - elseif all(SIZES(1,:) == 512) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432,480,512]; - elseif all(SIZES(1,:) > 512) - nextBest = [128,144,160,168,192,216,224,256,... - 288,300,320,336,360,384,400,432,480,512,... - 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]; - end + % if all(SIZES(1,:) <= 256) + % nextBest = [128,144,160,168,192,216,224,256]; + % elseif all(SIZES(1,:) <= 384) + % nextBest = [128,144,160,168,192,216,224,256,... + % 288,300,320,336,360,384]; + % elseif all(SIZES(1,:) <= 432) + % nextBest = [128,144,160,168,192,216,224,256,... + % 288,300,320,336,360,384,400,432]; + % elseif all(SIZES(1,:) <= 512) + % nextBest = [128,144,160,168,192,216,224,256,... + % 288,300,320,336,360,384,400,432,480,512]; + % else + % nextBest = [128,144,160,168,192,216,224,256,... + % 288,300,320,336,360,384,400,432,480,512,... + % 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]; + % end + + nextBest = repmat([128,144,160,168,192,216,224,256,... + 288,300,320,336,360,384,400,432,480,512,... + 540,576,640,648,720,756,768,810,864,896,960,972,1008,1024]',1,3); + + nextBest(nextBest(:,1) > SIZES(1,1),1) = 16; + nextBest(nextBest(:,2) > SIZES(1,2),2) = 16; + nextBest(nextBest(:,3) > SIZES(1,3),3) = 16; sizeImage = SIZES(2,:); sizeTemplate = SIZES(3,:); % the mask or kernel - sizeParticle = SIZES(4,:) ;% a subregion of sizeTemplate + sizeParticle = SIZES(4,:);% a subregion of sizeTemplate borderSizeCalc = floor((sizeTemplate + APODIZATION)./2); borderSizeKeep = borderSizeCalc + 2.*sizeParticle; -abs(sum(sizeImage - sizeTemplate)) - sum(0.1.*sizeImage) + if abs(sum(sizeImage - sizeTemplate)) < sum(0.1.*sizeImage) OUTPUT = [[0,0,0];[0,0,0] ;sizeImage; ... sizeImage; sizeImage ; [1,1,1]]; return end - score = zeros(length(nextBest),6); - + score = zeros(size(nextBest,1),10); - validCalc = repmat(nextBest',1,3) - 2.*repmat(borderSizeCalc,length(nextBest),1); - validKeep = repmat(nextBest',1,3) - 2.*repmat(borderSizeKeep,length(nextBest),1); - minIter= floor(repmat(sizeImage,length(nextBest),1)./validKeep); - postPad= repmat(nextBest',1,3)- ... - (repmat(sizeImage+borderSizeKeep,length(nextBest),1) -minIter.*(validKeep+1)); + validCalc = nextBest - 2.*repmat(borderSizeCalc,size(nextBest,1),1); + validKeep = nextBest - 2.*repmat(borderSizeKeep,size(nextBest,1),1); + minIter= floor(repmat(sizeImage,size(nextBest,1),1)./validKeep); + postPad= nextBest- ... + (repmat(sizeImage+borderSizeKeep,size(nextBest,1),1) -minIter.*(validKeep+1)); + + % Penalize large Z % Added this so I can work with test cases where the volume to be % searched is the same size as the reference - - score(:,2:4) = repmat(nextBest',1,3) ./ postPad .* (minIter >= 0) - + score(:,5:7) = minIter; + score(:,2:4) = nextBest ./ postPad .* (minIter >= 0); + adjustForIterations = minIter + 1; + adjustForIterations = (adjustForIterations).^2; + adjustForIterations(~isfinite(adjustForIterations)) = 1; + score(:,2:4) = score(:,2:4) ./ adjustForIterations; + score(:,1) = sum(score(:,2:4),2); + score + % Testing out the best overall score now [~, cX] = max(score(:,2)) ; [~, cY] = max(score(:,3)) ; - [~, cZ] = max(score(:,4)) ; + [~, cZ] = max(score(:,4)) ; chunkSize = nextBest([cX,cY,cZ]); + + % [ ~, best_score ] = max(score(:,1)); + % chunkSize = nextBest(best_score.*[1,1,1]); + - cY = cY + length(nextBest); - cZ = cZ + 2.*length(nextBest); + % cX = best_score; + % cY = best_score; + % cZ = best_score; + % TODO: review what is going on here + cY = cY + size(nextBest,1); + cZ = cZ + 2.*size(nextBest,1); validAreaKeep = validKeep([cX,cY,cZ]); validAreaCalc = validCalc([cX,cY,cZ]); nIters = minIter([cX,cY,cZ])+1; diff --git a/coordinates/BH_multi_recGeom.m b/coordinates/BH_multi_recGeom.m index 208e361c..c7665a01 100755 --- a/coordinates/BH_multi_recGeom.m +++ b/coordinates/BH_multi_recGeom.m @@ -17,6 +17,11 @@ nTomos = recFile.data(1); recCoords = recFile.data(2:end); +if (nTomos == 0) + fprintf("WARNING: No tomograms found in %s\n", reconCoordName); + error('The number of tomograms is zero'); +end + tilt_geometry_name = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', tiltName, mapBackIter+1); try tilt_geometry = load(tilt_geometry_name); diff --git a/ctf/BH_ctfCalc.m b/ctf/BH_ctfCalc.m index c4eb7518..06bd164b 100755 --- a/ctf/BH_ctfCalc.m +++ b/ctf/BH_ctfCalc.m @@ -214,7 +214,9 @@ else bFactor = 100; end - ctfMask = BH_bandpass3d(size(Hqz),0,800,lowCut,'GPU',maxRes); + + + ctfMask = BH_bandpass3d(size(Hqz),0.01,800,lowCut,'GPU',maxRes); % This term is straight from dTegunov's deconv snr = 10.^3.*exp((-2.2.*bFactor).*radialGrid); ctfMask = ctfMask .* Hqz ./ (Hqz.^2 + 1./snr); diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 3a53b3d1..36958ba0 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -69,31 +69,31 @@ recon_for_tomoCPR = false; recon_for_templateMatching = false; recon_for_subTomo = false; -if nargin > 2 - if isempty(EMC_str2double(varargin{1})) - error('Extra argument to ctf 3d should be a vector [THICKNESS, BINNING] tiltN, or a string templateSearch'); - else - reconstructionParameters = EMC_str2double(varargin{1}); - if length(varargin) > 2 - recon_for_tomoCPR = true; - % Full recon for tomoCPR - bh_global_turn_on_phase_plate = varargin{3}; - filterProjectionsForTomoCPRBackground = varargin{4}; - if length(varargin) > 4 - use_existing_tmpCache = varargin{5}; - end - else - error('This block should ont be reached.'); - end - end -elseif nargin > 1 + +recon_subset=[1,-1]; + +if nargin > 1 if strcmpi(varargin{1},'templateSearch') recon_for_templateMatching = true; if (bh_global_turn_on_phase_plate(1)) fprintf('WARNING: the filtered tomogram should only be used for viz, not template matching.'); end else - error('Extra argument to ctf 3d should be a vector [THICKNESS, BINNING] tiltN, or a string templateSearch'); + if strcmpi(varargin{1},'split') + error('Extra argument to ctf 3d either templateSearch/split and optionally a vector [iProjcess, nProcesses (from 1)]'); + end + end + + if nargin > 2 + if isempty(EMC_str2double(varargin{2})) + error('Extra argument to ctf 3d either templateSearch/split and optionally a vector [iProjcess, nProcesses (from 1)]'); + else + recon_subset = EMC_str2double(varargin{2}); + if numel(recon_subset) ~= 2 + error('Extra argument to ctf 3d either templateSearch/split and optionally a vector [iProjcess, nProcesses (from 1)]'); + end + + end end else % Default to zero for normal use @@ -103,6 +103,8 @@ end end +fprintf('recon_subset is [%d,%d]\n',recon_subset(1),recon_subset(2)); + if (recon_for_tomoCPR + recon_for_templateMatching + recon_for_subTomo ~= 1) error('Only one of the three modes can be used at a time'); end @@ -142,9 +144,6 @@ invertDose = 0; end -%cycleNumber = sprintf('cycle%0.3d',CYCLE); -%fprintf('cycle is %d\n',CYCLE); - fprintf('tiltweight is %f %f\n',tiltWeight); @@ -289,14 +288,19 @@ % If there is only one tilt, things break in a weird way nGPUs = min(nGPUs, nTilts); -[ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores); +if (recon_subset(2) > 0) + [ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores, recon_subset); +else + [ nParProcesses, iterList] = BH_multi_parallelJobs(nTilts, nGPUs, 256, emc.nCpuCores); +end + try EMC_parpool(nParProcesses) catch delete(gcp('nocreate')) EMC_parpool(nParProcesses) -end +end parfor iParProc = 1:nParProcesses @@ -371,9 +375,9 @@ % All data is handled through disk i/o so everything unique created in the parfor iParProc = 1:nParProcesses - % for iParProc = 1:nParProcesses %%revert + % for iParProc = 1:nParProcesses % iGPU = mod(iParProc,nGPUs); -% for iGPU = 1:nGPUs %%revert +% for iGPU = 1:nGPUs % % for iGPU = 1:nGPUs gpuDevice(iGPU+1); @@ -1135,7 +1139,6 @@ [Hqz, ~] = BH_ctfCalc(radialGrid,Cs,WAVELENGTH,defVect,fastFTSize,AMPCONT,-1,1,SNR); Hqz = (-1).^modPower.*(phakePhasePlate(1).*Hqz).^1; - modHqz = []; else if (pixel_size_angstroms < 2.0) @@ -1172,6 +1175,7 @@ correctedStack(:,:,TLT(iPrj,1)) = gather(correctedPrj./samplingMask); end + clear correctedPrj samplingMask tmpMask tmpCorrection clear iProjection iProjectionFT diff --git a/ctf/BH_ctf_Estimate.m b/ctf/BH_ctf_Estimate.m index 62efad8d..20d28152 100755 --- a/ctf/BH_ctf_Estimate.m +++ b/ctf/BH_ctf_Estimate.m @@ -143,7 +143,7 @@ % Tile size & overlap -tileOverlap = 2; +tileOverlap = emc.('ctf_tile_overlap'); % emc.ctf_tile_size = max(emc.ctf_tile_size, 384); if (emc.ctf_tile_size > 512) @@ -447,23 +447,14 @@ % that is the reason for the negative sign iEvalMask = iEvalMask.*(-1.*tand(TLT(tiltIDX,4))); - iEval_farther_from_focus = iEvalMask; - iEval_closer_to_focus = iEvalMask; % zShift is in SI in parameter file, but converted to pixels here. - % Select region that is at a smaller Z coordinate (farther from focus) - iEval_farther_from_focus = iEval_farther_from_focus - emc.zShift; - % Select region that is at a larger Z coordinate (closer to focus) - iEval_closer_to_focus = iEval_closer_to_focus + emc.zShift; - % Select region limited by defocus tolerance iEvalMask = ( iEvalMask > gpuArray(-emc.deltaZTolerance) & iEvalMask < gpuArray(emc.deltaZTolerance)); - iEval_farther_from_focus = ( iEval_farther_from_focus > gpuArray(-emc.deltaZTolerance) & iEval_farther_from_focus < gpuArray(emc.deltaZTolerance)); - iEval_closer_to_focus = ( iEval_closer_to_focus > gpuArray(-emc.deltaZTolerance) & iEval_closer_to_focus < gpuArray(emc.deltaZTolerance)); - tmpTile = zeros([halfX,paddedSize,3],'single','gpuArray'); + tmpTile = zeros([halfX,paddedSize,1],'single','gpuArray'); % % % % tmpTile = zeros([paddedSize.*[1,1],3],'single','gpuArray'); @@ -488,7 +479,7 @@ for i = 1+emc.ctf_tile_size/2:overlap:d1C-emc.ctf_tile_size/2 - if min([nT,nT2,nT3])< emc.ctfMaxNumberOfTiles && (iEvalMask(i) || iEval_farther_from_focus(i) || iEval_closer_to_focus(i)) + if min([nT,nT2,nT3])< emc.ctfMaxNumberOfTiles && (iEvalMask(i)) for j = 1+emc.ctf_tile_size/2:overlap:d2C-emc.ctf_tile_size/2 thisTile = iProjection( i-emc.ctf_tile_size/2+1:i+emc.ctf_tile_size/2,... @@ -509,19 +500,10 @@ nT = nT+1; tmpTile(:,:,1) = tmpTile(:,:,1) + thisTile; end - if ( iEval_farther_from_focus(i) ) - nT2 = nT2+1; - tmpTile(:,:,2) = tmpTile(:,:,2) + thisTile; - end - if (iEval_closer_to_focus(i) ) - nT3 = nT3+1; - tmpTile(:,:,3) = tmpTile(:,:,3) + thisTile; - end end % end of j end % end of if iEvalMask end % end of i - fprintf('%d tiles at dZ= 0\t%d tiles at dZ > 0\t%d tiles at dZ < 0, after tilt %d\n',nT,nT2,nT3,k); % Apply the dose filter to the sum of each projection to save a bunch of % multiplicaiton @@ -531,12 +513,11 @@ clear tmpTile toc - rotAvgPowerSpec = zeros([paddedSize,paddedSize,3],'single','gpuArray'); - for iTile = 1:3 + rotAvgPowerSpec = zeros([paddedSize,paddedSize,1],'single','gpuArray'); % tmp = bhF2.swapIndexFWD(psTile(:,:,iTile)); + iTile = 1; psTile(:,:,iTile) = bhF2.swapIndexFWD(psTile(:,:,iTile)); rotAvgPowerSpec(:,:,iTile) = BH_multi_makeHermitian(psTile(:,:,iTile),[paddedSize,paddedSize],1); - end clear psTile @@ -552,7 +533,7 @@ ROT1 = R(1).*rot1 + R(4).*rot2; ROT2 = R(2).*rot1 + R(5).*rot2; - for iTile = 1:3 + for iTile = 1 rotAvgPowerSpec(:,:,iTile) = rotAvgPowerSpec(:,:,iTile) + ... interpn(r1,r2,AvgPowerSpec(:,:,iTile),... ROT1,ROT2,'linear',0); @@ -574,10 +555,9 @@ currentDefocusEst = defEST; currentDefocusWin = defWIN; - measuredVsExpected = zeros(2,3); end % end of not skip fitting - for iTilt = 1:3 + for iTilt = 1 if (skipFitting && resample_stack) currentDefocusEst = defEST; @@ -790,7 +770,6 @@ refineCCC(n,:) = [iAng+mAng,mDef + iDelDF,iCCC]; n = n + 1; - fprintf('%d / %d fine astigmatism search\n',n,size(refineCCC,1)); end end end @@ -837,32 +816,12 @@ '%d\t%d\t%d\t%8.2f\n'], TLT'); fclose(fileID); - elseif iTilt == 2 - measuredVsExpected(2,1) = maxDef; - elseif iTilt == 3 - measuredVsExpected(2,3) = maxDef; end % Stuff we only do on the full determin (tilt1) end % Loop on handedness check - if sum(abs(diff(measuredVsExpected,1))) > sum(abs(measuredVsExpected(1,:) - flip(measuredVsExpected(2,:)))) - warnInvertedHand = 1; - else - warnInvertedHand = 0; - end - - fprintf('\n******************************************************\n\n'); - fprintf('\Farther from focus |\tAt focus |\Closer to focus\n\n'); - fprintf('Expected defocus %3.2f %3.2f %3.2f\n\n', abs(measuredVsExpected(1,:))); - fprintf('Measured defocus %3.2f %3.2f %3.2f\n\n' ,abs(measuredVsExpected(2,:))); - if ( warnInvertedHand ) - fprintf('\nIt looks like your handedness may be inverted!!\n'); - else - fprintf('\nIt looks like your handedness is probably correct.\n'); - end - fprintf('\n******************************************************\n\n\n'); else - + % Found an average score: 0.065745 and an average inverted hand score: 0.214261 for tilt tilt60_ali1_ctf if (resample_stack) [~, idx] = sortrows(abs(TLT(:,4)), -1); TLT = TLT(idx,:); diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index e062e4b2..d9ff14fb 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -72,7 +72,7 @@ end % Tile size & overlap -tileOverlap = 4; +tileOverlap = emc.('ctf_tile_overlap'); emc.ctf_tile_size = BH_multi_iterator(emc.ctf_tile_size.*[1,1],'fourier2d'); emc.ctf_tile_size = emc.ctf_tile_size(1); @@ -309,10 +309,11 @@ oXprj = ceil((size(iProjection,1)+1)./2); % Don't worry about extending the edges for thickness +half_width = (size(iProjection,1)/2); halfX = emc_get_origin_index(paddedSize); maxEval = (fraction_of_extra_tilt_data + ... - cosd(TLT(iPrj,4)).*(1-fraction_of_extra_tilt_data)) .* halfX; + cosd(TLT(iPrj,4)).*(1-fraction_of_extra_tilt_data)) .* half_width; iEvalMask = floor(oXprj-maxEval):ceil(oXprj+maxEval); % iEvalMask = BH_multi_gridCoordinates([size(iProjection,1),1,1],'Cartesian','GPU',{'none'},0,1,0); @@ -413,7 +414,7 @@ iTile = gpuArray(iProjection(iOuter-tileSize/2+1:iOuter+tileSize/2,y-tileSize/2+1:y+tileSize/2)); - % iTile(zeroed_coords,:) = 0; + iTile(zeroed_coords,:) = 0; iTile = iTile - mean(iTile(:)); iTile = iTile ./ rms(iTile(:)); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index e4db7fcf..5cf0c64d 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -776,6 +776,11 @@ end emc.ctf_tile_size = emc.ctf_tile_size + mod(emc.ctf_tile_size,2); +if isfield(emc, 'ctf_tile_overlap') + EMC_assert_numeric(emc.ctf_tile_overlap, 1); +else + emc.ctf_tile_overlap = 2; +end if isfield(emc, 'deltaZTolerance') EMC_assert_numeric(emc.deltaZTolerance, 1, [10e-9, 300e-9]); diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index ea813c07..8714fb50 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -18,9 +18,7 @@ buildTomo=1;% % % % % % % save_diagnostic_ccf=0; -% If false, this is faster, simplifies the code and permits defocus estimation -% This will likely be removed in favor of deleting all the blockes under its control -use_background_estimate = false; + % Default true, we don't need this after projection delete_background_estimate = true; @@ -425,34 +423,8 @@ sprintf('[%d,%d]',maxZ,samplingRate); tiltNameList{iTiltSeries}; - if (use_background_estimate) - backgroundName = sprintf('%scache/%s_%d_bin%d_backgroundEst.rec',CWD,tiltNameList{iTiltSeries},1, samplingRate); - fprintf('In tomocpr, using background estimate %s\n\n',backgroundName); - send_backgroundLowPassResolution = 28; - % TODO: investigate deviations from the default, which is to shut off the phakePhasePlate and to use a backgroundLowPassResolution of 28 - % Default false, we don't apply this filter - % if enabled, it currently only saves the filtered background estimate for visualization in addition to the normal version - % if (emc.save_mapback_classes) - % BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, 1, 3, tmpCache); - % end - % FIXME: calling like this does not use the surface fit for the background - send_phakePhasePlateOption = [0,0]; - BH_ctf_Correct3d(PARAMETER_FILE,sprintf('[%d,%d]',maxZ,samplingRate),tiltNameList{iTiltSeries}, send_phakePhasePlateOption, send_backgroundLowPassResolution, tmpCache); - - % re-initialize the parpool for each tilt series to free up mem. - delete(gcp('nocreate')) - EMC_parpool(nWorkers); - - - avgTomo{1} = OPEN_IMG('single',backgroundName); - avgTomo{1} = avgTomo{1} ./ (rmsScale*rms(avgTomo{1}(:))); - if (delete_background_estimate) - system(sprintf('rm -f %s',backgroundName)); - end - else - avgTomo{1} = zeros(reconstruction_size, 'single'); - end % if (use_background_estimate) + avgTomo{1} = zeros(reconstruction_size, 'single'); for iRef = 1:nRefs refVol{1}{iRef} = gpuArray(refVol{1}{iRef}); diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index fdbf2a0f..9e636e8b 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -841,7 +841,9 @@ stack_boundaries(end) = outputNumberOfSlices+1; end +do_initial = false +if (do_initial) % revert % %%%%%%%%%%%%%%%%%%%%%%%%% % Initial reconstruction % %%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -914,7 +916,7 @@ break; end end - if all_found + if all_founddo_initial break; else fprintf('Waiting for reconstructions to finish...\n'); @@ -1335,7 +1337,7 @@ system(sprintf('rm %s/%sdump_?_*.dat',tmpCache,output_prefix)); - +end % revert do _intial % Get the FSC cutoff for refinement @@ -1343,7 +1345,7 @@ fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) -if ~(do_initial) +if ~(do_initial) % revert fsc_last = fsc_cutoff / 0.94; end n_max_refinements = 7; From 915f0c56041c358a77eec533f9caddd4218ef4b7 Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 4 Mar 2025 14:54:24 -0500 Subject: [PATCH 089/151] forgot a debug breakpoint --- alignment/BH_templateSearch3d_2.m | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 2e99cc5d..95c7b192 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -320,7 +320,7 @@ fprintf('valid Calc %d %d %d\n', validCalc); fprintf('# of iterations %d %d %d\n', nIters); fprintf('-----\n'); -error('asdf') + valid_ratio = prod(sizeChunk) ./ prod(validCalc); size(tomogram) From 695eaeab15510031659c8ba90c05a041dd77e2ca Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 4 Mar 2025 17:06:38 -0500 Subject: [PATCH 090/151] swapped order of loop in TM for big speed up --- alignment/BH_templateSearch3d_2.m | 556 ++++++------------------------ coordinates/BH_multi_iterator.m | 10 +- metaData/BH_parseParameterFile.m | 6 + 3 files changed, 116 insertions(+), 456 deletions(-) diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 95c7b192..27b65943 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -9,7 +9,6 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -test_half = false; ctf3dNoSubTomoMeta = true; if length(varargin) > 0 @@ -51,11 +50,7 @@ samplingRate = emc.('Tmp_samplingRate'); -try - tmpDecoy = emc.('templateDecoy') -catch - tmpDecoy = 0 -end +test_half = emc.('Tmp_half_precision'); @@ -126,11 +121,6 @@ mean_r2 = 0; mean_r_mask = 0; reference_mask = []; -try - scale_mip = emc.('scale_mip'); -catch - scale_mip = false; -end try measure_noise_variance = emc.('measure_noise_variance'); @@ -284,32 +274,13 @@ %[ padVal ] = BH_multi_padVal( sizeTemp, sizeChunk ); %tempPre = padVal(1,:); %tempPost = padVal(2,:); - [ padBIN ] = BH_multi_padVal( sizeTempBIN, sizeChunk ); [ trimValid ] = BH_multi_padVal(sizeChunk, validArea); + RMSFACTOR = sqrt(prod(sizeTempBIN) / prod(sizeChunk)); -if ( tmpDecoy ) - % This is probably sample dependent. should search a small range and find - % the maximum rate of change in the ccc - - % the -1 searches for the next smallest fast fourier size - templateBIN = gpuArray(templateBIN); - - - - decoyTest = BH_reScale3d(templateBIN,'',tmpDecoy,'GPU'); - decoyTrim = BH_multi_padVal(size(decoyTest),size(templateBIN)); - decoyTest = fftn(BH_padZeros3d(decoyTest,decoyTrim(1,:),decoyTrim(2,:),'GPU','single')); - decoyShift = -1.*gather(BH_multi_xcf_Translational(decoyTest,conj(fftn(templateBIN)),'',[3,3,3])); - decoyNorm = gather(sum(abs(decoyTest(:)))./sum(abs(fftn(templateBIN(:))))); - padDecoy = BH_multi_padVal(size(decoyTest),sizeChunk) + decoyTrim; - clear decoyTest - templateBIN = gather(templateBIN); - fprintf('tmpDecoy %f normFactor %f and shift by %2.2f %2.2f %2.2f\n',tmpDecoy,decoyNorm,decoyShift); - -end + fprintf('\n-----\nProcessing in chunks\n\n'); @@ -342,9 +313,6 @@ % Array for storing chunk results these could probably be half-precision RESULTS_peak = zeros(sizeTomo, 'single'); RESULTS_angle= zeros(sizeTomo, 'single'); -if ( tmpDecoy ) - RESULTS_decoy = RESULTS_peak; -end RESULTS_sum = []; RESULTS_sum_sq = []; @@ -383,17 +351,6 @@ tomoIDX = 1; nTomograms = prod(nIters); -try - test_local = emc.('test_local'); -catch - test_local = false; -end -if (test_local) - scale_mip = false; -end - -test_local -scale_mip wanted_storage_precision = 'single'; if (test_half) @@ -401,9 +358,6 @@ end tomoStack = zeros([sizeChunk,nTomograms], wanted_storage_precision); -if test_local - localStack = zeros([sizeChunk,nTomograms], wanted_storage_precision); -end % tomoNonZero = zeros(nTomograms,6,'uint64'); % backgroundVol = zeros(sizeChunk,'single'); @@ -482,17 +436,7 @@ rmsMask = sqrt(rmsMask - averageMask.^2); - if (test_local) - if (test_half) - localStack(:,:,:,tomoIDX) = emc_halfcast(rmsMask, true); - else - localStack(:,:,:,tomoIDX) = gather(rmsMask); - end - else - if ~(scale_mip) - tomoChunk = (tomoChunk - averageMask) ./ rmsMask; - end - end + tomoChunk = (tomoChunk - averageMask) ./ rmsMask; clear rmsMask averageMask tomoChunk = gather(tomoChunk .*validCalcMask); @@ -532,9 +476,7 @@ -currentGlobalAngle = 1; ANGLE_LIST = zeros(nAngles(1),3, 'single'); -nComplete = 0; totalTime = 0; firstLoopOverTomo = true; @@ -546,33 +488,17 @@ theta_search = 1:size(angleStep,1); end +tomoIDX = 1; +firstLoopOverAngles = true; -for iAngle = theta_search - - if (emc.use_new_grid_search) - theta = gridSearch.parameter_map.theta(iAngle); - numRefIter = gridSearch.number_of_angles_at_each_theta(iAngle); - else - theta = angleStep(iAngle,1); - phiStep = angleStep(iAngle,3); - numRefIter = angleStep(iAngle,2)*length(inPlaneSearch)+1; - end - - - +for iTomo = 1:nTomograms + currentGlobalAngle = 1; + + fprintf('Working on tomo chu %d/%d\n',iTomo,nTomograms); tempImg = gpuArray(templateBIN); %%%%% NEW switch to bin - - % interpolationNormFactor = sum(abs(tempImg(:)).^2); - - - - tomoIDX = 1; - firstLoopOverAngle = true; - % Avoid repeated allocations tempPAD = zeros(size(tempImg) + padBIN(1,:) + padBIN(2,:),'single','gpuArray'); - tempPADMask = tempPAD; template_interpolator = ''; [template_interpolator, ~] = interpolator(tempImg,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); @@ -581,57 +507,40 @@ [templateMask_interpolator, ~] = interpolator(gpuArray(templateMask),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + % Iterate over the tomogram pulling each chunk one at a time. + % for iTomo = 1:nTomograms sqp loop + tic; + iCut = tomoCoords(iTomo,:); + % reset the angle count and value at the begining of loop + % inside, while each new outer loop changes the start values. - % Iterate over the tomogram pulling each chunk one at a time. - for iTomo = 1:nTomograms - tic; - iCut = tomoCoords(tomoIDX,:); - % reset the angle count and value at the begining of loop - % inside, while each new outer loop changes the start values. + % Truth value to initialize temp results matrix each new tomo + % chunk. + firstLoopOverChunk = true; - % nAngle = angleIncStart; - intraLoopAngle = 1; - - % Truth value to initialize temp results matrix each new tomo - % chunk. - firstLoopOverChunk = true; + + if (test_half) + % Convert and return on GPU + tomoFou = emc_halfcast(tomoStack(:,:,:,iTomo), true); + else + tomoFou = gpuArray(tomoStack(:,:,:,iTomo)); + end + + tomoFou = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)), 'fwd'); + + for iAngle = theta_search + + + if (emc.use_new_grid_search) - fprintf('Working on tilt(%d/%d) tomoChunk(%d/%d)\t' ... - ,iAngle,gridSearch.number_of_out_of_plane_angles, tomoIDX,nTomograms); + theta = gridSearch.parameter_map.theta(iAngle); else - fprintf('working on tilt(%d/%d) tomoChunk(%d/%d)\t' ... - ,iAngle,size(angleStep,1), tomoIDX,nTomograms); - end - - if (test_half) - % Convert and return on GPU - tomoFou = emc_halfcast(tomoStack(:,:,:,tomoIDX), true); - else - tomoFou = gpuArray(tomoStack(:,:,:,tomoIDX)); - end - - if test_local - if (test_half) - localFou = BH_padZeros3d(emc_halfcast(localStack(:,:,:,tomoIDX), true),trimValid(1,:),trimValid(2,:),'GPU','single'); - else - localFou = BH_padZeros3d(localStack(:,:,:,tomoIDX),trimValid(1,:),trimValid(2,:),'GPU','single'); - end + theta = angleStep(iAngle,1); + phiStep = angleStep(iAngle,3); + end - % localStack(:,:,:,tomoIDX)); - end - % % profile on - if (scale_mip) - tomoFou_2 = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou.^2)), 'fwd'); - end - - - tomoFou = bhF.swapPhase(bhF.fwdFFT(bhF.normalization_factor^3.*(tomoFou)), 'fwd'); - - - - if (emc.use_new_grid_search) phi_search = gridSearch.parameter_map.phi{iAngle}; else @@ -640,7 +549,6 @@ for iAzimuth = phi_search - if (emc.use_new_grid_search) phi = iAzimuth; else @@ -652,367 +560,126 @@ psi = iInPlane; %calc references only on first chunk - if (firstLoopOverAngle) - + if (firstLoopOverAngles) ANGLE_LIST(currentGlobalAngle,:) = [phi, theta, psi - phi]; - end - [ tempRot ] = template_interpolator.interp3d(... - [phi, theta, psi - phi],... - [0,0,0],rotConvention,... - 'forward','C1'); - - + % rather than using padzeros tempPAD = tempPAD .* 0; tempPAD(padBIN(1,1)+1: end - padBIN(2,1), ... padBIN(1,2)+1: end - padBIN(2,2), ... - padBIN(1,3)+1: end - padBIN(2,3)) = tempRot; + padBIN(1,3)+1: end - padBIN(2,3)) = template_interpolator.interp3d(... + [phi, theta, psi - phi],... + [0,0,0],rotConvention,... + 'forward','C1'); tempPAD = tempPAD - mean(tempPAD(:)); - if (scale_mip) - % I should probaly switch to using the SF3D masked reference, but that also changes the baseline implementation - % so I'll leave it for now. - - tempPADMask = tempPADMask .* 0; - - tempPADMask(padBIN(1,1)+1: end - padBIN(2,1), ... - padBIN(1,2)+1: end - padBIN(2,2), ... - padBIN(1,3)+1: end - padBIN(2,3)) = templateMask_interpolator.interp3d(... - [phi, theta, psi - phi],... - [0,0,0],rotConvention,... - 'forward','C1'); - - tempPADMask = tempPADMask ./ sum(tempPADMask(:)); - tempMaskFou = (conj(bhF.fwdFFT(bhF.normalization_factor.^-1 .* tempPADMask )));% ./ (sum(tempPADMask > 0.01)./sum(tempPADMask(:))))); - md = BH_padZeros3d(... - real(bhF.invFFT(tomoFou_2.*tempMaskFou)) - real(bhF.invFFT(tomoFou.*tempMaskFou)).^2,... - trimValid(1,:),trimValid(2,:),'GPU','single'); - - mip_scaling = sqrt(md); - - - - end - - - tempFou = conj(bhF.fwdFFT(tempPAD)); - - ccfmap = BH_padZeros3d(real(single(... - bhF.invFFT(tomoFou.*tempFou))),...%./(tomoNorm.*tempNorm))))),... + bhF.invFFT(tomoFou.* conj(bhF.fwdFFT(tempPAD))))),...%./(tomoNorm.*tempNorm))))),... trimValid(1,:),trimValid(2,:),'GPU','single'); % - if ~(scale_mip) - ccfmap = ccfmap ./ std(ccfmap(:)); - end - - if ( tmpDecoy > 0 ) - - if (firstLoopOverAngle) - - decoy = BH_padZeros3d(BH_reScale3d(tempRot./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - padDecoy(1,:),padDecoy(2,:),'GPU','single'); - else - % Probably just make a second decoy stack to avoid - % re-interpolating. If it works, then do this. - error('This is temp broken with new interpolator'); - % decoy = BH_padZeros3d(BH_reScale3d(referenceStack(:,:,:,intraLoopAngle)./decoyNorm,'',tmpDecoy,'GPU',decoyShift),... - % padDecoy(1,:),padDecoy(2,:),'GPU','single'); - end - - - - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*conj(fftn(decoy)))))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU','single'); - - - elseif ( tmpDecoy < 0 ) - - % Just use the mirror image of the template, i.e. take the conj - % (of the conj) so just the padded FFT of the ref. - decoy = BH_padZeros3d(fftshift(real(single( ... - ifftn(tomoFou.*tempFou)))),..../(decoyNorm.*tomoNorm))))), - trimValid(1,:), ... - trimValid(2,:),'GPU','single'); - - end - - if (scale_mip) - - % tempFou = conj(bhF.fwdFFT(tempPADMask.*bhF.normalization_factor^2)); - % tempFou = conj(bhF.fwdFFT(tempPADMask.*bhF.normalization_factor^0)); - - % mip_scaling = (1.0/mean_r_mask) .* ... - % BH_padZeros3d(real(single(... - % bhF.invFFT(tomoFou.*tempFou))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single').^2; - - % mip_scaling = BH_padZeros3d(real(single(... - % bhF.invFFT(tomoFou_2.*tempFou))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single') ... - % - ... - % mip_scaling; - - - - - % try - - % mip_scaling = sqrt(mean_r2) .*sqrt(mip_scaling); % FIXME add check on zero - % catch - - % lowval = mip_scaling < 0; - % numel(mip_scaling) - % sum(lowval,'all') - % mean(mip_scaling(lowval),'all') - % mean(mip_scaling(~lowval),'all') - % fprintf('\nmean_r_mask %3.3e mean_r2 %3.3e\n',mean_r_mask,mean_r2); - % fprintf('norm factor %f\n', bhF.normalization_factor); - % a = conj(bhF.fwdFFT(tempPAD.^2)); - % b = conj(bhF.fwdFFT(tempPADMask)); - % a(2:end) = 0; - % b(2:end) = 0; - % a = bhF.invFFT(a); - % b = bhF.invFFT(b); - % fprintf('mean_r_mask %3.3e mean_r2 %3.3e\n',1.0/b(1),a(1)); - % fprintf('mean_r_mask %3.3e mean_r2 %3.3e\n',1.0/b(5),a(5)); - - - % fprintf('iAngle %d idx %d iAzimuth %d iInPlane %d\n',iAngle,tomoIDX,iAzimuth,iInPlane); - % SAVE_IMG(tempPADMask,sprintf('tempPADMask_%d.mrc',tomoIDX)); - % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX)); - % SAVE_IMG( BH_padZeros3d(real(single(... - % bhF.invFFT(tomoFou))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('tomoFou_%d.mrc',tomoIDX)); - % SAVE_IMG( BH_padZeros3d((real(single(... - % bhF.invFFT(tomoFou_2)))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('tomoFou_2_%d.mrc',tomoIDX)); - % SAVE_IMG(BH_padZeros3d(real(single(... - % bhF.invFFT(tomoFou_2.*tempFou))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single'), sprintf('mip_scaling_2_%d.mrc',tomoIDX)); - % SAVE_IMG(... - % BH_padZeros3d((real(single(... - % bhF.invFFT(tomoFou.*tempFou)))),...%./(tomoNorm.*tempNorm))))),... - % trimValid(1,:),trimValid(2,:),'GPU','single').^2, sprintf('mip_scaling_%d.mrc',tomoIDX)); - - % error('Faild on mip scaling caclulation'); - % end - - % mip_scaling(abs(mip_scaling) < 1e-6) = 1e-6; - - % print the min,max and mean of the mip_scaling - % fprintf('\nmip_scaling min %3.3e max %3.3e mean %3.3e\n',min(mip_scaling(:)),max(mip_scaling(:)),mean(mip_scaling(:))); - ccfmap = ccfmap ./ mip_scaling; - - % Now scale the CCF to be an SNR by using the global variance which should be mostly noise peaks. - ccfmap = ccfmap ./ std(ccfmap(:)); - - end + ccfmap = ccfmap ./ std(ccfmap(:)); - if test_local - % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX)); - ccfmap = ccfmap ./ localFou; - ccfmap = ccfmap ./ std(ccfmap(:)); - % SAVE_IMG(ccfmap,sprintf('ccfmap_%d.mrc',tomoIDX+1)); - % error('asdf') - end - clear tempRot % If first loop over tomo, initialize the storage volumes, if % first loop over the chunk but not over the tomo, pull storage % chunks from storage volume. - if (firstLoopOverTomo && firstLoopOverChunk) + if (firstLoopOverChunk) %store ccfmap as complex with phase = angle of reference magTmp = ccfmap; - if ( tmpDecoy ) - decoyTmp = decoy; - end angTmp = ones(size(magTmp), 'single','gpuArray'); - - firstLoopOverTomo = false; - firstLoopOverChunk = false; - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - - elseif (firstLoopOverChunk) - % These double cuts are old, and don't really make sense. Make - % this more consistant with current operations when there is - % time. - magTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - angTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); if (measure_noise_variance) - sumTmp = RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sumSqTmp = RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - end - - if ( tmpDecoy ) - decoyTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyTmp = gpuArray(decoyTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end - - - magTmp = gpuArray(magTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - angTmp = gpuArray(angTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - - if (measure_noise_variance) - sumTmp = gpuArray(sumTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - sumSqTmp = gpuArray(sumSqTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3))); - - sumTmp = sumTmp + ccfmap; - sumSqTmp = sumSqTmp + ccfmap.^2; + ccfmap(abs(ccfmap) > 3) = 0; + sumTmp = ccfmap; + sumSqTmp = ccfmap.^2; end firstLoopOverChunk = false; - - replaceTmp = ( magTmp < ccfmap ); - - magTmp(replaceTmp) = ccfmap(replaceTmp); - angTmp(replaceTmp) = currentGlobalAngle; - - - - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; - clear replaceTmp - else % update higher values of ccfmap with new reference if applicable. replaceTmp = ( magTmp < ccfmap ); magTmp(replaceTmp) = ccfmap(replaceTmp); angTmp(replaceTmp) = currentGlobalAngle; - if ( tmpDecoy ) - decoyTmp(decoyTmp < decoy) = decoy(decoyTmp < decoy); - end if (measure_noise_variance) + ccfmap(abs(ccfmap) > 3) = 0; sumTmp = sumTmp + ccfmap; sumSqTmp = sumSqTmp + ccfmap.^2; end - intraLoopAngle = intraLoopAngle + 1; - currentGlobalAngle = currentGlobalAngle + 1; clear replaceTmp - end - nComplete = nComplete + 1; - end - end - % profile viewer - % return - % After searching all angles on this chunk, but out meaningful - % portion for storage. - - % FIXME this double cutting and temporary allocation is ridiculous. - magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + end % end if firstLoopOverChunk + currentGlobalAngle = currentGlobalAngle + 1; + + end % end psi loop over in plane angles + end % end phi loop over azimuth angles + end % end theta loop over out of plane angles + + % FIXME this double cutting and temporary allocation is ridiculous. + magStoreTmp = RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1); + + + magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); + angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,2) + 1:end - vA(2,2), ... + vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); + + + RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; + + clear magStoreTmp + + RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + iCut(2):iCut(2)+sizeChunk(2)-1,... + iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; + clear angStoreTmp + + if (measure_noise_variance) + sumStoreTmp = RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... iCut(2):iCut(2)+sizeChunk(2)-1,... iCut(3):iCut(3)+sizeChunk(3)-1); - angStoreTmp = RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + sumSqStoreTmp = RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... iCut(2):iCut(2)+sizeChunk(2)-1,... iCut(3):iCut(3)+sizeChunk(3)-1); - magStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + sumStoreTmp(vA(1,1) + 1:end - vA(2,1), ... vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(magTmp); - angStoreTmp(vA(1,1) + 1:end - vA(2,1), ... + vA(1,3) + 1:end - vA(2,3)) = gather(sumTmp); + sumSqStoreTmp(vA(1,1) + 1:end - vA(2,1), ... vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(angTmp); + vA(1,3) + 1:end - vA(2,3)) = gather(sumSqTmp); - RESULTS_peak(iCut(1):iCut(1)+sizeChunk(1)-1,... + RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = magStoreTmp; + iCut(3):iCut(3)+sizeChunk(3)-1) = sumStoreTmp; - clear magStoreTmp + clear sumStoreTmp - RESULTS_angle(iCut(1):iCut(1)+sizeChunk(1)-1,... + RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = angStoreTmp; - clear angStoreTmp - - if (measure_noise_variance) - sumStoreTmp = RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - sumSqStoreTmp = RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - - - sumStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sumTmp); - sumSqStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(sumSqTmp); - - - RESULTS_sum(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sumStoreTmp; - - clear sumStoreTmp - - RESULTS_sum_sq(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = sumSqStoreTmp; - clear sumSqStoreTmp - end - - if ( tmpDecoy ) - decoyStoreTmp = RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1); - decoyStoreTmp(vA(1,1) + 1:end - vA(2,1), ... - vA(1,2) + 1:end - vA(2,2), ... - vA(1,3) + 1:end - vA(2,3)) = gather(decoyTmp); - RESULTS_decoy(iCut(1):iCut(1)+sizeChunk(1)-1,... - iCut(2):iCut(2)+sizeChunk(2)-1,... - iCut(3):iCut(3)+sizeChunk(3)-1) = decoyStoreTmp; - - end - + iCut(3):iCut(3)+sizeChunk(3)-1) = sumSqStoreTmp; + clear sumSqStoreTmp - tomoTime = toc; - totalTime = totalTime + toc; timeEstimate = totalTime * (nTomograms*nAngles(1)./(nComplete-1)); - fprintf('elapsed time = %f s est remain %f s\n', tomoTime, timeEstimate); - tomoIDX = tomoIDX + 1; - firstLoopOverAngle = false; - currentGlobalAngle = currentGlobalAngle - intraLoopAngle + 1; - end - currentGlobalAngle = currentGlobalAngle + intraLoopAngle - 1; - + firstLoopOverAngles = false; + end end %save('angle_list.txt','angle_list','-ascii'); clear tomoStack @@ -1034,14 +701,7 @@ 1+tomoPre(3):end-tomoPost(3)); end -if ( tmpDecoy ) - RESULTS_decoy = RESULTS_decoy(1+tomoPre(1):end-tomoPost(1),... - 1+tomoPre(2):end-tomoPost(2),... - 1+tomoPre(3):end-tomoPost(3)); - % RESULTS_decoy = RESULTS_decoy ./ std(RESULTS_decoy(:)); - RESULTS_decoy(RESULTS_decoy < 1) = 1; - -end + gpuDevice(useGPU); @@ -1067,20 +727,14 @@ noiseVarOUT = sprintf('./%s/%s_noise_variance.mrc',convTMPNAME,mapName); if (measure_noise_variance) - n_angles_searched = sum(any(ANGLES,2)); + n_angles_searched = sum(any(ANGLE_LIST,2)); noiseVar = (RESULTS_sum_sq./n_angles_searched - (RESULTS_sum./n_angles_searched).^2); + noiseVar(noiseVar == 0) = 1; + SAVE_IMG(noiseVar,{noiseVarOUT,'half'}); end % SAVE_IMG(MRCImage(RESULTS_angle),anglesOUT); -if ( tmpDecoy ) - decoyOUT = sprintf('./%s/%s_decoy.mrc',convTMPNAME,mapName); - SAVE_IMG(RESULTS_decoy,{decoyOUT,'half'}); - diffOUT = sprintf('./%s/%s_convmap-decoy.mrc',convTMPNAME,mapName); - decoyLogical = mag < RESULTS_decoy; - mag(decoyLogical) = 0; - mag(~decoyLogical) = mag(~decoyLogical) - RESULTS_decoy(~decoyLogical); clear RESULTS_decoy - SAVE_IMG(mag,{diffOUT,'half'}); -end + angleFILE = fopen(angleListOUT,'w'); fprintf(angleFILE,'%2.2f\t%2.2f\t%2.2f\n', ANGLE_LIST'); fclose(angleFILE); diff --git a/coordinates/BH_multi_iterator.m b/coordinates/BH_multi_iterator.m index 90f360e6..bb8434c3 100755 --- a/coordinates/BH_multi_iterator.m +++ b/coordinates/BH_multi_iterator.m @@ -146,17 +146,17 @@ score(:,5:7) = minIter; score(:,2:4) = nextBest ./ postPad .* (minIter >= 0); adjustForIterations = minIter + 1; - adjustForIterations = (adjustForIterations).^2; + adjustForIterations = (adjustForIterations); adjustForIterations(~isfinite(adjustForIterations)) = 1; score(:,2:4) = score(:,2:4) ./ adjustForIterations; score(:,1) = sum(score(:,2:4),2); score % Testing out the best overall score now - [~, cX] = max(score(:,2)) ; - [~, cY] = max(score(:,3)) ; - [~, cZ] = max(score(:,4)) ; + [~, cX] = max(score(:,2)); + [~, cY] = max(score(:,3)); + [~, cZ] = max(score(:,4)); - chunkSize = nextBest([cX,cY,cZ]); + chunkSize = [nextBest(cX,1), nextBest(cY,2), nextBest(cZ,3)]; % [ ~, best_score ] = max(score(:,1)); % chunkSize = nextBest(best_score.*[1,1,1]); diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 5cf0c64d..085a9ed5 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -757,6 +757,12 @@ emc.Tmp_bandpass = [0.001, 1200, 28]; end +if isfield(emc, 'Tmp_half_precision') + EMC_assert_boolean(emc.Tmp_half_precision); +else + emc.Tmp_half_precision = false; +end + if isfield(emc, 'Pca_bandpass') EMC_assert_numeric(emc.Pca_bandpass, 3); else From 92dcb7d19e53009d05d3baa7dc299882a380b317 Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 5 Mar 2025 06:25:59 -0500 Subject: [PATCH 091/151] v1.8.2.2 full_enchilada round_1 --- logicals/BH_isWindowValid.m | 40 +++---------------------------------- testScripts/mCompile.sh | 2 +- 2 files changed, 4 insertions(+), 38 deletions(-) diff --git a/logicals/BH_isWindowValid.m b/logicals/BH_isWindowValid.m index 3f226835..2b633ded 100755 --- a/logicals/BH_isWindowValid.m +++ b/logicals/BH_isWindowValid.m @@ -39,13 +39,7 @@ % %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% -use_noise_instead_of_ignoring = false; -if nargin > 4 - if strcmpi(varargin{1}, 'pca') - use_noise_instead_of_ignoring = true; - end -end -minSizeMask = (max(MASK_RADIUS)+6).*[2,2,2]; +minSizeMask = max(MASK_RADIUS) .* [2,2,2]; winLowCorner = ceil((WINDOW_SIZE-1) ./ 2); % if window size is odd then there should be as many pixels to the left and to % the right of the origin. mod(ODD -1,2) = 0 otherwise it is 1 setting this @@ -78,38 +72,10 @@ % Should keep track of this and figure into the quality weight somehow. availableArea = WINDOW_SIZE - PADVALUES(1,:) - PADVALUES(2,:); -if any(availableArea - minSizeMask < -2) - % fprintf(['\nvs %d %d %d\nws %d %d %d\nmr %2.1f %2.1f %2.1f\n',... - % 'minArea %d %d %d\navailArea %d %d %d\nc %2.1f %2.1f %2.1f\n'], ... - % VOLUME_SIZE, WINDOW_SIZE, MASK_RADIUS, minSizeMask, availableArea, CENTER); +if (any(availableArea - minSizeMask < -2) || any(isnan(PADVALUES(:)))) INDICES = 'noUse'; - PADVALUES = [availableArea]; -end -if any(isnan(PADVALUES(:))) - % fprintf('center %f %f %f\n',CENTER); - % fprintf('min %f %f %f\n',minSizeMask); - % fprintf('winLowCorner %f %f %f\n', winLowCorner); - % fprintf('top %f %f %f\n',winTopCorner); - % fprintf('%f %f %f\n',winCenter); - % fprintf('del %f %f %f\n',deltaWinCenter); - % fprintf('%f %f %f\n',LOW); - % fprintf('%f %f %f\n',TOP); - % error('\n\nFound a NaN in the pad values. But Why ben why?\n\n'); - INDICES='noUse'; - PADVALUES = [availableArea]; + PADVALUES = availableArea; end -% padSUM = sum(PADVALUES(:,1)) .* WINDOW_SIZE(2) .* WINDOW_SIZE(3); -% padSUM = padSUM + sum(PADVALUES(:,2)) .* WINDOW_SIZE(1) .* WINDOW_SIZE(3); -% padSUM = padSUM + sum(PADVALUES(:,3)) .* WINDOW_SIZE(1) .* WINDOW_SIZE(2); -% padSUM = padSUM ./ prod(WINDOW_SIZE); - -% minimal sampling -% if (padSUM) > .30 || any(any((PADVALUES - [WINDOW_SIZE;WINDOW_SIZE]) >0)) -% INDICES = 'noUse'; -% PADVALUES = padSUM; -% end - - end diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 6a4faeaa..16746437 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -45,7 +45,7 @@ outName="$(basename ${mFile} .m)${post}" major=1 minor=8 bugs=2 -nightly=1 +nightly=2 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" From 57a0d00f8477b4516eac4555c84e500752bd730c Mon Sep 17 00:00:00 2001 From: himesb Date: Wed, 5 Mar 2025 09:01:18 -0500 Subject: [PATCH 092/151] Remove check on duplicates during initialization since this is now done before averaging anyway. Change the duplicate check in RemoveDupicates to use the EMC_convn seperable kernel approach. --- ctf/BH_ctf_Updatefft.m | 8 +---- metaData/BH_geometryInitialize.m | 53 ++++---------------------------- metaData/BH_parseParameterFile.m | 6 ++++ metaData/BH_removeDuplicates.m | 6 ++-- transformations/BH_average3d.m | 7 +++-- 5 files changed, 20 insertions(+), 60 deletions(-) diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 99751875..fc5ff1c3 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -89,13 +89,7 @@ STACK_PRFX = ITER_LIST{iGPU}{iTilt}; - try - l = load(sprintf('fixedStacks/ctf/%s_ali3_ctf.tlt',STACK_PRFX)); - fprintf('FOUND geometry for %s\n',STACK_PRFX); - continue; - catch - fprintf('reloading geometry for %s\n',STACK_PRFX); - end + if (mapBackIter) mapBackPrfx = sprintf('mapBack%d/%s_ali%d_ctf',mapBackIter,STACK_PRFX,mapBackIter) diff --git a/metaData/BH_geometryInitialize.m b/metaData/BH_geometryInitialize.m index f32cfe2b..c4c0206e 100755 --- a/metaData/BH_geometryInitialize.m +++ b/metaData/BH_geometryInitialize.m @@ -95,7 +95,7 @@ splitOnTomos = emc.('fscGoldSplitOnTomos'); if (splitOnTomos) nGPUs = 1; - fprintf('override nGPUs to just 1 for initial step to evenly split crowded tomos because fscGoldSplitOnTomos is true') + fprintf('override nGPUs to just 1 for initial step to evenly split crowded tomos because fscGoldSplitOnTomos is true\n'); end % Resolution lower than this is not gold standard, and will also be mixed @@ -332,10 +332,11 @@ iHeader = getHeader(MRCImage(tiltName)); % first convert the imod model file to a temporary text file tmpFile = sprintf('tmp_%d.txt',iGPU); - system(sprintf('model2point convmap/%s.mod %s', mapName, tmpFile)) + fprintf('Converting model %s (%d/%d) file to search for peaks\n', mapName, iTomo, nTomos); + % TODO: send this to tmp cache + system(sprintf('model2point convmap/%s.mod %s &> /dev/null', mapName, tmpFile)); modGeom = load(tmpFile); system(sprintf('rm %s', tmpFile)); - flgLookForPoints = 1; % leave IDX in main memory because it is just for reference. sx = floor(iHeader.nX); @@ -344,52 +345,10 @@ catch fprintf('did not find convmap, assuming synthetic data, with no deleted model points\n'); - flgLookForPoints = 0; end - - % Make sure nothing has gone wrong in translating the convmap to the - % full size - if (flgLookForPoints) - - positionMatrix = zeros(sx, sy, sz, 'single', 'gpuArray'); - positionIDX = zeros(sx, sy, sz, 'uint32'); - - % Make a volume with ones in the position of the centers of the tomos. - for iSubTomo = 1:size(tmpSearchGeom,1) - - subTomoOrigin = fix(tmpSearchGeom(iSubTomo,11:13)./dupInTheLoop); - if any(subTomoOrigin < 1 + dupRadius) || any([sx,sy,sz] < subTomoOrigin + dupRadius) - tmpSearchGeom(iSubTomo,26:26:26*emc.nPeaks) = -9999; - else - positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = 1; - positionIDX(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... - tmpSearchGeom(iSubTomo, 4); - end - end % loop building position matrix - - for iSubTomo = 1:size(modGeom,1) - - subTomoOrigin = fix(modGeom(iSubTomo,:)); - if all(subTomoOrigin > 1) && all([sx,sy,sz] > subTomoOrigin) - - positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) = ... - positionMatrix(subTomoOrigin(1),subTomoOrigin(2),subTomoOrigin(3)) + 1; - end % sometimes a point gets moved out of bounds. - end - - % Convolve positionmatrix with duplicate mask. Just in case there are - % rounding errors at any point use convolution to check a neighborhood of - % +/- 1 pixel. - - overlapMatrix = convn(positionMatrix, gpuArray(dupMask), 'same'); - - idxList = positionIDX((overlapMatrix > 1)); - - tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom(ismember(tmpSearchGeom(:,4), idxList),:); - else + - tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom; - end + tomoResults.(fileInfo{iTomo,2}) = tmpSearchGeom; % Fix the retained points end diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 085a9ed5..5d4c247f 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -805,3 +805,9 @@ else emc.('ctfMaxNumberOfTiles') = 10000; end + +if isfield(emc, 'remove_duplicates') + EMC_assert_boolean(emc.remove_duplicates); +else + emc.remove_duplicates = true; +end diff --git a/metaData/BH_removeDuplicates.m b/metaData/BH_removeDuplicates.m index ec339f0b..52e98cd0 100755 --- a/metaData/BH_removeDuplicates.m +++ b/metaData/BH_removeDuplicates.m @@ -32,8 +32,8 @@ dupRadius = max(1,floor(0.2*min(latticeRadius)/pixelSize)); dupTolerance = (2.*dupRadius)+1; -dupMask(dupTolerance, dupTolerance, dupTolerance) = gpuArray(single(0)); -dupMask = dupMask + 1; + +dupMask = ones([1, dupTolerance], 'single','gpuArray'); if (nargin ~= 2) error('args = PARAMETER_FILE, CYCLE') @@ -94,7 +94,7 @@ % resulting matrix will correspond to the number of duplicates within the % specified radius. - overlapMatrix = convn(positionMatrix, dupMask, 'same'); + overlapMatrix = EMC_convn(positionMatrix, dupMask); % Positions inbetween particle origins will also be non-zero, so restrict % search to be particle origins that are within radius. diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index ebaaa200..368d571e 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -15,18 +15,19 @@ startTime = datetime("now"); CYCLE = EMC_str2double(CYCLE); +emc = BH_parseParameterFile(PARAMETER_FILE); if strcmpi(STAGEofALIGNMENT, 'RawAlignment') % Ensure we don't have any duplicates: TODO: add an override flag % This modifies the RawAlign geometry, so should be cycle -1 - if (CYCLE > 0) + if (CYCLE > 0 && emc.remove_duplicates) BH_removeDuplicates(PARAMETER_FILE,sprintf('%d', CYCLE-1)); end end cycleNumber = sprintf('cycle%0.3u', CYCLE); -emc = BH_parseParameterFile(PARAMETER_FILE); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); @@ -875,7 +876,7 @@ else if iCCC < avgCCC % Downweight higher frequency in all subTomos with iCCC below the mean - iBfactor = -1.*(emc.flgQualityWeight.*(acosd(iCCC) - acosd(avgCCC)))^2; + iBfactor = -1.*(emc.flgQualityWeight.*(acosd(iCCC/avgCCC) - acosd(avgCCC/avgCCC)))^2; iCCCweight = exp(iBfactor.*cccWeight); else From 84a53cfe7a6d3f9ecd42d104ee52a48f17bfdaa3 Mon Sep 17 00:00:00 2001 From: himesb Date: Sat, 16 Aug 2025 22:48:28 -0400 Subject: [PATCH 093/151] whole lotta chains in this mutha --- @MRCImage/SAVE_IMG.m | 11 +- alignment/BH_alignRaw3d_v2.m | 54 +- alignment/BH_templateSearch3d_2.m | 57 +- coordinates/BH_defineMatrix.m | 39 +- coordinates/eulerSearch.m | 23 +- ctf/BH_ctf_Correct3d.m | 274 +++----- ctf/BH_ctf_Refine2.m | 9 +- ctf/BH_ctf_Updatefft.m | 36 +- ctf/BH_runCtfFind.m | 4 +- logicals/BH_multi_parallelJobs.m | 1 + masking/BH_bandpass3d.m | 28 +- metaData/BH_geometryAnalysis.m | 612 ++++++++++++++---- metaData/BH_parseParameterFile.m | 58 +- metaData/BH_rawAlignmentsApply.m | 8 +- metaData/EMC_get_class_idx.m | 12 + metaData/EMC_parpool.m | 1 + metaData/emc_check_for_valid_image_file.m | 7 +- statistics/BH_clusterPub.m | 88 ++- statistics/BH_fscGold_class.m | 1 + statistics/BH_fscSplit.m | 11 + statistics/BH_multi_cRef_Vnorm.m | 38 +- statistics/BH_multi_cRef_wgtCritical.m | 7 +- statistics/BH_pcaPub.m | 7 +- synthetic/BH_synthetic_mapBack.m | 598 +++++++---------- synthetic/BH_to_cisTEM_mapBack.m | 86 ++- testScripts/EMC_assert_string_value.m | 29 + testScripts/adamOptimizer.m | 64 ++ testScripts/emClarity.m | 178 ++++- testScripts/emc_to_cpp.md | 94 +++ testScripts/fourierTransformer.m | 32 +- testScripts/logFile/emClarity.logfile | 32 + testScripts/mCompile.sh | 4 +- testScripts/optimize_gmm_weights.m | 106 +++ .../metaData/emc_parse_parameter_file.py | 212 ++++++ .../python/metaData/emc_type_traits.py | 59 ++ .../metaData/test_emc_parse_parameter_file.py | 48 ++ .../python/metaData/test_emc_type_traits.py | 52 ++ testScripts/test_adamOptimizer.m | 96 +++ transformations/BH_average3d.m | 137 +++- transformations/BH_montage4d_halfGrid.m | 2 +- 40 files changed, 2282 insertions(+), 933 deletions(-) create mode 100644 metaData/EMC_get_class_idx.m create mode 100644 testScripts/EMC_assert_string_value.m create mode 100644 testScripts/adamOptimizer.m create mode 100644 testScripts/emc_to_cpp.md create mode 100644 testScripts/logFile/emClarity.logfile create mode 100644 testScripts/optimize_gmm_weights.m create mode 100644 testScripts/python/metaData/emc_parse_parameter_file.py create mode 100644 testScripts/python/metaData/emc_type_traits.py create mode 100644 testScripts/python/metaData/test_emc_parse_parameter_file.py create mode 100644 testScripts/python/metaData/test_emc_type_traits.py create mode 100644 testScripts/test_adamOptimizer.m diff --git a/@MRCImage/SAVE_IMG.m b/@MRCImage/SAVE_IMG.m index d273361f..3e709aa7 100644 --- a/@MRCImage/SAVE_IMG.m +++ b/@MRCImage/SAVE_IMG.m @@ -134,7 +134,16 @@ end else % normal (not complex) data - if (mRCImage.header.minDensity == 0.0 && mRCImage.header.maxDensity ==0 ) + try + do_fix_header = mRCImage.header.minDensity == 0.0 && ... + mRCImage.header.maxDensity == 0.0; + catch + mRCImage.header.minDensity + mRCImage.header.maxDensity + error('Failed to read header min/max density values. Check the file is not corrupt.'); + end + + if (do_fix_header ) if numel(mRCImage.volume) < 768^3 mRCImage.header.minDensity = min(min(min(mRCImage.volume))); mRCImage.header.maxDensity = max(max(max(mRCImage.volume))); diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index b56d5b06..c97f2d0a 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -71,7 +71,6 @@ -flgRaw_shapeMask = 0;%= emc.('experimentalOpts')(3) samplingRate = emc.('Ali_samplingRate'); emc.pixel_size_angstroms = emc.pixel_size_angstroms.*samplingRate; @@ -99,9 +98,6 @@ % Check and override the rotational convention to get helical averaging. % Replaces the former hack of adding a fifth dummy value to the angular search -if ( emc.doHelical ) - rotConvention = 'Helical'; -end % if (emc.classification) % refName = emc.('Ref_className'); @@ -173,7 +169,28 @@ sortedTomoList = zeros(nTomograms,1); for iTomo = 1:nTomograms sortedTomoList(iTomo) = sum(geometry.(tomoList{iTomo})(:,26)~=-9999); + + % If we have previously aligned this tomo, it will be skipped, so don't count it + % Previously, only the existence of the file was checked in the parfor loop, but this + % adds a check that the number of lines matches the number of subtomos, so that if the alignment was + previousAlignment = sprintf('alignResume/%s/%s.txt',outputPrefix,tomoList{iTomo}); + if exist(previousAlignment,'file') + % Sometimes when multiple nodes are used, an extra line is added. + % In the parfor loop, multiple processes are prevented from writing this from the final iterlist but here we would have to use a random tmp file + tmp_filename = tempname; + system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s; ', previousAlignment, tmp_filename)); + test_load = importdata(tmp_filename); + if (size(test_load,1) ~= sortedTomoList(iTomo)) + fprintf('Warning: Number of lines in %s (%d) does not match number of subtomos for %s (%d)\n', previousAlignment, size(test_load,1), tomoList{iTomo}, sortedTomoList(iTomo)); + pause(1); + delete(previousAlignment); + else + fprintf('Skipping previously aligned tomogram %s\n', tomoList{iTomo}); + sortedTomoList(iTomo) = 0; + end + end end + [~, sortedTomoIDX] = sort(sortedTomoList,'descend'); tomoList = tomoList(sortedTomoIDX); % mask defines area for angular search, peakRADIUS restricts translational @@ -199,6 +216,7 @@ end end + if any(peakSearch > maskRadius) fprintf('\n\n\tpeakRADIUS should be <= maskRADIUS!!\n\n') peakSearch( (peakSearch > maskRadius) ) = ... @@ -322,16 +340,9 @@ end -if ( flgRaw_shapeMask ) - - [ volMask ] = gather(sqrt(volMask .* ... - EMC_maskReference(refIMG{1}{iRef}+refIMG{2}{iRef}, emc.pixel_size_angstroms, {'fsc', true}))); - -else - % % % % % % % [ volMask ] = gather(BH_mask3d(maskType, sizeWindow, maskRadius, maskCenter)); - [ volMask ] = gather(EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter})); + +[ volMask ] = gather(EMC_maskShape(maskType, sizeWindow, maskRadius, 'gpu', {'shift', maskCenter})); -end @@ -483,7 +494,6 @@ %%%%%%%%%%%%%%%%%%%%% Determine the angular search, if any are zero, don't %%%%%%%%%%%%%%%%%%%%% search at all in that dimension. -updateWeights = false; gridSearch = ''; if (emc.use_new_grid_search) gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... @@ -493,7 +503,9 @@ try symmetry_constrained_search = emc.('symmetry_constrained_search'); - fprintf('Using symmetry constrained search\n'); + if (symmetry_constrained_search) + fprintf('Using symmetry constrained search\n'); + end catch symmetry_constrained_search = false; end @@ -529,10 +541,7 @@ flgRefine = false; end - if sum(angleStep(:,2) > 0) - updateWeights = true; - else - end + end @@ -611,7 +620,7 @@ % TODO fix this workaround system(sprintf('awk ''{if($10 != "") print $0 }'' %s > %s_clean; mv %s_clean %s',... previousAlignment,previousAlignment,previousAlignment,previousAlignment)); - bestAngles_tmp.(tomoList{iTomo}) = load(previousAlignment); + bestAngles_tmp.(tomoList{iTomo}) = importdata(previousAlignment); fprintf('Using existing alignment info for %s\n', tomoList{iTomo}); else % There is some memory leak somewhere that I haven't been able to figure @@ -700,7 +709,7 @@ tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry_tmp.(tomoList{iTomo}); - + binShift = [0,0,0]; nSubTomos = size(positionList,1); @@ -1577,10 +1586,9 @@ % save('bestAnglesTemp.mat', 'bestAngles'); save('bestAngles.mat', 'bestAngles'); - [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, emc.nPeaks, rotConvention, updateWeights, emc.update_class_by_ccc); + [ rawAlign ] = BH_rawAlignmentsApply( gather(geometry), bestAngles, samplingRate, emc.nPeaks, rotConvention, emc.update_class_by_ccc); subTomoMeta.(cycleNumber).('RawAlign') = rawAlign; subTomoMeta.(cycleNumber).('newIgnored_rawAlign') = gather(nIgnored); - subTomoMeta.('updatedWeights') = true; clear bestAngles rawAlign save(emc.('subTomoMeta'), 'subTomoMeta'); diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 27b65943..8c7698dd 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -235,24 +235,24 @@ % Initialize a whole mess of control variables and storage volumes. % %Out of plane range inc (starts from 1.* inc) rotConvention = 'Bah'; -% Check and override the rotational convention to get helical averaging. -% Replaces the former hack of adding a fifth dummy value to the angular search -if ( emc.doHelical ) - rotConvention = 'Helical'; -end + if (emc.use_new_grid_search) gridSearch = eulerSearch(emc.symmetry, angleSearch(1),... angleSearch(2),angleSearch(3),angleSearch(4), 0, 0, false); + gridSearch.HelicalRestriction(emc.helical_search_theta_constraint); nAngles = sum(gridSearch.number_of_angles_at_each_theta); inPlaneSearch = gridSearch.parameter_map.psi; else - + if (emc.helical_search_theta_constraint ~= 0) + error('Helical search theta constraint not implemented for old grid search'); + end [ nInPlane, inPlaneSearch, angleStep, nAngles] ... = BH_multi_gridSearchAngles(angleSearch) end + highThr=sqrt(2).*erfcinv(ceil(peakThreshold.*0.10).*2./(prod(size(tomogram)).*nAngles(1))) @@ -481,30 +481,33 @@ firstLoopOverTomo = true; - if (emc.use_new_grid_search) - theta_search = 1:gridSearch.number_of_out_of_plane_angles; + theta_search = gridSearch.active_theta_positions; else theta_search = 1:size(angleStep,1); end + tomoIDX = 1; firstLoopOverAngles = true; + + +% Avoid repeated allocations +tempPAD = zeros(size(templateBIN) + padBIN(1,:) + padBIN(2,:),'single','gpuArray'); + +use_only_once = false; +template_interpolator = ''; +[template_interpolator, ~] = interpolator(gpuArray(templateBIN),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', use_only_once); + +% templateMask_interpolator = ''; +% [templateMask_interpolator, ~] = interpolator(gpuArray(templateMask),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', use_only_once); + for iTomo = 1:nTomograms currentGlobalAngle = 1; + currentSearchPosition = 0; - fprintf('Working on tomo chu %d/%d\n',iTomo,nTomograms); - tempImg = gpuArray(templateBIN); %%%%% NEW switch to bin - - % Avoid repeated allocations - tempPAD = zeros(size(tempImg) + padBIN(1,:) + padBIN(2,:),'single','gpuArray'); - - template_interpolator = ''; - [template_interpolator, ~] = interpolator(tempImg,[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); - - templateMask_interpolator = ''; - [templateMask_interpolator, ~] = interpolator(gpuArray(templateMask),[0,0,0],[0,0,0], 'Bah', 'forward', 'C1', false); + fprintf('Working on tomo chunk %d/%d from %s\n', iTomo, nTomograms, mapName); % Iterate over the tomogram pulling each chunk one at a time. @@ -548,7 +551,9 @@ end for iAzimuth = phi_search - + % currentSearchPosition = currentSearchPosition + 1; + % fprintf('Working search position %d/%d for tomo chunk %d/%d from %s\n',currentSearchPosition, nAngles/length(inPlaneSearch), iTomo, nTomograms, mapName); + if (emc.use_new_grid_search) phi = iAzimuth; else @@ -574,6 +579,8 @@ [0,0,0],rotConvention,... 'forward','C1'); + + tempPAD = tempPAD - mean(tempPAD(:)); @@ -803,7 +810,13 @@ end this_try = 0; -while n <= 2.*peakThreshold && (this_try < max_tries) && MAX > highThr +if (ignore_threshold) + search_limit = peakThreshold +else + search_limit = 2 .* peakThreshold +end + +while n <= search_limit && (this_try < max_tries) && MAX > highThr this_try = this_try + 1; % @@ -821,7 +834,7 @@ try c = gather([i,j,k]); catch - print('Ran into some trouble gathering the i,j,k. Breaking out\n'); + fprint('Ran into some trouble gathering the i,j,k. Breaking out\n'); break end diff --git a/coordinates/BH_defineMatrix.m b/coordinates/BH_defineMatrix.m index 1213e694..fe03fdca 100755 --- a/coordinates/BH_defineMatrix.m +++ b/coordinates/BH_defineMatrix.m @@ -83,8 +83,10 @@ %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% if strcmpi(DIRECTION, 'forward') || strcmpi(DIRECTION, 'fwd') || strcmpi(DIRECTION, 'invVector') + helical_pre = false; angles = -1.*angles; elseif strcmpi(DIRECTION, 'inverse') || strcmpi(DIRECTION, 'inv') || strcmpi(DIRECTION, 'fwdVector') + helical_pre = true; % For interpolation the vectors are applied to a grid, so the sense must % be inverted to make the final transformation active. @@ -148,19 +150,30 @@ 0,0,1] ; case 'Helical' - - - % ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Ry(angles(1)); - - ROTATION_MATRIX = [cosA(3),-sinA(3),0;... - sinA(3),cosA(3),0;... - 0,0,1] * ... - [1,0,0; ... - 0,cosA(2),-sinA(2);... - 0,sinA(2),cosA(2)] * ... - [cosA(1),0,sinA(1); ... - 0,1,0;... - -sinA(1),0,cosA(1)]; + + % ROTATION_MATRIX = Ry(angles(3)) * Rz(angles(2)) * Ry(angles(1)) ; + + ROTATION_MATRIX = [cosA(3),0,sinA(3); ... + 0,1,0;... + -sinA(3),0,cosA(3)] * ... + [cosA(2),-sinA(2),0;... + sinA(2),cosA(2),0;... + 0,0,1] * ... + [cosA(1),0,sinA(1); ... + 0,1,0;... + -sinA(1),0,cosA(1)]; + + % % ROTATION_MATRIX = Rz(angles(3)) * Rx(angles(2)) * Ry(angles(1)); + + % ROTATION_MATRIX = [cosA(3),-sinA(3),0;... + % sinA(3),cosA(3),0;... + % 0,0,1] * ... + % [1,0,0; ... + % 0,cosA(2),-sinA(2);... + % 0,sinA(2),cosA(2)] * ... + % [cosA(1),0,sinA(1); ... + % 0,1,0;... + % -sinA(1),0,cosA(1)]; case 'IMOD' diff --git a/coordinates/eulerSearch.m b/coordinates/eulerSearch.m index 5cd30aa4..42887588 100644 --- a/coordinates/eulerSearch.m +++ b/coordinates/eulerSearch.m @@ -8,6 +8,7 @@ number_of_search_dimensions = 0; number_of_search_positions = 0; number_of_out_of_plane_angles = 1; % poorly named. Theta of zero is still searched but not "outofplane" + active_theta_positions = []; number_of_angles_at_each_theta = []; best_parameters_to_keep = 0; list_of_search_parameters = {}; @@ -26,6 +27,7 @@ max_search_x = 0.0; max_search_y = 0.0; bipolar_search = false; + initialized = false; parameter_map = struct( ... 'phi', [] , ... 'theta', [], ... @@ -64,11 +66,11 @@ end - + function [] = CalculateGridSearchPositions(obj) - + obj.initialized = true; theta_max_local = obj.theta_max; obj.parameter_map.psi = -obj.psi_max./2 : obj.psi_step : obj.psi_max/2; obj.number_of_search_positions = 0; @@ -82,6 +84,7 @@ end obj.parameter_map.theta = theta_search; obj.number_of_out_of_plane_angles = length(theta_search); + obj.active_theta_positions = 1:length(theta_search); obj.number_of_angles_at_each_theta = zeros(obj.number_of_out_of_plane_angles,1); obj.parameter_map.phi = cell(obj.number_of_out_of_plane_angles,1); @@ -123,6 +126,22 @@ end + function [] = HelicalRestriction(obj, max_deviation_from_xy_plane) + if ~obj.initialized + error('Must call Init before calling HelicalRestrictions'); + end + if (max_deviation_from_xy_plane ~= 0.0) + included_angles = abs(obj.parameter_map.theta - 90) <= max_deviation_from_xy_plane; + if (sum(included_angles) == 0) + error('No angles are within the helical restriction'); + end + obj.active_theta_positions = find(included_angles); + obj.number_of_out_of_plane_angles = length(obj.active_theta_positions); + obj.number_of_search_positions = sum(obj.number_of_angles_at_each_theta(obj.active_theta_positions)); + + end + end + function [] = SetSymmetryLimits(obj) switch obj.symmetry_symbol(1) diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index 36958ba0..a0c09845 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -9,8 +9,6 @@ % emClarity ctf 3d paramN.m 'templateMatching' = use this to make recs % for higher res template matching (in the works) -% Read in 2dCtf stacks to trouble shoot -PosControl2d=0; emc = BH_parseParameterFile(PARAMETER_FILE); % Apply a Wiener filter with this many zeros during Ctf multiplication @@ -66,7 +64,6 @@ filterProjectionsForTomoCPRBackground=0; flgWhitenPS = [0,0,0.0]; use_existing_tmpCache=''; -recon_for_tomoCPR = false; recon_for_templateMatching = false; recon_for_subTomo = false; @@ -80,6 +77,12 @@ end else if strcmpi(varargin{1},'split') + % Default to zero for normal use + recon_for_subTomo = true; + if isempty(bh_global_turn_on_phase_plate) + bh_global_turn_on_phase_plate = 0; + end + else error('Extra argument to ctf 3d either templateSearch/split and optionally a vector [iProjcess, nProcesses (from 1)]'); end end @@ -105,8 +108,8 @@ fprintf('recon_subset is [%d,%d]\n',recon_subset(1),recon_subset(2)); -if (recon_for_tomoCPR + recon_for_templateMatching + recon_for_subTomo ~= 1) - error('Only one of the three modes can be used at a time'); +if ( recon_for_templateMatching + recon_for_subTomo ~= 1) + error('Only one of the two modes can be used at a time'); end try @@ -137,12 +140,6 @@ useSurfaceFit = emc.('useSurfaceFit') -try - % Not for normal use, pass the total dose less first frame to flip values. - invertDose = emc.('invertDose') -catch - invertDose = 0; -end fprintf('tiltweight is %f %f\n',tiltWeight); @@ -150,7 +147,7 @@ [tmpCache, flgCleanCache, CWD] = EMC_setup_tmp_cache(emc.fastScratchDisk, use_existing_tmpCache, 'ctf3d', false); -if (recon_for_tomoCPR || recon_for_templateMatching) +if ( recon_for_templateMatching) useSurfaceFit = false; end @@ -177,18 +174,6 @@ end -try - flgDampenAliasedFrequencies = emc.('flgDampenAliasedFrequencies'); -catch - flgDampenAliasedFrequencies = 0; -end - -try - flg2dCTF = emc.('flg2dCTF'); -catch - flg2dCTF = 0; -end - try % Part of the experiment with template matching using higher res info, also % allow for a median filter post CTF correction, pre reconstruction to @@ -201,28 +186,24 @@ %%%%% Take these from param file later. -if (recon_for_tomoCPR) - samplingRate = reconstructionParameters(2); +if (recon_for_subTomo) + samplingRate = emc.('Ali_samplingRate'); + % This number is used to roughly balance the trade off between + % achievable resolution, and run time during reconstruction as + % determined by the thickness of each 3d slab reconstructed. Given that + % we expect the resolution to improve beyond our current value, we + % multiply by 1/2, which gives a (only loosely optimized) resTarget. + resTarget = mean(subTomoMeta.('currentResForDefocusError')*0.5); + if (emc.whitenPS(1)) + emc.whitenPS(2) = resTarget; + end else - if (recon_for_subTomo) - samplingRate = emc.('Ali_samplingRate'); - % This number is used to roughly balance the trade off between - % achievable resolution, and run time during reconstruction as - % determined by the thickness of each 3d slab reconstructed. Given that - % we expect the resolution to improve beyond our current value, we - % multiply by 1/2, which gives a (only loosely optimized) resTarget. - resTarget = mean(subTomoMeta.('currentResForDefocusError')*0.5); - if (emc.whitenPS(1)) - emc.whitenPS(2) = resTarget; - end - else - % For template search - samplingRate = emc.('Tmp_samplingRate'); - try - resTarget = emc.('lowResCut'); - catch - resTarget = 12; - end + % For template search + samplingRate = emc.('Tmp_samplingRate'); + try + resTarget = emc.('lowResCut'); + catch + resTarget = 12; end end @@ -253,32 +234,21 @@ if (recon_for_subTomo) [tiltList, nTilts] = BH_returnIncludedTilts(subTomoMeta.mapBackGeometry); else - if (recon_for_tomoCPR) - tiltList{1} = varargin{2}; - nTilts = 1; - if ~isfield(subTomoMeta.mapBackGeometry.(tiltList{1}).(tomoList)) - error('Did not find any tomograms for tilt-series %s',tiltList{1}); - end - tomoList = subTomoMeta.mapBackGeometry.(tiltList{1}).(tomoList); - else - % TODO set up a check on the recon folder to get what is needed for - % templateSearch - getCoords = dir('recon/*.coords'); - nTilts = length(getCoords); - if (nTilts == 0) - error('Did not find any tomogram coordinates in recon/TS*.coords'); - end - tiltList = cell(nTilts,1); - tiltRecGeom = cell(nTilts,1); - tiltTomoList = cell(nTilts,1); - for iStack = 1:nTilts - % Since we are calling this for templateSearch nTomosPossible == nTomos - % After template matching, there may be inactive tomos, but we'll have the same amount - [ tiltRecGeom{iStack}, tiltName, tiltTomoList{iStack}, tilt_geometry] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); - tiltList{iStack} = tiltName; - end - - + % TODO set up a check on the recon folder to get what is needed for + % templateSearch + getCoords = dir('recon/*.coords'); + nTilts = length(getCoords); + if (nTilts == 0) + error('Did not find any tomogram coordinates in recon/TS*.coords'); + end + tiltList = cell(nTilts,1); + tiltRecGeom = cell(nTilts,1); + tiltTomoList = cell(nTilts,1); + for iStack = 1:nTilts + % Since we are calling this for templateSearch nTomosPossible == nTomos + % After template matching, there may be inactive tomos, but we'll have the same amount + [ tiltRecGeom{iStack}, tiltName, tiltTomoList{iStack}, tilt_geometry] = BH_multi_recGeom( sprintf('recon/%s',getCoords(iStack).name), mapBackIter); + tiltList{iStack} = tiltName; end end @@ -366,7 +336,6 @@ mapBackIter,... 1,... samplingRate,... - PosControl2d,... tiltWeight,... flgMedianFilter); @@ -411,22 +380,10 @@ iCoords{iCoordIdx} = tiltRecGeom{iTilt}{iCoordIdx}; end else - if (recon_for_tomoCPR) - % Get a copy of the tomoCoords (all tomodata will be deleted and only the tilt info kept for tomoCPR) - % place in a cell for consistency - - iCoords{1} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{1}); - iCoords{1}.is_active = 1; - iCoords{1}.dX_specimen_to_tomo = 0; - iCoords{1}.dY_specimen_to_tomo = 0; - iCoords{1}.dZ_specimen_to_tomo = 0; - else - for iCoordIdx = 1:nTomos - % each element of this cell is a struct tomoCoords - iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iCoordIdx}); - end + for iCoordIdx = 1:nTomos + % each element of this cell is a struct tomoCoords + iCoords{iCoordIdx} = subTomoMeta.mapBackGeometry.tomoCoords.(tomoList{iCoordIdx}); end - end @@ -448,56 +405,33 @@ samplingRate, cycleNumber,... 0,1); else - if (recon_for_tomoCPR) - NX = size(maskedStack,1); - NY = size(maskedStack,2); - - % NY = size(maskedStack,2)-1; - NZ = floor(reconstructionParameters(1)); - specimen_NZ_nm = NZ * emc.pixel_size_angstroms / 10; - - if (nTomos ~= 1) - error('For tomoCPR, only one tomo can be reconstructed at a time'); - end - for iCoordIdx = 1:nTomos - iCoords{iCoordIdx}.tomoCoords.NX = NX * samplingRate; - iCoords{iCoordIdx}.tomoCoords.NY = NY * samplingRate; - iCoords{iCoordIdx}.tomoCoords.NZ = NZ * samplingRate; - end - else - [ ~, specimen_NZ_nm, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... - tomoList,nTomos, emc.pixel_size_angstroms, ... - samplingRate, cycleNumber,... - 0,1); - end + [ ~, specimen_NZ_nm, ~ ] = calcAvgZ('dummy',iCoords,tiltList{iTilt}, ... + tomoList,nTomos, emc.pixel_size_angstroms, ... + samplingRate, cycleNumber,... + 0,1); end - if ( flg2dCTF || recon_for_tomoCPR) - n_slabs_to_reconstruct = 1; - ctf3dDepth = specimen_NZ_nm * 10 ^ -9; - else - % TODO: for very thick specimen, this may be preventing the avg from getting to high enough - % resolution to be useful. So far, this is only optimized on in vitro samples. - dampeningMax = 0.90; - - [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... - TLT(1,17),TLT(1,18),abs(TLT(1,15)), ... - 2048, TLT(1,19), ... - resTarget,specimen_NZ_nm*10, ... - dampeningMax,CYCLE); - fprintf('\n\nCalculated a ctfDepth of %2.2f nm for %s\n\n',ctf3dDepth*10^9,tiltList{iTilt}); - if (ctf3dDepth > emc.max_ctf3dDepth) - ctf3dDepth = emc.max_ctf3dDepth; - fprintf('Calculated ctfDepth exceeds user specified max, so actually using a max_ctfDepth of %2.2f nm\n',ctf3dDepth*10^9); - end - % sections centered at 0, which for now is also supposed to coincide with - % the mean defocus determination, although this could be corrected using - % knowledge of particle positions given assurance that particles are the - % primary source of signal (and not carbon for example). - n_slabs_to_reconstruct = ceil(specimen_NZ_nm/(ctf3dDepth*10^9)); - % max odd number - n_slabs_to_reconstruct = n_slabs_to_reconstruct + ~mod(n_slabs_to_reconstruct,2); + % TODO: for very thick specimen, this may be preventing the avg from getting to high enough + % resolution to be useful. So far, this is only optimized on in vitro samples. + dampeningMax = 0.90; + + [ ctf3dDepth ] = BH_ctfCalcError( samplingRate*mean(TLT(:,16)), ... + TLT(1,17),TLT(1,18),abs(TLT(1,15)), ... + 2048, TLT(1,19), ... + resTarget,specimen_NZ_nm*10, ... + dampeningMax,CYCLE); + fprintf('\n\nCalculated a ctfDepth of %2.2f nm for %s\n\n',ctf3dDepth*10^9,tiltList{iTilt}); + if (ctf3dDepth > emc.max_ctf3dDepth) + ctf3dDepth = emc.max_ctf3dDepth; + fprintf('Calculated ctfDepth exceeds user specified max, so actually using a max_ctfDepth of %2.2f nm\n',ctf3dDepth*10^9); end + % sections centered at 0, which for now is also supposed to coincide with + % the mean defocus determination, although this could be corrected using + % knowledge of particle positions given assurance that particles are the + % primary source of signal (and not carbon for example). + n_slabs_to_reconstruct = ceil(specimen_NZ_nm/(ctf3dDepth*10^9)); + % max odd number + n_slabs_to_reconstruct = n_slabs_to_reconstruct + ~mod(n_slabs_to_reconstruct,2); fprintf('with %3.3f nm sections, correcting %d tilt-series\n', ctf3dDepth*10^9, n_slabs_to_reconstruct); % For each tomo create a list of slices that are to be reconstructed @@ -538,27 +472,23 @@ end - if (PosControl2d) - correctedStack = maskedStack; - else % I would have thought the global would be recognized, but it looks % like there is something odd about its use with a parfor loop % FIXME, when setting up the iterator, make clean copies for each % worker that are local in scope.e - [ correctedStack ] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... - avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... - specimen_NZ_nm*10/emc.pixel_size_angstroms,flgDampenAliasedFrequencies,... - preCombDefocus,samplingRate,... - applyExposureFilter,surfaceFit,... - useSurfaceFit,invertDose,... - bh_global_turn_on_phase_plate,... - filterProjectionsForTomoCPRBackground,... - emc.whitenPS, ... - flip_defocus_offset, ... - flip_tilt_offset); - end + [ correctedStack ] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... + avgZ,TLT,emc.pixel_size_angstroms,maskedStack,... + specimen_NZ_nm*10/emc.pixel_size_angstroms,... + preCombDefocus,samplingRate,... + applyExposureFilter,surfaceFit,... + useSurfaceFit,... + bh_global_turn_on_phase_plate,... + filterProjectionsForTomoCPRBackground,... + emc.whitenPS, ... + flip_defocus_offset, ... + flip_tilt_offset); % Write out the stack to the cache directory as a tmp file if (flgEraseBeads_aferCTF) @@ -583,10 +513,7 @@ reconName = sprintf('%s/%s_ali%d_%d_%d.rec', tmpCache, tiltList{iTilt}, mapBackIter+1, this_tomo_idx, iSection); - if (recon_for_tomoCPR) - TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{1}),1); - TA = TA(:,4); - end + if (recon_for_subTomo) TA = sortrows(subTomoMeta.tiltGeometry.(tomoList{iTomo}),1); @@ -728,8 +655,6 @@ % be checked first. if (bh_global_turn_on_phase_plate(1)) reconNameFull = sprintf('cache/%s_bin%d_filtered.rec', tomoList{iTomo}, samplingRate); - elseif recon_for_tomoCPR - reconNameFull = sprintf('%scache/%s_bin%d_backgroundEst.rec', CWD, tomoList{iTomo}, samplingRate); else reconNameFull = sprintf('cache/%s_bin%d.rec', tomoList{iTomo},samplingRate); end @@ -810,19 +735,14 @@ mapBackIter, ... usableArea,... samplingRate,... - PosControl2d,... tiltWeight,... flgMedianFilter) -if (PosControl2d) - prefix = 'ctf'; - suffix = '_ctf'; -else - prefix = 'ali'; - suffix = ''; -end + +prefix = 'ali'; +suffix = ''; % TODO: this could all be in loadOrBin fullStack = sprintf('%sStacks/%s_ali%d%s.fixed', prefix,STACK_PRFX, mapBackIter+1, suffix); @@ -963,10 +883,10 @@ function [correctedStack] = ctfMultiply_tilt(n_slabs_to_reconstruct,iSection,ctf3dDepth, ... avgZ,TLT,pixel_size_angstroms,maskedStack,... - specimen_NZ_nm,flgDampenAliasedFrequencies,... + specimen_NZ_nm,... preCombDefocus,samplingRate,... applyExposureFilter,surfaceFit,... - useSurfaceFit,invertDose, ... + useSurfaceFit, ... phakePhasePlate, ... filterProjectionsForTomoCPRBackground,... flgWhitenPS, ... @@ -1015,34 +935,16 @@ end -if ( flgDampenAliasedFrequencies ) - % Experiment with dampning higher frequencies where aliasing is going to - % result in nonsens. - flgDampenAlias = 1; - fprintf('\n\nExperimental dampening of aliased CTF terms\n'); -else - flgDampenAlias = 0; -end - -apoSize = 6; fastFTSize = BH_multi_iterator([d1,d2],'fourier2d'); - % These should be constant for a given tiltseries Cs = TLT(1,17); WAVELENGTH = TLT(1,18); AMPCONT = TLT(1,19); -if ( flgDampenAlias ) - % Calculate a centered grid b/c real space convolution - [radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... - 'Cylindrical','GPU', ... - {'none'},1,1,0); -else - [radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... - 'Cylindrical','GPU', ... - {'none'},1,0,0); -end +[radialGrid,phi,~,~,~,~] = BH_multi_gridCoordinates(fastFTSize, ... + 'Cylindrical','GPU', ... + {'none'},1,0,0); radialGrid = {radialGrid./(pixel_size_angstroms*10^-10),0,phi}; phi = []; diff --git a/ctf/BH_ctf_Refine2.m b/ctf/BH_ctf_Refine2.m index d9ff14fb..cd973132 100755 --- a/ctf/BH_ctf_Refine2.m +++ b/ctf/BH_ctf_Refine2.m @@ -250,8 +250,8 @@ SAVE_IMG(MRCImage(gather(psTile)),sprintf('fixedStacks/ctf/%s-PS.mrc',fileName),pixelSize); bpLog = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); bpLog = bpLog > 0.99; - bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); - bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); + bp = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],0.0314,max(8,2.*pixelSize),2.*pixelSize,'GPU',pixelSize)); + bp2 = fftshift(BH_bandpass3d([size(psTile(:,:,1)),1],1e-6,40,2.*pixelSize,'GPU',pixelSize)); for iPrj = 1:d3 iTile = gpuArray(psTile(:,:,iPrj)); @@ -265,10 +265,7 @@ SAVE_IMG(MRCImage(gather(psTile_inv)),sprintf('fixedStacks/ctf/%s-PS_inv.mrc',fileName),pixelSize); - bpLog = fftshift(BH_bandpass3d([size(psTile_inv(:,:,1)),1],0,0,2.2.*pixelSize,'GPU',pixelSize)); - bpLog = bpLog > 0.99; - bp = fftshift(BH_bandpass3d([size(psTile_inv(:,:,1)),1],0.25,20,2.*pixelSize,'GPU',pixelSize)); - bp2 = fftshift(BH_bandpass3d([size(psTile_inv(:,:,1)),1],1e-6,400,2.*pixelSize,'GPU',pixelSize)); + for iPrj = 1:d3 iTile = gpuArray(psTile_inv(:,:,iPrj)); diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index fc5ff1c3..45c419b3 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -22,13 +22,13 @@ mapBackIter = 0; end -if isnan(EMC_str2double(STACK_PRFX)) - % It is a name, run here. - nGPUs = 1; - flgParallel = 0; - STACK_LIST = {STACK_PRFX}; - ITER_LIST = {STACK_LIST}; -else +% if isnan(EMC_str2double(STACK_PRFX)) +% % It is a name, run here. +% nGPUs = 1; +% flgParallel = 0; +% STACK_LIST = {STACK_PRFX}; +% ITER_LIST = {STACK_LIST}; +% else flgParallel = 1; updateCMD = sprintf('%s,%d,TiltAlignment,UpdateTilts,[%d,0,0],STD', ... @@ -36,14 +36,16 @@ subTomoMeta.currentCycle); % fprintf('%s/n',updateCMD); nGPUs = emc.('nGPUs'); - ITER_LIST = cell(nGPUs,1); + nWorkers_per_gpu = 2; + nWorkers = nWorkers_per_gpu*nGPUs; + ITER_LIST = cell(nWorkers,1); [STACK_LIST, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); clear STACK_LIST_tmp - for iGPU = 1:nGPUs - ITER_LIST{iGPU} = STACK_LIST(iGPU:nGPUs:nTiltSeries); + for iGPU = 1:nWorkers + ITER_LIST{iGPU} = STACK_LIST(iGPU:nWorkers:nTiltSeries); end -end +% end eucShiftsResults = 0; if emc.eucentric_fit @@ -57,10 +59,10 @@ end try - EMC_parpool(nGPUs); + EMC_parpool(nWorkers); catch delete(gcp('nocreate')); - EMC_parpool(nGPUs); + EMC_parpool(nWorkers); end % Assuming that mapBackIter > 0 since we are updating @@ -72,14 +74,14 @@ % For some reason matlab was geeking out about calling this in the parfor % loop, getting confused about whether it is a variable or a function. -parfor iGPU = 1:nGPUs +parfor iGPU = 1:nWorkers % for iGPU = 1:nGPUs % for iTilt = 1:length(ITER_LIST{iGPU}) if ( flgParallel ) - useGPU = iGPU; - gDev = gpuDevice(useGPU); + useGPU = floor((1+iGPU)/nWorkers_per_gpu); + gpuDevice(useGPU); else useGPU = BH_multi_checkGPU(-1); gDev = gpuDevice(useGPU); @@ -329,7 +331,7 @@ end end else - error('ctf update should only be called after tomoCPR (mapBackIter > 0), you can change this with emClarity geometry paramX.m X SwitchCurrentTomoCPR [mapBackIter,0,0] STD'); + error('ctf update should only be called after tomoCPR (mapBackIter > 0), you can change this with emClarity geometry paramX.m X TiltAlignment SwitchCurrentTomoCpr [mapBackIter,0,0] STD'); end diff --git a/ctf/BH_runCtfFind.m b/ctf/BH_runCtfFind.m index 7fbe5edd..52a6f85a 100644 --- a/ctf/BH_runCtfFind.m +++ b/ctf/BH_runCtfFind.m @@ -64,8 +64,8 @@ ctfParams(1:4), ... d1, ... 30,3*ctfParams(1)./cosd(tiltAngles(tltIDX,4)).^0.4,... - 0.75*meanDefocus,... - 1.25*meanDefocus,... + 0.9*meanDefocus,... + 1.1*meanDefocus,... 25.0); fprintf(fID,'no\nno\nyes\n500.0\nno\nno\nno\neof\n\n'); end diff --git a/logicals/BH_multi_parallelJobs.m b/logicals/BH_multi_parallelJobs.m index 3ca9ceb0..2f72acd2 100644 --- a/logicals/BH_multi_parallelJobs.m +++ b/logicals/BH_multi_parallelJobs.m @@ -92,5 +92,6 @@ end end + end diff --git a/masking/BH_bandpass3d.m b/masking/BH_bandpass3d.m index fa0cc64b..72476380 100755 --- a/masking/BH_bandpass3d.m +++ b/masking/BH_bandpass3d.m @@ -1,5 +1,5 @@ function [ BANDPASS ] = BH_bandpass3d( SIZE, HIGH_THRESH, HIGH_CUT, LOW_CUT, ... - METHOD, PIXEL_SIZE ) + METHOD, PIXEL_SIZE, varargin ) %Create a bandpass filter, to apply to fft of real space 3d images. % % Input variables: @@ -77,26 +77,44 @@ gaussian = @(x,m,s) exp( -1.*(x-m).^2 ./ (2.*s.^2) ); +half_grid = false; +if nargin > 6 + if strcmpi(varargin{1},'halfGrid') + half_grid = true; + end +end - +if (half_grid) + nX = emc_get_origin_index(bSize(1)); +else + nX = bSize(1); +end % initialize window of appropriate size if strcmp(METHOD, 'GPU') - mWindow(bSize(1),bSize(2),bSize(3)) = gpuArray(single(0)); + mWindow(nX,bSize(2),bSize(3)) = gpuArray(single(0)); else - mWindow(bSize(1),bSize(2),bSize(3)) = single(0); + mWindow(nX,bSize(2),bSize(3)) = single(0); end mWindow = mWindow + 1; %%%% This is ~ 120x faster than = gpuArray(ones(bSize, 'single')); % initialize nd grids of appropriate size - +if (half_grid) + [ radius,~,~,~,~,~] = BH_multi_gridCoordinates( bSize, 'Cartesian', METHOD, ... + {'single',... + [1,0,0;0,1,0;0,0,1],... + [0,0,0]','forward',1,1}, ... + 1, 0, 1, {'halfGrid'} ); +else [ radius,~,~,~,~,~] = BH_multi_gridCoordinates( bSize, 'Cartesian', METHOD, ... {'single',... [1,0,0;0,1,0;0,0,1],... [0,0,0]','forward',1,1}, ... 1, 0, 1 ); +end + % Calc lowpass filter mWindow = ... diff --git a/metaData/BH_geometryAnalysis.m b/metaData/BH_geometryAnalysis.m index 0f50347c..9dd0e98b 100755 --- a/metaData/BH_geometryAnalysis.m +++ b/metaData/BH_geometryAnalysis.m @@ -1,8 +1,44 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... STAGEofALIGNMENT, OPERATION, ... VECTOR_OP, HALF_SET) -%Check the alignment at various stages. -% Detailed explanation goes here +% BH_geometryAnalysis +% Purpose +% Perform geometry edits, exports, and bookkeeping across several alignment +% stages (TiltAlignment, RawAlignment, Cluster_cls/Cluster). This includes +% class-based filtering, montage-tiling operations, and utility exports. +% +% Inputs +% PARAMETER_FILE - emClarity parameters (.mat) parsed via BH_parseParameterFile +% CYCLE - cycle index (numeric or string convertible) +% STAGEofALIGNMENT - 'TiltAlignment' | 'RawAlignment' | 'Cluster_cls' +% OPERATION - One of the supported operations (see below) +% VECTOR_OP - Operation-specific vector or path (see below) +% HALF_SET - 'ODD' | 'EVE' | 'STD' (both) +% +% Selected operations (new/extended) +% AssignToBranch +% - VECTOR_OP: rows [class_idx, , x, y] with 1-based montage coords. +% class_idx==1 means ignore; class_idx>1 maps to branch (class_idx-1). +% - Validates each montage tile: must have exactly one positive label; unlabeled +% or conflicting tiles raise errors. Builds per-branch geometry copies in-memory +% and defers saving (writes subTomoMeta_branch_.mat at end of function). +% - No modification to the in-memory masterTM during the case execution. +% +% AssignToTrunk +% - VECTOR_OP: same format as AssignToBranch. +% - Writes TSV files in working directory: cycleNNN_trunk_.txt +% each with columns: tName (tomogram), subtomoIDX (col 4), mappedClass=b. +% - Validates labels and tiles in the same way; no metadata mutation. +% +% AssignAndMerge +% - Consumes all mapping files cycleNNN_trunk_*.txt, determines current branch +% from subTomoMeta name suffix "_branch_", and sets class column to that +% branch for matching subtomoIDX; others are set to -9999. Only affects the +% current in-memory geometry and subsequent save of subTomoMeta. +% +% Notes +% - Column conventions used here: class column=26, subtomo index column=4. +% - Many operations are stage-specific; errors are thrown if misused. @@ -10,8 +46,11 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... error('args = (PARAMETER_FILE, CYCLE, STAGEofALIGNMENT, OPERATION, REMOVE_CLASS, HALF_SET)') end +remove_cycles = false; +assignBranchTriggered = false; % set true when OPERATION == 'AssignToBranch' +assignBranch_nBranches = 0; % number of branches = max(classIDX)-1 +assignBranch_geometry = {}; % cell array of geometries per branch index CYCLE = EMC_str2double(CYCLE); -cycleNumber = sprintf('cycle%0.3u', CYCLE); undoOP = 0; listTomos = 0; if strcmpi(VECTOR_OP, 'undo') @@ -28,7 +67,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... % FIXME if any path this will not work. if ~(strcmp(VECTOR_OP,'tomoList.txt')) [~,modNAME,~] = fileparts(VECTOR_OP); - system(sprintf('model2point %s %s.txt > /dev/null',VECTOR_OP,modNAME)); + system(sprintf('model2point -ObjectAndContour %s %s.txt > /dev/null',VECTOR_OP,modNAME)); VECTOR_OP = importdata(sprintf('%s.txt',modNAME)); end @@ -36,22 +75,20 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... catch % or a 3-vector containing shifts to apply to all volumes, or range for % randomization of 3 euler angles - - VECTOR_OP = EMC_str2double(VECTOR_OP); - if size(VECTOR_OP) == [1,3] - shiftXYZ = VECTOR_OP' - elseif size(VECTOR_OP) == [3,1] - shiftXYZ = VECTOR_OP - elseif ~any(size(VECTOR_OP)) - % size of EMC_str2double('undo') = [0 0] - - else - error('shift values must be a three vector or string "undo"\n'); - end + VECTOR_OP = EMC_str2double(VECTOR_OP); + if isequal(size(VECTOR_OP), [1,3]) + shiftXYZ = VECTOR_OP'; + elseif isequal(size(VECTOR_OP), [3,1]) + shiftXYZ = VECTOR_OP; + elseif ~any(size(VECTOR_OP)) + % size of EMC_str2double('undo') = [0 0] + else + error('shift values must be a three vector or string "undo"\n'); + end end end -startTime = datetime("now"); +% start time intentionally omitted (unused) cycleNumber = sprintf('cycle%0.3u', CYCLE); @@ -85,6 +122,10 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... end +% Column indices (replace magic numbers) +COL_SUBTOMO_IDX = 4; +COL_CLASS = 26; + switch STAGEofALIGNMENT case 'TiltAlignment' @@ -100,7 +141,8 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... end -samplingRate = emc.(sprintf('%s_samplingRate','Ali')); +% sampling rate not used in this function scope +% samplingRate = emc.(sprintf('%s_samplingRate','Ali')); className = emc.(sprintf('%s_className',fieldPrefix)); @@ -126,30 +168,23 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... end case 'Cluster' - try - classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); - catch - classVector{1} = emc.(sprintf('%s_classes',fieldPrefix)); - end - - classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); try - classCoeffs{1} = emc.('Pca_coeffs_odd'); - classCoeffs{2} = emc.('Pca_coeffs_eve'); + class_coeffs{1} = emc.('Pca_coeffs_odd'); + class_coeffs{2} = emc.('Pca_coeffs_eve'); catch - classCoeffs{1} = emc.('Pca_coeffs'); + class_coeffs{1} = emc.('Pca_coeffs'); end - cN = sprintf('%s_%d_%d_nClass_%d_%s',outputPrefix,classCoeffs{halfNUM(1)}(1,1), ... - classCoeffs{halfNUM(1)}(1,end), className, halfSet) - imgNAME = sprintf('class_%d_Locations_%s_%s_NoWgt', className, fieldPrefix, halfSet); - imgNAMEStd = sprintf('class_%d_Locations_%s_%s_NoWgt', className, fieldPrefix, 'ODD'); + cluster_key = sprintf('%s_%d_%d_nClass_%d_%s',outputPrefix,class_coeffs{halfNUM(1)}(1,1), ... + class_coeffs{halfNUM(1)}(1,end), className, halfSet) + img_name = sprintf('class_%d_Locations_%s_%s_NoWgt', className, fieldPrefix, halfSet); + img_name_std = sprintf('class_%d_Locations_%s_%s_NoWgt', className, fieldPrefix, 'ODD'); if (undoOP) - subTomoMeta.(cycleNumber).ClusterResults.(cN) = ... - subTomoMeta.(cycleNumber).(sprintf('Pre_%s_ClusterResults', OPERATION)).(cN); + subTomoMeta.(cycleNumber).ClusterResults.(cluster_key) = ... + subTomoMeta.(cycleNumber).(sprintf('Pre_%s_ClusterResults', OPERATION)).(cluster_key); save(emc.('subTomoMeta'), 'subTomoMeta'); error('No Error, just exiting.\n') else @@ -162,12 +197,14 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... clusterGeom = 'ClusterRefGeom'; end - %geometry = subTomoMeta.(cycleNumber).ClusterResults.(cN); - subTomoMeta.(cycleNumber).(sprintf('Pre_%s_ClusterResults', OPERATION)).(cN) = geometry; + %geometry = subTomoMeta.(cycleNumber).ClusterResults.(cluster_key); + subTomoMeta.(cycleNumber).(sprintf('Pre_%s_ClusterResults', OPERATION)).(cluster_key) = geometry; try - locations= subTomoMeta.(cycleNumber).(imgNAME){2}; + locations= subTomoMeta.(cycleNumber).(img_name){2}; + classVector{1} = subTomoMeta.(cycleNumber).(img_name){3}(1,:); catch - locations= subTomoMeta.(cycleNumber).(imgNAMEStd){2}; + locations= subTomoMeta.(cycleNumber).(img_name_std){2}; + classVector{1} = subTomoMeta.(cycleNumber).(img_name_std){3}(1,:); end end otherwise @@ -197,23 +234,23 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... % While transitioning , edit the dose column in the tilt geometry. for iTomo = 1:nTomograms - TLT = geometry.(tomoList{iTomo}); - nPrjs = size(TLT,1); + tilt_table = geometry.(tomoList{iTomo}); + n_projs = size(tilt_table,1); % Assuming a bi-directional tilt scheme with smallest abs value as first tilt, % which could be wrong - dosePerTilt = [TLT(:,1), TLT(:,4), zeros(nPrjs,1)]; + dose_per_tilt = [tilt_table(:,1), tilt_table(:,4), zeros(n_projs,1)]; % Make sure arranged from negative to positive - dosePerTilt = sortrows(dosePerTilt,2); - [~,firstTilt] = min(abs(dosePerTilt(:,2))); - dosePerTilt(1:firstTilt,:) = sortrows(dosePerTilt(1:firstTilt,:),-2); - exposure = VECTOR_OP(1)./nPrjs - for iExposure = 1:nPrjs - dosePerTilt(iExposure,3) = iExposure .* exposure; + dose_per_tilt = sortrows(dose_per_tilt,2); + [~,first_tilt] = min(abs(dose_per_tilt(:,2))); + dose_per_tilt(1:first_tilt,:) = sortrows(dose_per_tilt(1:first_tilt,:),-2); + exposure = VECTOR_OP(1)./n_projs + for iExposure = 1:n_projs + dose_per_tilt(iExposure,3) = iExposure .* exposure; end - dosePerTilt - for iPrj = 1:nPrjs - CUMeDOSE = dosePerTilt(find(dosePerTilt(:,1) == TLT(iPrj,1)),3); - geometry.(tomoList{iTomo})(iPrj,11) = CUMeDOSE + dose_per_tilt + for iPrj = 1:n_projs + cum_dose = dose_per_tilt(find(dose_per_tilt(:,1) == tilt_table(iPrj,1)),3); + geometry.(tomoList{iTomo})(iPrj,11) = cum_dose end end case 'UpdateTilts' @@ -230,16 +267,16 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... system('rm cache/*.rec*'); end - [STACK_LIST, nTiltSeries] = BH_returnIncludedTilts( masterTM.mapBackGeometry ); + [stack_list, ~] = BH_returnIncludedTilts( masterTM.mapBackGeometry ); - for iStack = 1:length(STACK_LIST) - STACK_PRFX = STACK_LIST{iStack}; - tomo_names = masterTM.mapBackGeometry.(STACK_PRFX).tomoList; + for iStack = 1:length(stack_list) + stack_prfx = stack_list{iStack}; + tomo_names = masterTM.mapBackGeometry.(stack_prfx).tomoList; for iTomo = 1:length(tomo_names) - newTLT = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', STACK_PRFX,mapBackIter+1); - geometry.(tomo_names{iTomo}) = load(newTLT); - fprintf('Updating TLT %s\n', newTLT); + new_tlt = sprintf('fixedStacks/ctf/%s_ali%d_ctf.tlt', stack_prfx,mapBackIter+1); + geometry.(tomo_names{iTomo}) = load(new_tlt); + fprintf('Updating TLT %s\n', new_tlt); end end @@ -248,59 +285,56 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... case 'WriteCsv' !mkdir -p csv for iTomo = 1:nTomograms - positionList = geometry.(tomoList{iTomo}); - csvOUT = sprintf('./csv/%s_%s_%s.csv',cycleNumber,tomoList{iTomo},fieldPrefix); - csvID = fopen(csvOUT, 'w'); - for iSubTomo = 1:size(positionList,1) - if ( VECTOR_OP(1) == -1 && positionList(iSubTomo,26) ~= -9999 ) || ... - ( ismember(positionList(iSubTomo,26), VECTOR_OP) ) - fprintf(csvID,'%-06.3f %-06.3f %-06.3f %-04d %-06.3f %-06.3f %-04d %-04d %-06.3f %-06.3f %-07.3f %-07.3f %-07.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-4d\n',... - positionList(iSubTomo,:)); + position_list = geometry.(tomoList{iTomo}); + csv_out = sprintf('./csv/%s_%s_%s.csv',cycleNumber,tomoList{iTomo},fieldPrefix); + csv_fid = fopen(csv_out, 'w'); + for i_subtomo = 1:size(position_list,1) + if ( VECTOR_OP(1) == -1 && position_list(i_subtomo,26) ~= -9999 ) || ... + ( ismember(position_list(i_subtomo,26), VECTOR_OP) ) + fprintf(csv_fid,'%-06.3f %-06.3f %-06.3f %-04d %-06.3f %-06.3f %-04d %-04d %-06.3f %-06.3f %-07.3f %-07.3f %-07.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-03.3f %-4d\n',... + position_list(i_subtomo,:)); end end - fclose(csvID); + fclose(csv_fid); end case 'RemoveClasses' - classesToKeep = []; - classesToKill = []; + classes_to_keep = []; % make a 2d array same size as montage clear blankClassMont % montage is always square, and the first row is always full, so use this to % determine the size. - % if it is raw alignment, assume we are still deleting classes - hacked in to test, note cycle number is bogus - % if strcmpi(STAGEofALIGNMENT, 'RawAlignment') - % imgNAME = sprintf('class_%d_Locations_%s_%s_NoWgt', className, 'Cls', halfSet); - % imgNAMEStd = sprintf('class_%d_Locations_%s_%s_NoWgt', className, 'Cls', 'ODD'); - % try - % locations= masterTM.('cycle005').(imgNAME){2}; - % catch - % locations= masterTM.('cycle005').(imgNAMEStd){2}; - % end - % end - sizeMontage = max(locations{end}(2),locations{end}(4)) - blankClassMont(sizeMontage, sizeMontage) = single(0); + size_montage = max(locations{end}(2),locations{end}(4)); + blankClassMont(size_montage, size_montage) = single(0); % put a 1 at each x,y from the model file identifying classes to kill - VECTOR_OP = round(VECTOR_OP) - for iKill = 1:size(VECTOR_OP,1) - blankClassMont(VECTOR_OP(iKill,1),VECTOR_OP(iKill,2)) = 1; + % columns are obj cont x y z + VECTOR_OP = round(VECTOR_OP); + for i_kill = 1:size(VECTOR_OP,1) + blankClassMont(VECTOR_OP(i_kill,3),VECTOR_OP(i_kill,4)) = 1; end + % FIXME: I did not run into this until 2025, so for now just working around. If not all of the classes are included in the class average, then + % This could skip classes that are to be removed. For now, I'm just going to assume anything not averaged should be removed. + % In this case, the class was not averaged b/c there was 2 in the odd set and 0 in the eve, which created a NAN in averaging (which is also a bug, it should be caught, I must just be checking one halfset?) + classes_not_in_average = ~ismember(1:className,classVector{1}); + classesToKill = find(classes_not_in_average); for iClass = 1:length(locations) - iCut = blankClassMont(locations{iClass}(1):locations{iClass}(2), ... - locations{iClass}(3):locations{iClass}(4)); - - - - if any(iCut(:)) - classesToKill = [classesToKill, iClass]; - else - classesToKeep = [classesToKeep, iClass]; - end - end + iCut = blankClassMont(locations{iClass}(1):locations{iClass}(2), ... + locations{iClass}(3):locations{iClass}(4)); + + + + if any(iCut(:)) + classesToKill = [classesToKill, iClass]; + else + classes_to_keep = [classes_to_keep, iClass]; + end + + end + classesToKill = sort(classesToKill,'ascend'); fileOUT = fopen(sprintf('%s_ClassMods_%s.txt',cycleNumber, halfSet), 'w'); fprintf(fileOUT, '%s\n','Classes removed:'); @@ -308,12 +342,14 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... fprintf(fileOUT, ',%g', classesToKill(2:end)); fprintf(fileOUT, ']\n\n'); - fprintf(fileOUT, '%s\n','Classes retained:'); - fprintf(fileOUT, '[%g',classesToKeep(1)); - fprintf(fileOUT, ',%g', classesToKeep(2:end)); - fprintf(fileOUT, '; 1.*ones(1,%d)]\n\n',numel(classesToKeep)); + fprintf(fileOUT, '%s\n','Classes retained:'); + fprintf(fileOUT, '[%g',classes_to_keep(1)); + fprintf(fileOUT, ',%g', classes_to_keep(2:end)); + fprintf(fileOUT, '; 1.*ones(1,%d)]\n\n',numel(classes_to_keep)); + + + - nTotal = 0; nRemoved = 0; nRemain = 0; @@ -340,6 +376,186 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... fprintf(fileOUT, '\nremoved:\t%d\nremaining:%d\norig:%d\n',nRemoved,nRemain,nTotal); fclose(fileOUT); + case 'AssignToBranch' + % Assign montage tiles to branches using VECTOR_OP labels. + % Contract: + % - Input VECTOR_OP: [class_idx, , x, y] with class 1 == ignore. + % - Each tile must have exactly one positive label inside its bbox; unlabeled or conflicting labels error. + % - Output: no mutation to masterTM here; per-branch geometries cached and saved at end. + % Failure modes: out-of-bounds label, unlabeled tile, conflicting labels. + + if ~(strcmpi(STAGEofALIGNMENT, 'Cluster')) + error('AssignToBranch is only valid at STAGEofALIGNMENT=Cluster'); + end + + % Build tile mapping (and branch count) using shared helper; validates VECTOR_OP + [tile_to_branch, n_branches] = emc_build_tile_to_branch(VECTOR_OP, locations, 'AssignToBranch'); + assignBranch_nBranches = n_branches; + + % Build per-branch geometries without altering 'geometry' + % Build per-branch geometries without altering 'geometry' + branch_geometries = emc_assign_branch_geometries(geometry, tomoList, tile_to_branch, n_branches, COL_CLASS); + + % Mark that we will handle saving at the end + assignBranch_geometry = branch_geometries; + assignBranchTriggered = true; + + % Summary logging: tiles per branch and per-tomo assignment counts + tiles_ignored = sum(tile_to_branch == 0); + fprintf('AssignToBranch: tiles ignored=%d\n', tiles_ignored); + for b = 1:n_branches + fprintf('AssignToBranch: tiles -> branch %d = %d\n', b, sum(tile_to_branch == b)); + end + % Per-tomo particle distribution based on original classes + total_per_branch = zeros(n_branches,1); + total_ignored = 0; + for iTomo = 1:nTomograms + tName = tomoList{iTomo}; + position_list_orig = geometry.(tName); + included_mask = (position_list_orig(:,COL_CLASS) ~= -9999); + orig_classes = position_list_orig(included_mask, COL_CLASS); + ignore_count = sum(ismember(orig_classes, find(tile_to_branch == 0))); + counts_line = zeros(1,n_branches); + for b = 1:n_branches + cnt_b = sum(ismember(orig_classes, find(tile_to_branch == b))); + counts_line(b) = cnt_b; total_per_branch(b) = total_per_branch(b) + cnt_b; + end + total_ignored = total_ignored + ignore_count; + fprintf('AssignToBranch: %s -> branch_counts=%s, ignored=%d\n', tName, mat2str(counts_line), ignore_count); + end + fprintf('AssignToBranch: totals per branch=%s, total ignored=%d\n', mat2str(total_per_branch'), total_ignored); + + case 'AssignToTrunk' + % Prepare mapping files for future AssignAndMerge without modifying metadata. + % Contract: + % - Input VECTOR_OP: [class_idx, , x, y] with class 1 == ignore. + % - Output files: cycleNNN_trunk_.txt (TSV: tName, subtomoIDX, mappedClass=b). + % Failure modes: out-of-bounds label, unlabeled/conflicting tiles. + + if ~(strcmpi(STAGEofALIGNMENT, 'Cluster')) + error('AssignToTrunk is only valid at STAGEofALIGNMENT=Cluster'); + end + + % Build tile mapping (and branch count) using shared helper + [tile_to_branch, n_branches] = emc_build_tile_to_branch(VECTOR_OP, locations, 'AssignToTrunk'); + + % Open one output file per branch in the working directory, prefixed by cycle number + file_ids = cell(n_branches,1); + lines_per_branch = zeros(n_branches,1); + for b = 1:n_branches + out_path = sprintf('%s_trunk_%d.txt', cycleNumber, b); + fid = fopen(out_path, 'w'); + if fid == -1, error('AssignToTrunk: failed to open %s for writing', out_path); end + fprintf(fid, '# tName\tsubtomoIDX\tmappedClass\n'); + file_ids{b} = fid; + end + + % Write assignments: for each tile mapped to branch b, list all rows with origClass==iClass + for iTomo = 1:nTomograms + tomo_name = tomoList{iTomo}; + position_list = geometry.(tomo_name); + original_class_labels = position_list(:,COL_CLASS); + subtomo_idx = position_list(:,COL_SUBTOMO_IDX); + for tile_index = 1:length(tile_to_branch) + b = tile_to_branch(tile_index); + if b <= 0, continue; end + class_mask = (original_class_labels == tile_index); + if any(class_mask) + fid = file_ids{b}; + idx_list = find(class_mask)'; + for k = idx_list + fprintf(fid, '%s\t%d\t%d\n', tomo_name, subtomo_idx(k), b); + end + lines_per_branch(b) = lines_per_branch(b) + numel(idx_list); + end + end + end + + % Close files + for b = 1:n_branches + fclose(file_ids{b}); + fprintf('AssignToTrunk: wrote %s_trunk_%d.txt (rows=%d)\n', cycleNumber, b, lines_per_branch(b)); + end + fprintf('AssignToTrunk: tiles ignored=%d\n', sum(tile_to_branch==0)); + for b = 1:n_branches + fprintf('AssignToTrunk: tiles -> branch %d = %d\n', b, sum(tile_to_branch==b)); + end + + case 'AssignAndMerge' + % Merge assignments: choose current branch based on subTomoMeta name, then + % for all cycle-prefixed mapping files, set class=b for matches and -9999 otherwise. + % Contract: + % - Input files: cycleNNN_trunk_*.txt (TSV header + rows tName,subtomoIDX,mappedClass). + % - Behavior: detect current branch from subTomoMeta suffix _branch_ and apply. + % Failure modes: missing trunk files, invalid branch suffix, file read errors. + if ~(strcmpi(STAGEofALIGNMENT, 'Cluster')) + error('AssignAndMerge is only valid at STAGEofALIGNMENT=Cluster'); + end + meta_name = emc.('subTomoMeta'); + branch_tokens = regexp(meta_name, '_branch_(\d+)$', 'tokens'); + if isempty(branch_tokens) + error('AssignAndMerge: could not determine branch from subTomoMeta "%s"; expected suffix _branch_', meta_name); + end + current_branch = str2double(branch_tokens{1}{1}); + if isnan(current_branch) || current_branch < 1 + error('AssignAndMerge: invalid branch number parsed from subTomoMeta "%s"', meta_name); + end + + % Discover mapping files for this cycle + file_list = dir(sprintf('%s_trunk_*.txt', cycleNumber)); + if isempty(file_list) + error('AssignAndMerge: no mapping files found matching %s_trunk_*.txt', cycleNumber); + end + + % Build a keep list (by tName) for current branch + keep_by_tomo = struct(); + total_rows_current_branch = 0; + for file_i = 1:numel(file_list) + file_path = file_list(file_i).name; + fid = fopen(file_path, 'r'); + if fid == -1, error('AssignAndMerge: failed to open %s', file_path); end + % header then rows: tName\tsubtomoIDX\tmappedClass + header_line = fgetl(fid); %#ok + parsed_cols = textscan(fid, '%s%d%d', 'Delimiter', '\t'); + fclose(fid); + if isempty(parsed_cols{1}), continue; end + tomo_names = parsed_cols{1}; subtomo_ids = parsed_cols{2}; mapped_classes = parsed_cols{3}; + % Filter to current branch + is_current_branch = (mapped_classes == current_branch); + tomo_names = tomo_names(is_current_branch); subtomo_ids = subtomo_ids(is_current_branch); + total_rows_current_branch = total_rows_current_branch + numel(subtomo_ids); + for id_i = 1:numel(subtomo_ids) + t_name = tomo_names{id_i}; + if ~isfield(keep_by_tomo, t_name) + keep_by_tomo.(t_name) = subtomo_ids(id_i); + else + keep_by_tomo.(t_name) = unique([keep_by_tomo.(t_name); subtomo_ids(id_i)]); + end + end + end + + % Apply to geometry: set -9999 by default; set class=current_branch for matches + total_kept = 0; total_ignored = 0; + for iTomo = 1:nTomograms + tName = tomoList{iTomo}; + position_list = geometry.(tName); + included_mask = (position_list(:,COL_CLASS) ~= -9999); + % default ignore for all included + position_list(included_mask,COL_CLASS) = -9999; + if isfield(keep_by_tomo, tName) + ids_to_keep = keep_by_tomo.(tName); + % match by column 4 (subtomoIDX) + keep_mask = ismember(position_list(:,COL_SUBTOMO_IDX), ids_to_keep); + position_list(keep_mask,COL_CLASS) = current_branch; + end + kept_here = sum(position_list(:,COL_CLASS) == current_branch); + ignored_here = sum(included_mask) - kept_here; + total_kept = total_kept + kept_here; total_ignored = total_ignored + ignored_here; + fprintf('AssignAndMerge: %s -> kept=%d, ignored=%d\n', tName, kept_here, ignored_here); + geometry.(tName) = position_list; + end + fprintf('AssignAndMerge: files_read=%d, rows_for_branch=%d, total_kept=%d, total_ignored=%d\n', numel(file_list), total_rows_current_branch, total_kept, total_ignored); + case 'ShiftAll' % No option to shift eve/odd separately. for iTomo = 1:nTomograms @@ -366,25 +582,26 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... end case 'ListTomos' - system(sprintf('mkdir -p tomoList_%s',cycleNumber)); - tomoListOUT = fopen(sprintf('tomoList_%s/tomoList.txt',cycleNumber),'w'); + system(sprintf('mkdir -p tomoList_%s',cycleNumber)); + tomo_list_fid = fopen(sprintf('tomoList_%s/tomoList.txt',cycleNumber),'w'); for iTomo = 1:nTomograms positionList = geometry.(tomoList{iTomo}); includeList = (positionList(:,26) ~= -9999); - sTremaining = sum(includeList); - sTtotal = length(includeList); - figure('Visible','off'); hist(positionList(includeList,1),floor(sTremaining./5)+1); + n_remaining = sum(includeList); + n_total = length(includeList); + figure('Visible','off'); hist(positionList(includeList,1),floor(n_remaining./5)+1); title({sprintf('%s',tomoList{iTomo})},'Interpreter','none'); xlabel('CCC'); file_out = sprintf('tomoList_%s/%s.pdf', cycleNumber, tomoList{iTomo}); saveas(gcf, file_out,'pdf') - fprintf(tomoListOUT,'%s\t%d/%d\n',tomoList{iTomo},sTremaining,sTtotal); + fprintf(tomo_list_fid,'%s\t%d/%d\n',tomoList{iTomo},n_remaining,n_total); + fclose(tomo_list_fid); end case 'RemoveTomos' if (listTomos) - f=fieldnames(masterTM.mapBackGeometry.tomoName); + f=fieldnames(masterTM.mapBackGeometry.tomoName); tomoFid = fopen('tomoList.txt','w'); fprintf(tomoFid,'%s\n',f{:}); fclose(tomoFid); @@ -393,15 +610,14 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... f=fieldnames(masterTM.mapBackGeometry.tomoName); for iOrig = 1:length(f) - flgRemove = 1; + flg_remove = 1; for iToKeep = 1:length(tomoList) if strcmp(f{iOrig},tomoList{iToKeep}) - tomoList{iToKeep}; - flgRemove = 0; + flg_remove = 0; break; end end - if (flgRemove) + if (flg_remove) if isfield(masterTM.reconGeometry.(f{iOrig})) masterTM.reconGeometry = rmfield(masterTM.reconGeometry,f{iOrig}); end @@ -417,8 +633,8 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... tName = masterTM.mapBackGeometry.tomoName.(f{iOrig}).tiltName; if isfield(masterTM.mapBackGeometry,tName) masterTM.mapBackGeometry.(tName).nTomos = masterTM.mapBackGeometry.(tName).nTomos - 1; - keepRows = ismember(1:size(masterTM.mapBackGeometry.(tName).coords,1),tN); - masterTM.mapBackGeometry.(tName).coords = masterTM.mapBackGeometry.(tName).coords(~keepRows,:); + keep_rows = ismember(1:size(masterTM.mapBackGeometry.(tName).coords,1),tN); + masterTM.mapBackGeometry.(tName).coords = masterTM.mapBackGeometry.(tName).coords(~keep_rows,:); masterTM.mapBackGeometry.tomoName = rmfield(masterTM.mapBackGeometry.tomoName,f{iOrig}); if masterTM.mapBackGeometry.(tName).nTomos == 0 masterTM.mapBackGeometry = rmfield(masterTM.mapBackGeometry,tName); @@ -503,7 +719,11 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... case 'RemoveIgnoredParticles' % Get distribution of CCC from given cycle rawAlignment % Save histogram, also remove given bottom percentage and report CCC cutoff - fprintf('\n\t\nRemoving ignored particles from the meta data to save space, this must be run after alignment or after a call to emClarity skip, post classification\n\n.'); + fprintf('\n\t\nRemoving ignored particles from the meta data to save space, this must be run after alignment or after a call to emClarity skip, post classification\n\n'); + if (length(shiftXYZ) == 3) + remove_cycles = true; + fprintf('Removing any alignment cycles before cycle %d\n',shiftXYZ(3)); + end if ~(strcmpi(STAGEofALIGNMENT, 'RawAlignment')) error('Can only remove fraction at RawAlignment Stage') end @@ -537,9 +757,9 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... for iVol = 1:nVols weightedCurve(iVol,2) = mean(snrVector(1:iVol,1).*sqrt(iVol)); end - length([1:nVols]') + length((1:nVols)') length(weightedCurve(:,2)) - r = fit([1:nVols]',weightedCurve(:,2),'linear'); + r = fit((1:nVols)',weightedCurve(:,2),'linear'); maxVal = find(r(2:nVols+1)-r(1:nVols) <= 0, 1,'first') maxCCC = cccVector(maxVal) @@ -553,7 +773,7 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... fprintf(fID,'%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t\n%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t%-4.3f\t\n',percentiles'); fclose(fID); otherwise - error('OPERATION must be WriteCsv, RemoveClasses, ShiftAll, RemoveFraction, RemoveIgnoredParticles, not %s', OPERATION) + error('OPERATION must be WriteCsv, RemoveClasses, AssignToBranch, AssignToTrunk, AssignAndMerge, ShiftAll, RemoveFraction, RemoveIgnoredParticles, not %s', OPERATION) end % Redundant for WriteCsv, otherwise update the new geometry, which was backed up @@ -574,8 +794,162 @@ function BH_geometryAnalysis( PARAMETER_FILE, CYCLE, ... STAGEofALIGNMENT); end - subTomoMeta = masterTM; - save(emc.('subTomoMeta'), 'subTomoMeta'); +% Currently set only if calling RemoveIgnoredParticles +if (remove_cycles) + for iCycle = shiftXYZ(3):-1:0 + cycleName = sprintf('cycle%0.3u', iCycle); + if isfield(masterTM, cycleName) + masterTM = rmfield(masterTM, cycleName); + end + end +end + +if assignBranchTriggered + % Save per-branch variants without modifying masterTM + for b = 1:assignBranch_nBranches + branchMasterTM = masterTM; + % Only Cluster stage supported for AssignToBranch + if strcmpi(STAGEofALIGNMENT, 'Cluster') + branchMasterTM.(cycleNumber).(clusterGeom) = assignBranch_geometry{b}; + else + error('AssignToBranch saving encountered unexpected stage: %s', STAGEofALIGNMENT); + end + subTomoMeta = branchMasterTM; %#ok + outBase = sprintf('%s_branch_%d', emc.('subTomoMeta'), b); + save(outBase, 'subTomoMeta'); + fprintf('AssignToBranch: wrote %s.mat\n', outBase); + end +else + subTomoMeta = masterTM; + save(emc.('subTomoMeta'), 'subTomoMeta'); +end +end + + + + + + +% ---- Local helpers (behavior-preserving) --------------------------------- +function [tile_to_branch, n_branches] = emc_build_tile_to_branch(VECTOR_OP, locations, whoami) +% emc_build_tile_to_branch +% Build per-tile branch mapping from montage label points. +% +% Inputs +% VECTOR_OP - N×4 array; columns [class_idx, , x, y], 1-based. +% class_idx==1 => ignore; class_idx>1 => branch (class_idx-1). +% locations - cell array of montage tile bounding boxes: [x1 x2 y1 y2]. +% whoami - string used for clearer error messages (optional). +% +% Outputs +% tile_to_branch - vector of length num_tiles; 0 for ignore, >0 for branch idx. +% n_branches - number of branches implied by maximum class_idx minus one. +% +% Errors +% - Out-of-bounds or invalid class_idx rows. +% - Any tile without a positive label. +% - Any tile that contains multiple distinct labels. + if nargin < 3, whoami = 'Assign'; end + VECTOR_OP = round(VECTOR_OP); + if size(VECTOR_OP,2) < 4 + error('%s: expects VECTOR_OP with 4 columns: [classIDX, , x, y]', whoami); + end + + montage_size = max(locations{end}(2),locations{end}(4)); + label_montage = zeros(montage_size, montage_size, 'single'); + + for label_idx = 1:size(VECTOR_OP,1) + class_idx = VECTOR_OP(label_idx,1); + x_idx = VECTOR_OP(label_idx,3); y_idx = VECTOR_OP(label_idx,4); + if class_idx < 1 + error('%s: classIDX must be >= 1 at row %d (got %d)', whoami, label_idx, class_idx); + end + if x_idx < 1 || y_idx < 1 || x_idx > montage_size || y_idx > montage_size + error('%s: coordinate out of bounds at row %d: (%d,%d) not in [1,%d]', whoami, label_idx, x_idx, y_idx, montage_size); + end + label_montage(x_idx,y_idx) = class_idx; %#ok + end + + max_class_idx = max(VECTOR_OP(:,1)); + n_branches = max_class_idx - 1; + if n_branches < 1 + error('%s: need at least one classIDX > 1 to form branches (max classIDX=%d)', whoami, max_class_idx); + end + + n_tiles = length(locations); + row_starts = zeros(n_tiles,1); col_starts = zeros(n_tiles,1); + for ii = 1:n_tiles + row_starts(ii) = locations{ii}(1); + col_starts(ii) = locations{ii}(3); + end + row_vals_top = sort(unique(row_starts),'ascend'); + n_rows = numel(row_vals_top); + tile_to_branch = zeros(n_tiles,1,'int32'); + for tile_index = 1:n_tiles + x1 = locations{tile_index}(1); x2 = locations{tile_index}(2); + y1 = locations{tile_index}(3); y2 = locations{tile_index}(4); + region = label_montage(x1:x2, y1:y2); + vals = unique(region(:)); vals = vals(vals > 0); + row_from_top = find(row_vals_top == x1, 1, 'first'); if isempty(row_from_top), row_from_top = 1; end + tiles_in_row = find(row_starts == x1); [~, col_order] = sort(col_starts(tiles_in_row), 'ascend'); + col_idx = find(tiles_in_row(col_order) == tile_index, 1, 'first'); if isempty(col_idx), col_idx = 1; end + row_from_bottom = n_rows - row_from_top + 1; %#ok % retained for error context + if isempty(vals) + error('%s: tile (row=%d, col=%d) has no label; every tile must have at least one label [bbox %d:%d,%d:%d]', ... + whoami, row_from_bottom, col_idx, x1, x2, y1, y2); + end + unique_labels = unique(vals); + if numel(unique_labels) > 1 + error('%s: tile (row=%d, col=%d) has conflicting labels %s [bbox %d:%d,%d:%d]', ... + whoami, row_from_bottom, col_idx, mat2str(unique_labels'), x1, x2, y1, y2); + end + if unique_labels == 1 + tile_to_branch(tile_index) = 0; + else + tile_to_branch(tile_index) = int32(unique_labels - 1); + end + end +end + +function branch_geometries = emc_assign_branch_geometries(geometry, tomo_list, tile_to_branch, n_branches, COL_CLASS) +% emc_assign_branch_geometries +% Construct per-branch copies of geometry with reassigned class labels. +% +% Inputs +% geometry - struct: fields per tomogram name, each an array (rows=subtomos). +% tomo_list - cellstr of tomogram names (fieldnames of geometry). +% tile_to_branch - vector mapping class (tile index) -> branch index (0 ignore). +% n_branches - number of branches. +% COL_CLASS - column index of class label (usually 26). +% +% Output +% branch_geometries - cell of structs, one per branch, same schema as geometry. + n_tiles = length(tile_to_branch); + branch_geometries = cell(n_branches,1); + for b = 1:n_branches + branch_geometries{b} = geometry; + end + for iTomo = 1:length(tomo_list) + tName = tomo_list{iTomo}; + position_list_orig = geometry.(tName); + original_class_labels = position_list_orig(:,COL_CLASS); + included_mask = (position_list_orig(:,COL_CLASS) ~= -9999); + for b = 1:n_branches + pl = position_list_orig; + pl(included_mask,COL_CLASS) = -9999; + branch_geometries{b}.(tName) = pl; + end + for tile_index = 1:n_tiles + b = tile_to_branch(tile_index); + if b <= 0, continue; end + class_mask = (original_class_labels == tile_index); + if any(class_mask) + branch_pl = branch_geometries{b}.(tName); + branch_pl(class_mask,COL_CLASS) = b; + branch_geometries{b}.(tName) = branch_pl; + end + end + end end diff --git a/metaData/BH_parseParameterFile.m b/metaData/BH_parseParameterFile.m index 5d4c247f..9386f47a 100755 --- a/metaData/BH_parseParameterFile.m +++ b/metaData/BH_parseParameterFile.m @@ -24,7 +24,8 @@ stringValues = {'subTomoMeta'; ... 'Ali_mType';'Cls_mType';'Cls_mType';'Raw_mType';'Fsc_mType'; ... 'Pca_distMeasure';'Kms_mType';'flgPrecision';'Tmp_xcfScale';... - 'fastScratchDisk';'Tmp_eraseMaskType';'startingDirection';'Peak_mType';'symmetry'}; + 'fastScratchDisk';'Tmp_eraseMaskType';'startingDirection';'Peak_mType';'symmetry'; ... + 'gmm_covariance_type';'distance_metric'}; for i = 1:size(p2,1) pNameVal = strsplit(p2{i,1},'='); if length(pNameVal) == 1 @@ -34,7 +35,7 @@ fprintf("Last successfully parsed parameter: %s\n", string(last_parsed_parameter)); error('To many colons in\n\t %s',char(pNameVal)) else - if any(strcmp(stringValues, pNameVal{1})) + if any(strcmpi(stringValues, pNameVal{1})) emc.(pNameVal{1}) = pNameVal{2}; else emc.(pNameVal{1}) = EMC_str2double(pNameVal{2}); @@ -207,12 +208,13 @@ end -% Check and override the rotational convention to get helical averaging. -% Replaces the former hack of adding a fifth dummy value to the angular search -if isfield(emc,'doHelical') - EMC_assert_boolean(emc.doHelical) +% Helix or filament with axis on Z, this will restrain the search angles to be +/- this many degrees from the X/y plane. +% A full search should still be specified +% Only affects templateSearch +if isfield(emc,'helical_search_theta_constraint') + EMC_assert_numeric(emc.helical_search_theta_constraint, 1) else - emc.doHelical = false; + emc.helical_search_theta_constraint = 0; end if isfield(emc,'eucentric_fit') @@ -365,14 +367,18 @@ end % Allowed values are validated inside BH_clusterPub.m -if ~isfield(emc, 'Pca_distMeasure'); +emc = EMC_assert_deprecated_substitution(emc, 'distance_metric', 'Pca_distMeasure'); +if isfield(emc, 'distance_metric') + EMC_assert_string_value(emc.distance_metric, {'sqeuclidean', 'cosine', 'gaussian','ward','neural'}, false); +else emc.distance_metric = 'sqeuclidean'; end -if isfield(emc, 'Pca_nReplicates'); - EMC_assert_numeric(emc.Pca_nReplicates, 1, [100, 1000]); +emc = EMC_assert_deprecated_substitution(emc, 'n_replicates', 'Pca_nReplicates'); +if isfield(emc, 'n_replicates') + EMC_assert_numeric(emc.n_replicates, 1, [1, 1000]); else - emc.n_replicates = 256; + emc.n_replicates = 64; end if isfield(emc, 'Pca_refineKmeans') @@ -410,6 +416,26 @@ emc.Pca_som_topologyFcn = 'hextop'; end + +% specifying shared diagonal will approximate K-means +if isfield(emc, 'gmm_covariance_type') + EMC_assert_string_value(emc.gmm_covariance_type, {'full', 'diagonal'},false); +else + emc.gmm_covariance_type = 'full'; +end + +if isfield(emc, 'gmm_covariance_shared_between_clusters') + EMC_assert_boolean(emc.gmm_covariance_shared_between_clusters); +else + emc.gmm_covariance_shared_between_clusters = false; +end + +if isfield(emc, 'gmm_regularize_value') + EMC_assert_numeric(emc.gmm_regularize_value, 1, [0.0, 1.0]); +else + emc.gmm_regularize_value = 0.01; +end + if isfield(emc, 'spike_prior') EMC_assert_boolean(emc.spike_prior); else @@ -508,6 +534,12 @@ emc.tomoCPR_random_subset = -1; end +if isfield(emc, 'tomoCPR_n_particles_minimum') + EMC_assert_numeric(emc.tomoCPR_n_particles_minimum, 1, [1, 100000]); +else + emc.tomoCPR_n_particles_minimum = 10; +end + if isfield(emc, 'tomoCPR_target_n_patches_x_y') EMC_assert_numeric(emc.tomoCPR_target_n_patches_x_y, 2, [0, 100]); else @@ -627,7 +659,7 @@ emc = EMC_assert_deprecated_substitution(emc, 'tomo_cpr_defocus_range', 'tomoCprDefocusRange'); if isfield(emc, 'tomo_cpr_defocus_range') - EMC_assert_numeric(emc.tomo_cpr_defocus_range, 1, [0.0, 10000e-9]); + EMC_assert_numeric(emc.tomo_cpr_defocus_range, 1, [0.0, 100000e-9]); else emc.tomo_cpr_defocus_range = 500e-9; end @@ -639,7 +671,7 @@ emc.tomo_cpr_defocus_step = 100e-9; end -emc = EMC_assert_deprecated_substitution(emc, 'tomo_cpr_defocus_refine', 'calcCTF'); +emc = EMC_assert_deprecated_substitution(emc, 'tomo_cpr_defocus_refine', 'tomoCprDefocusRefine'); if isfield(emc, 'tomo_cpr_defocus_refine') EMC_assert_boolean(emc.tomo_cpr_defocus_refine); else diff --git a/metaData/BH_rawAlignmentsApply.m b/metaData/BH_rawAlignmentsApply.m index 9c442754..85083ec0 100755 --- a/metaData/BH_rawAlignmentsApply.m +++ b/metaData/BH_rawAlignmentsApply.m @@ -1,7 +1,7 @@ function [UPDATED_GEOMETRY] = ... BH_rawAlignmentsApply( INPUT_GEOMETRY , ... - BEST_ANGLES, SAMPLING, nPeaks, rotConvention, updateWeights, updateClassByBestReferenceScore) + BEST_ANGLES, SAMPLING, nPeaks, rotConvention, updateClassByBestReferenceScore) %Apply class alignments to the full set of subTomograms % % @@ -97,11 +97,7 @@ positionList(iParticle,1+26*(iPeak-1)) = gather(newAlignment(pIndex,[6] + 10*(iPeak-1))); - % I don't think I'm using this column anywhere else but double check - % classification maybe. FIXME. - if (updateWeights) - positionList(iParticle,2+26*(iPeak-1)) = gather(newAlignment(pIndex,[7] + 10*(iPeak-1))); - end + if (updateClassByBestReferenceScore) positionList(iParticle,26+26*(iPeak-1)) = gather(newAlignment(pIndex,[1] + 10*(iPeak-1))); diff --git a/metaData/EMC_get_class_idx.m b/metaData/EMC_get_class_idx.m new file mode 100644 index 00000000..456f3f98 --- /dev/null +++ b/metaData/EMC_get_class_idx.m @@ -0,0 +1,12 @@ +function [class_idx, class_probability] = EMC_get_class_idx(val) + + EMC_assert_numeric(val, 1) + if (val == -9999) + class_idx = -9999; + class_probability = 0.0; + else + class_idx = floor(val); + class_probability = max(0, min(1, 1-(val - floor(val)))); + end + +end \ No newline at end of file diff --git a/metaData/EMC_parpool.m b/metaData/EMC_parpool.m index 3d91d026..3aea389d 100644 --- a/metaData/EMC_parpool.m +++ b/metaData/EMC_parpool.m @@ -34,6 +34,7 @@ saveAsProfile(emc_parcluster, emc_rand_name); end + [ pool ] = parpool(emc_rand_name, nWorkers); end diff --git a/metaData/emc_check_for_valid_image_file.m b/metaData/emc_check_for_valid_image_file.m index 65ac3c35..bfa88f2b 100644 --- a/metaData/emc_check_for_valid_image_file.m +++ b/metaData/emc_check_for_valid_image_file.m @@ -12,9 +12,10 @@ catch is_valid = false; end - % So all we know is that the header was readable, now check that there are at least the right number of bytes in the file - is_valid = checkFullFile(test_header); - + if is_valid + % So all we know is that the header was readable, now check that there are at least the right number of bytes in the file + is_valid = checkFullFile(test_header); + end else is_valid = false; end diff --git a/statistics/BH_clusterPub.m b/statistics/BH_clusterPub.m index 837c8b7d..014468d2 100755 --- a/statistics/BH_clusterPub.m +++ b/statistics/BH_clusterPub.m @@ -75,32 +75,36 @@ % kAlgorithm = 'neuralNetwork' switch emc.distance_metric + case 'gaussian' + kDistMeasure = 'gaussian'; + kAlgorithm = 'gmm'; case 'sqeuclidean' - kDistMeasure = 'sqeuclidean' + kDistMeasure = 'sqeuclidean'; case 'cityblock' - kDistMeasure = 'cityblock' + kDistMeasure = 'cityblock'; case 'cosine' - kDistMeasure = 'cosine' + kDistMeasure = 'cosine'; case 'correlation' - kDistMeasure = 'correlation' + kDistMeasure = 'correlation'; case 'ward' - kDistMeasure = 'ward' - kAlgorithm = 'HAC' + kDistMeasure = 'ward'; + kAlgorithm = 'HAC'; case 'neural' - kDistMeasure = 'neural' - kAlgorithm = 'neuralNetwork' + kDistMeasure = 'neural'; + kAlgorithm = 'neuralNetwork'; fprintf('Input params for neural network are %d %d %s\n', ... emc.coverSteps, emc.Pca_som_initNeighbor, emc.topologyFcn); otherwise - kDistMeasure = 'sqeuclidean' + kDistMeasure = 'sqeuclidean'; fprintf(['\nDefaulting to sqeuclidean b/c %s was not recognized'] ... , kDist); end - + fprintf('\n\tClustering using %s, with distance metric %s\n\n', kAlgorithm, kDistMeasure); + try oldPca = load(coeffMatrix); - coeffsUNTRIMMED = oldPca.coeffs + coeffsUNTRIMMED = oldPca.coeffs; idxList = oldPca.idxList; if emc.nPeaks > 1 peakList = oldPca.idxList; @@ -110,7 +114,7 @@ clear oldPca; catch - error('trouble loading the previous pcs mat file.') + error('trouble loading the previous pca mat file.') end try @@ -178,14 +182,14 @@ if strcmpi(kAlgorithm, 'kMeans') [class, classCenters, sumd, D] = kmeans(coeffMat', nClusters, ... - 'replicates', emc.n_replicates, ... + 'Replicates', emc.n_replicates, ... 'Distance', kDistMeasure, ... 'MaxIter', 50000, ... % Default was 100 'Options', statset('UseParallel', 1) ); elseif strcmpi(kAlgorithm, 'kMedoids') [class, classCenters, sumd, D] = kmedoids(coeffMat', nClusters, ... - 'replicates', emc.n_replicates, ... + 'Replicates', emc.n_replicates, ... 'Distance', kDistMeasure, ... 'Options', statset('UseParallel', 1, ... 'MaxIter', 50000) ); @@ -204,9 +208,24 @@ [net, tr] = train(net, coeffMat); y = net(coeffMat) class = vec2ind(y) - + elseif strcmpi(kAlgorithm, 'gmm') + % This is a Gaussian Mixture Model + + GMM = fitgmdist(coeffMat', nClusters, ... + 'Regularize', emc.gmm_regularize_value, ... + 'Replicates', emc.n_replicates, ... + 'CovarianceType', emc.gmm_covariance_type, ... + 'SharedCovariance', emc.gmm_covariance_shared_between_clusters, ... % covariance can vary between clusters + 'Start', 'plus', ... + 'Options', statset('UseParallel', 1, 'MaxIter', 10000) ); + posterior_prob = posterior(GMM, coeffMat'); + % Still hard cutoff, but keep prob to weight the volume + [class_probability, class] = max(posterior_prob, [], 2); + % class = cluster(GMM, coeffMat'); + classCenters = GMM.mu; + sumd = 0; else - error('kAlgorithm must be kMeans, or kMedoids, not %s', kAlgorithm); + error('kAlgorithm must be kMeans, kMedoids, HAC, neuralNetwork, or gmm not %s', kAlgorithm); end @@ -217,7 +236,7 @@ fprintf('Total kmeans dist = %g\n', totSum1) fprintf('Total kmeans std = %g\n', totStd1) - if (emc.Pca_refineKmeans) + if (emc.Pca_refineKmeans && (strcmpi(kAlgorithm, 'kMeans') || strcmpi(kAlgorithm, 'kMedoids'))) % Using the postions found, refine the original estimates kMin = min(classCenters,[],1); @@ -253,14 +272,14 @@ % 'Options', statset('UseParallel', 1) ); if strcmpi(kAlgorithm, 'kMeans') [class, classCenters, sumd,D] = kmeans(coeffMat', nClusters, ... - 'replicates', emc.n_replicates, ... + 'Replicates', emc.n_replicates, ... 'Distance', kDistMeasure, ... 'MaxIter', 50000, ... % Default was 100 'Options', statset('UseParallel', 1) ); elseif strcmpi(kAlgorithm, 'kMedoids') [class, classCenters, sumd,D] = kmedoids(coeffMat', nClusters, ... - 'replicates', emc.n_replicates, ... + 'Replicates', emc.n_replicates, ... 'Distance', kDistMeasure, ... 'Options', statset('UseParallel', 1, ... 'MaxIter', 50000) ); @@ -293,8 +312,20 @@ % list of classIDX with highest count first [~, ndx] = sort(classCount, 'descend'); newClass = class; + originalClass = class; + if strcmpi(kAlgorithm, 'gmm') + sorted_class_probabilities = class_probability; + end + for i = 1:nClusters newClass(class == ndx(i)) = i; + assignment_mask = find(class == ndx(i)); + % Re-assign the most populated class to 1, next most populated to 2, etc. + newClass(assignment_mask) = i; + if strcmpi(kAlgorithm, 'gmm') + % Re-map the probabilities to the new class labels + sorted_class_probabilities(assignment_mask) = class_probability(assignment_mask); + end end fileOUT = fopen(sprintf('%s_%s_ClassIDX.txt',emc.('subTomoMeta'),cycleNumber), 'a'); @@ -315,9 +346,13 @@ % This isn't great, and maybe my brain is just tired. if ( strcmpi(kAlgorithm, 'kMedoids') || strcmpi(kAlgorithm, 'kMeans') ) - save(sprintf('clusterTrouble_%d.mat',iCluster), 'idxList', 'class','classCenters','D'); + + save(sprintf('clusterTrouble_%d.mat',nClusters), 'idxList', 'class', 'originalClass', 'sumd', 'classCenters','D','-v7.3'); + elseif strcmpi(kAlgorithm, 'gmm') + data_vector = coeffMat'; + save(sprintf('clusterTrouble_%d.mat',nClusters), 'idxList', 'class', 'originalClass', 'sumd', 'classCenters','GMM','data_vector', 'sorted_class_probabilities','-v7.3'); else - save(sprintf('clusterTrouble_%d.mat',iCluster), 'idxList', 'class'); + save(sprintf('clusterTrouble_%d.mat',nClusters), 'idxList', 'originalClass', 'class','-v7.3'); end for iTomo = 1:nTomograms @@ -337,6 +372,9 @@ for thisIDX = 1:length(lIndClass) for iPeak = 0:emc.nPeaks-1 positionList(lIndPart(thisIDX), 26 + 26*iPeak) = class(lIndClass(thisIDX)+iPeak); + if strcmpi(kAlgorithm, 'gmm') + positionList(lIndPart(thisIDX), 2 + 26*iPeak + 1) = sorted_class_probabilities(lIndClass(thisIDX)+iPeak); + end % fprintf('iTomo %d iSubtomo %d iPeak %d Class %d\n',iTomo,lIndPart(thisIDX),iPeak+1,class(lIndClass(thisIDX)+iPeak)); end end @@ -344,6 +382,9 @@ fprintf('Size iTomo %d %d\n',size(positionList)); else positionList(lIndPart, 26) = class(lIndClass); % this is where it is breaking + if strcmpi(kAlgorithm, 'gmm') + positionList(lIndPart, 2) = sorted_class_probabilities(lIndClass); + end geometry.(tomoList{iTomo}) = positionList; end @@ -359,6 +400,11 @@ % Save a copy of the geometry in the subTomoMeta and also save the name for % easy reference in a text file. masterTM.(cycleNumber).('ClusterResults').(fout) = geometry; + if strcmpi(kAlgorithm, 'gmm') + masterTM.(cycleNumber).('ClusterResults').('class_weights') = true; + else + masterTM.(cycleNumber).('ClusterResults').('class_weights') = false; + end diff --git a/statistics/BH_fscGold_class.m b/statistics/BH_fscGold_class.m index dda0c541..51178907 100755 --- a/statistics/BH_fscGold_class.m +++ b/statistics/BH_fscGold_class.m @@ -432,6 +432,7 @@ halfSet = 'STD'; end + imgFilt1 = BH_bandLimitCenterNormalize(img1.*volMask{1}.*shapeMask_1,bandpassFilt{iGold}, ... (volMask{1} > 0.01),padCalc, 'single'); imgFilt1 = real(ifftn(imgFilt1)); diff --git a/statistics/BH_fscSplit.m b/statistics/BH_fscSplit.m index 3c6905c1..0360ce16 100755 --- a/statistics/BH_fscSplit.m +++ b/statistics/BH_fscSplit.m @@ -51,6 +51,17 @@ tomoList = fieldnames(geometry); nTomograms = length(tomoList); +% Sort tomos by number of peaks so that if we split by tomo, we get a +% roughly balanced split. +nPeaksList = zeros(nTomograms,1); +for iTomo = 1:nTomograms + nPeaksList(iTomo) = size(geometry.(tomoList{iTomo}),1); +end + +[~,sortIDX] = sort(nPeaksList); +tomoList = tomoList(sortIDX); + + floorCeil = 0; for iTomo = 1:nTomograms diff --git a/statistics/BH_multi_cRef_Vnorm.m b/statistics/BH_multi_cRef_Vnorm.m index 235ca3c2..08809d2b 100755 --- a/statistics/BH_multi_cRef_Vnorm.m +++ b/statistics/BH_multi_cRef_Vnorm.m @@ -1,31 +1,27 @@ function [weightedImgs] = BH_multi_cRef_Vnorm( ... - fscParams, aliParams, mskParams,... - imgs, weights, ... - flgCombine, flgReference, ... - pixelSize, bFactor,varargin) + fscParams, aliParams, mskParams,... + imgs, weights, weights_sqrt, ... + flgCombine, flgReference, ... + pixelSize, bFactor,varargin) % Don't force to zero - particularly for comparison with ground truth, should % only be used with a very small (i.e. non-zero but no real amplification) % value. -if any(bFactor < 0) - bFactor = abs(bFactor) - noForceMask = 1 -else - noForceMask = 0 -end -if nargin == 10 + +if nargin == 11 gpuDevice(varargin{1}) end -if nargin == 11 +if nargin == 12 highPassFilter = varargin{2}; else highPassFilter = [0,0]; end -[ padVal ] = BH_multi_padVal(size(imgs{1}), size(weights{1})); +[ padVal ] = BH_multi_padVal(size(imgs{1}), size(weights{1})) + @@ -56,12 +52,9 @@ imgs{iWgt} = imgs{iWgt} - mean(imgs{iWgt}(:)); imgs{iWgt} = gpuArray(imgs{iWgt} ./rms(imgs{iWgt}(:))); - [weights{iWgt}, ~] = BH_multi_cRef_wgtCritical(gpuArray(weights{iWgt})); - -end - - + % [weights{iWgt}, ~] = BH_multi_cRef_wgtCritical(gpuArray(weights{iWgt})); +end % The mask used in the FSC calc @@ -96,16 +89,17 @@ + radialGrid = BH_multi_gridCoordinates(size(weights{1}),'Cartesian','GPU', ... {'none'},1,0,1); radialGrid = radialGrid ./ pixelSize; -[ anisoFSC, avgCTF ] = calc_anisoFSC(fscParams, radialGrid,weights, bFactor, pixelSize); +[ anisoFSC, ~ ] = calc_anisoFSC(fscParams, radialGrid,weights, bFactor, pixelSize); if any(bFactor) - [ bFactor, bandFilter ] = calc_bfact(fscParams, radialGrid,flgReference, bFactor, noForceMask,highPassFilter,pixelSize); + [ bFactor, bandFilter ] = calc_bfact(fscParams, radialGrid,flgReference, bFactor, 0,highPassFilter,pixelSize); else bFactor = {1}; end @@ -127,7 +121,7 @@ snrWeight = 1; [ weightedImgs{iBfact} ] = gather(apply_weights((anisoFSC), ... - (avgCTF{1}+avgCTF{2})./2, ... + (weights_sqrt{1}+weights_sqrt{2}).^0.5, ... (radialGrid < 0.5./pixelSize),... (imgs{1}+imgs{2}), ... ifftshift(weights{1}+weights{2}),... @@ -137,7 +131,7 @@ snrWeight = 0.5; for iGold = 1:2 [ weightedImgs{iGold} ] = gather(apply_weights((anisoFSC), ... - (avgCTF{iGold}), ... + (weights_sqrt{iGold}.^0.5), ... (radialGrid < 0.5./pixelSize),... (imgs{iGold}), ... ifftshift(weights{iGold}),... diff --git a/statistics/BH_multi_cRef_wgtCritical.m b/statistics/BH_multi_cRef_wgtCritical.m index fdf56e2e..206b7536 100644 --- a/statistics/BH_multi_cRef_wgtCritical.m +++ b/statistics/BH_multi_cRef_wgtCritical.m @@ -4,14 +4,16 @@ % Detailed explanation goes here startingMax = max(sf3d(:)); -valAtZero = max(10,0.1*startingMax); % ~ value at zero sampling (a bit less after the subtraction to keep the +% ~ value at zero sampling (a bit less after the subtraction to keep the +valAtZero = max(10,log(startingMax)); % value at minNumSampled unchanged with a smooth transition. wienerThreshold = (1.5.*(median(sf3d(sf3d(:)>valAtZero))-valAtZero)); +% value below where a penalty is add (very little until low numbers) -minNumSampled = 0.2.*median(sf3d(sf3d(:)>10)); % value below where a penalty is add (very little until low numbers) +minNumSampled = 0.2.*median(sf3d(sf3d(:)>10), 'all'); minFactor = 75/minNumSampled; minWeight = gpuArray(10); % decreasing this increase the downweighting as you move from 0 to minNumSampled @@ -23,6 +25,7 @@ sf3d(m) = (sf3d(m)+1) + valAtZero.^(minWeight.^((minFactor.*sf3d(m)+1).^-1)); sf3d(m) = sf3d(m)-valAtZero.^(minWeight.^(minFactor*minNumSampled+1).^-1)+1; m = sf3d > startingMax; + sf3d(m) = startingMax + log(sf3d(m)); diff --git a/statistics/BH_pcaPub.m b/statistics/BH_pcaPub.m index 839d78d7..3330b73a 100755 --- a/statistics/BH_pcaPub.m +++ b/statistics/BH_pcaPub.m @@ -141,6 +141,7 @@ error('PREVIOUS_PCA should be 1,0,-1,-2,-3') end + flgWMDs = 3; cycleNumber = sprintf('cycle%0.3u', CYCLE); @@ -667,7 +668,7 @@ gpuMasks.('scaleMask').(stSCALE) = gpuArray(masks.('scaleMask').(stSCALE)); end gpuMasks.('highPass').(stSCALE) = gpuArray(masks.('highPass').(stSCALE)); - if (use_notch_filter) + if (use_notch_filter && ~test_multi_ref_diffmap) gpuMasks.('highPass').(stSCALE) = gpuMasks.('highPass').(stSCALE) .* gpuArray(masks.('scaleMask').(stSCALE)); end @@ -721,9 +722,7 @@ tiltGeometry = subTomoMeta.tiltGeometry.(tomoList{iTomo}); fprintf('Working on %d/%d volumes %s\n',iTomo,nTomograms,tomoName); - if (iTomo ~= 3) - continue; %revertH68_1_label_101_2H68_1_label_101_2 - end + % Load in the geometry for the tomogram, and get number of subTomos. positionList = geometry.(tomoList{iTomo}); diff --git a/synthetic/BH_synthetic_mapBack.m b/synthetic/BH_synthetic_mapBack.m index 8714fb50..f7d661fc 100755 --- a/synthetic/BH_synthetic_mapBack.m +++ b/synthetic/BH_synthetic_mapBack.m @@ -1,4 +1,4 @@ -function [ ] = BH_synthetic_mapBack(PARAMETER_FILE, CYCLE, tiltStart) +function [ ] = BH_synthetic_mapBack(PARAMETER_FILE, CYCLE, STAGEofALIGNMENT, tiltStart) % Map back and align using the subtomograms as fiducial markers. @@ -17,7 +17,6 @@ % FIXME: is this even relevant any more? buildTomo=1;% % % % % % % -save_diagnostic_ccf=0; % Default true, we don't need this after projection delete_background_estimate = true; @@ -26,6 +25,8 @@ emc = BH_parseParameterFile(PARAMETER_FILE); + + CYCLE = EMC_str2double(CYCLE); cycle_numerator = ''; cycle_denominator =''; @@ -140,7 +141,7 @@ calcCTF = emc.('tomo_cpr_defocus_refine'); -[tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ) +[tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); @@ -161,12 +162,29 @@ % Cycle 0 is named differently - I'll be deleting this in an overhaul of the way % the subTomoMeta is written. if (CYCLE) - try - fprintf('Using Alignment geometry %s\n',cycleNumber); - geometry = subTomoMeta.(cycleNumber).RawAlign; - catch + if strcmpi(STAGEofALIGNMENT, 'Cluster') + geometry = subTomoMeta.(cycleNumber).('ClusterClsGeom'); + fprintf('Using ClusterClsGeom geometry %s\n',cycleNumber); + % FIXME: THis logic is confusing, the two ClusterCls/REf are the product of the average step ... + elseif strcmpi(STAGEofALIGNMENT, 'RawAlignment') + if ( emc.multi_reference_alignment ) + if (emc.classification) + geometry = subTomoMeta.(cycleNumber).('ClusterClsGeom'); + fprintf('Using ClusterClsGeom geometry %s\n',cycleNumber); + else + geometry = subTomoMeta.(cycleNumber).('ClusterRefGeom'); + fprintf('Using ClusterRefGeom geometry %s\n',cycleNumber); + end + else + geometry = subTomoMeta.(cycleNumber).RawAlign; + fprintf('Using Alignment geometry %s\n',cycleNumber); + end + else + if ~strcmpi(STAGEofALIGNMENT, 'Avg') + error('Unknown stage of alignment %s', STAGEofALIGNMENT); + end + geometry = subTomoMeta.(cycleNumber).('Avg_geometry'); fprintf('Using Average geometry %s\n',cycleNumber); - geometry = subTomoMeta.(cycleNumber).Avg_geometry; end else try @@ -236,19 +254,31 @@ mbOUT = {[tmpCache],[mapBackIter+1],'dummy'}; +continue_an_existing_run = false; +% Check for an existing completed mapBack dir exist, if so move to backup +if exist(sprintf('mapBack%d',mapBackIter+1), 'dir') + [y,m,d] = ymd(datetime); + [h,mi,s] = hms(datetime); + system(sprintf('mv mapBack%d mapBack%d_%d%0.2d%0.2d_%d_%d_%d',mapBackIter+1,mapBackIter+1,y,m,d,h,mi,floor(s))); + clear y m d h mi s +end +% Now check for an existing incomplete mapBack dir, if so, continue +if exist(sprintf('cache/mapBack%d',mapBackIter+1), 'dir') + continue_an_existing_run = true; +else + system(sprintf('mkdir -p cache/mapBack%d',mapBackIter+1)); +end + +system(sprintf('mkdir -p %smapBack%d',tmpCache,mapBackIter+1)); + for iTiltSeries = tiltStart:nTiltSeries skip_this_tilt_series_bc_alignments_are_bonkers = false; if (skip_to_the_end_and_run) continue; end - - - - % mapBackRePrjSize = min(64,subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).('tomoCprRePrjSize')) - mapBackRePrjSize = 512 - % % % iViewGroup = subTomoMeta.mapBackGeometry.viewGroups.(tiltNameList{iTiltSeries}); + mapBackRePrjSize = 512; nTomograms = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).nTomos if nTomograms == 0 % No points were saved after template matching so skip this tilt series @@ -257,6 +287,8 @@ end skip_this_tilt_series_because_it_is_empty = false(nTomograms,1); + + % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); tomoList = subTomoMeta.mapBackGeometry.(tiltNameList{iTiltSeries}).tomoList; @@ -267,6 +299,28 @@ [~,tiltBaseName,~] = fileparts(tilt_filename); mbOUT{3} = tiltBaseName; + + if (continue_an_existing_run) + tilt_already_mapped_back = true; + % A minimum check see if we have the model file + check_mod = sprintf('cache/mapBack%d/%s_fit-full.fid',mbOUT{2:3}); + if ~(exist(check_mod,'file')) + tilt_already_mapped_back = false; + fprintf('Tilt series %s has not been started\n',tiltNameList{iTiltSeries}); + else + % Okay they both exist lets see if they are readable, the same length and > 0 + [ model_is_not_okay ] = system(sprintf('model2point %s %s.tmp >> /dev/null', check_mod, check_mod)); + if (model_is_not_okay) + tilt_already_mapped_back = false; + fprintf('Tilt series %s has not been started\n',tiltNameList{iTiltSeries}); + end + system(sprintf('rm %s.tmp', check_mod)); + end + if (tilt_already_mapped_back) + fprintf('Tilt series %s has already been mapped back, skipping\n',tiltNameList{iTiltSeries}); + continue; + end + end if (mapBackIter) @@ -346,16 +400,7 @@ end end - % Check that an existing mapBack dir doen't exist, if so move to backup - if exist(sprintf('mapBack%d',mapBackIter+1), 'dir') - [y,m,d] = ymd(datetime); - [h,mi,s] = hms(datetime); - system(sprintf('mv mapBack%d mapBack%d_%d%0.2d%0.2d_%d_%d_%d',mapBackIter+1,mapBackIter+1,y,m,d,h,mi,floor(s))); - clear y m d h mi s - end - system(sprintf('mkdir -p %smapBack%d',tmpCache,mapBackIter+1)); - system(sprintf('mkdir -p cache/mapBack%d',mapBackIter+1)); % re-initialize the parpool for each tilt series to free up mem. if ~isempty(gcp('nocreate')) @@ -398,6 +443,8 @@ if (emc.save_mapback_classes && save_mapback_masks) avgColor = zeros(reconstruction_size, 'int16'); end + + TLT = tiltGeometry.(tomoList{1}); @@ -419,6 +466,7 @@ refVol{2}{iRef} = gather(refVol{2}{iRef}); particleMask{iRef} = gather(particleMask{iRef}); end + sprintf('[%d,%d]',maxZ,samplingRate); tiltNameList{iTiltSeries}; @@ -438,19 +486,21 @@ avgColor = zeros(reconstruction_size, 'int16'); end + if (buildTomo) coordOUT = fopen(sprintf('%smapBack%d/%s.coord',mbOUT{1:3}),'w'); coordSTART = fopen(sprintf('%smapBack%d/%s.coord_start',mbOUT{1:3}),'w'); if (emc.save_mapback_classes) coordCLASS = fopen(sprintf('%smapBack%d/%s.coord_class',mbOUT{1:3}),'w'); - coordCLASS_perTomo = cell(nTomograms,1); + coordClass_perTomo = cell(nTomograms,1); for iCoordClassPerTomo = 1:nTomograms - coordClass_perTomo{iCoordClassPerTomo} = fopen(sprintf('%smapBack%d/%s.coord_class',mbOUT{1:2},tomoList{iCoordClassPerTomo}),'w'); + coordClass_perTomo{iCoordClassPerTomo} = fopen(sprintf('%smapBack%d/%s_%s.coord_class',mbOUT{1:3},tomoList{iCoordClassPerTomo}),'w'); end end defOUT = fopen(sprintf('%smapBack%d/%s.defAng',mbOUT{1:3}),'w'); end - + + % Track the number of fiducials in order to scale the K-factor to more or less % aggressivley downweight outliers in the alignment @@ -462,9 +512,16 @@ positionList{iTomo} = positionList{iTomo}(positionList{iTomo}(:,26) ~= -9999,:); n_possible_particles = n_possible_particles + size(positionList{iTomo},1); end + + n_possible_particles + if (emc.tomoCPR_n_particles_minimum > n_possible_particles) + fprintf('There are fewer particles available (%d) than tomoCPR_n_particles_minimum (%d)\n', n_possible_particles, emc.tomoCPR_n_particles_minimum); + skip_this_tilt_series_because_it_is_empty = true(size(skip_this_tilt_series_because_it_is_empty)); + end + % Now check to see if we need to limit the number of fiducials - if (emc.tomoCPR_random_subset == -1 || emc.tomoCPR_random_subset > n_possible_particles) + if (emc.tomoCPR_random_subset == -1 || emc.tomoCPR_random_subset >= n_possible_particles) fprintf('Using all of the %d available fiducials\n',size(positionList{iTomo},1)); else fprintf('Using a random subset of %d fiducials from the %d available\n',... @@ -747,13 +804,16 @@ end % loop over subtomos end %%%% temp condition to skip building full tomo - + fprintf('Finished processing %s\n',tomoList{iTomo}); end % end of loop over tomograms on this tilt-series + fprintf('Finished processing tilt series %s\n',tiltNameList{iTiltSeries}); % No subtomos remain if all( skip_this_tilt_series_because_it_is_empty ) continue; end + fprintf('Finished processing tilt series after continue %s\n',tiltNameList{iTiltSeries}); + buildTomo if (buildTomo) @@ -764,15 +824,18 @@ p2m = sprintf(['point2model -sphere 6 -thick 6 -scat ',... '%smapBack%d/%s.coord_class %smapBack%d/%s_classIdx.3dfid'], ... mbOUT{1:3},mbOUT{1:3}); + fprintf('Running point2model on %s\n',p2m); system(p2m); for iCoordClassPerTomo = 1:nTomograms fclose(coordClass_perTomo{iCoordClassPerTomo}); p2m = sprintf(['point2model -sphere 6 -thick 6 -scat ',... - '%smapBack%d/%s.coord_class %smapBack%d/%s_classIdx.3dfid'], ... - mbOUT{1:2},tomoList{iCoordClassPerTomo},mbOUT{1:2},tomoList{iCoordClassPerTomo}); + '%smapBack%d/%s_%s.coord_class %smapBack%d/%s_%s_classIdx.3dfid'], ... + mbOUT{1:3},tomoList{iCoordClassPerTomo},mbOUT{1:3},tomoList{iCoordClassPerTomo}); + fprintf('Running point2model for iTomo %d on %s\n',iCoordClassPerTomo,p2m); system(p2m); end end + p2m = sprintf(['point2model -zero -circle 3 -color 0,0,255 -values -1 ',... '%smapBack%d/%s.coord %smapBack%d/%s.3dfid'], ... mbOUT{1:3},mbOUT{1:3}); @@ -812,7 +875,6 @@ clear avgTomo{1} wgt end - % % % % It may be faster to work with a rotated vol since the reading in may cause % % % % problems, but the projection is so slow, that this isn't worth dealing with % % % % now. @@ -1073,7 +1135,7 @@ end % re-write the projected coords - system(sprintf(['model2point -float -contour -zero ', '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'], mbOUT{1:3}, mbOUT{1:3})) + system(sprintf(['model2point -contour -zero ', '%smapBack%d/%s.fid %smapBack%d/%s.coordPrj'], mbOUT{1:3}, mbOUT{1:3})) end @@ -1159,17 +1221,9 @@ % Zero and only changed if CTF is refined. defocusShifts = cell(nPrjs,1); - if (calcCTF) - nToCheck = floor(ctfRange./ctfInc); - defShiftVect = ctfInc.*[-nToCheck:nToCheck]'; - else - nToCheck = 1; - defShiftVect = 0; - end - nDefTotal = length(defShiftVect); defocusCCC = cell(nPrjs,1); - expectedDefocusPerFiducial=cell(nPrjs,1); + defocusVAL = cell(nPrjs,1); if samplingRate > 1 @@ -1200,8 +1254,8 @@ % I must specify the number of fiducials somehwere else, replace the % unique when there is time. defocusShifts{iPrj} = 0; - defocusCCC{iPrj} = zeros(nDefTotal, nFidsTotal,'single','gpuArray'); - expectedDefocusPerFiducial{iPrj} = zeros(nDefTotal,nFidsTotal,'single'); + defocusCCC{iPrj} = zeros( nFidsTotal,1,'single','gpuArray'); + defocusVAL{iPrj} = zeros( nFidsTotal,1,'single','gpuArray'); end %Put back into a natural order @@ -1211,10 +1265,10 @@ emc.k_factor_scaling = 10 / sqrt(nFidsTotal); end - + % for iPrj = 1:nPrjs % recert parfor iPrj = 1:nPrjs - + % % For some reason if these mrc objects are created before the parfor % loop begins, they fail to load. It is fine as a regular for loop % though - annoying, but very little overhead. It would be nice @@ -1372,6 +1426,7 @@ mRx mRy COM + save('full_trbl.mat', '-v7'); SAVE_IMG(MRCImage(gather(cccPrj)),'err.mrc'); error('failed to box out the cross-correlation for image\n%s\non Projection %d\n', ... tiltName, iPrj); @@ -1418,9 +1473,13 @@ coordOUT = fopen(sprintf('%smapBack%d/%s_%03d.coordFIT',mbOUT{1:3},iPrj),'w'); - tmpOut = zeros([CTFSIZE,size(wrkFid,1)],'single','gpuArray'); + n_tiles = size(wrkFid,1); dXY = [0,0]; - for iFid = 1:size(wrkFid,1) + initial_shifts = zeros(n_tiles,2); + score_sum = 0.0; + score_sum_sq = 0.0; + n_at_end_of_range = 0; + for iFid = 1:n_tiles if wrkFid(iFid,2) == -9999 fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), [-4,-4], -9999); @@ -1479,11 +1538,6 @@ dataTile = dataTile./rms(dataTile(:)); refTile = refTile ./ rms(refTile(:)); - if (save_diagnostic_ccf) - SAVE_IMG(dataTile, 'dataTile.mrc'); - SAVE_IMG(refTile, 'refTile.mrc'); - end - dataTile = ctfMask.*BH_padZeros3d(dataTile,'fwd',padCTF, ... 'GPU','singleTaper'); @@ -1491,154 +1545,152 @@ refTile = ctfMask.*BH_padZeros3d(refTile,'fwd',padCTF, ... 'GPU','singleTaper'); - if (save_diagnostic_ccf) - SAVE_IMG(dataTile, 'dataTile_pad.mrc'); - SAVE_IMG(refTile, 'refTile_pad.mrc'); - end - + % TODO: make sure this is matching how the calc is done in toCistem df1 = (wrkDefAngTilt(iFid,1) + wrkPar(iFid,5)) * 10; df2 = (wrkDefAngTilt(iFid,1) - wrkPar(iFid,5)) * 10; dfA = wrkPar(iFid,6); - if (calcCTF) - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixel_size),pixel_size]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,40,min(min_res_for_ctf_fitting,sqrt(2).*pixel_size),pixel_size])); - else - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixel_size]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixel_size])); - end - - bestScore = -inf; - bestCTF = 1; - for deltaCTF = 1:nDefTotal - iRefCTF = refFT .* ... - mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... - single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... - single(df1 + defShiftVect(deltaCTF)),single(df2 + defShiftVect(deltaCTF)),single(dfA),single(TLT(iPrj,18))); - % try - iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')); - cccMap = dataFT .* iRefCTF; + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixel_size]),'fwd'); - if (use_PCF) - cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.001); - end - - cccMap = peakMask.*real(bhF.invFFT(cccMap)); + dataFT = dataFT ./ sqrt(2.*sum(abs(dataFT(1:end-bhF.invTrim,:)).^2,'all')); + + refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixel_size])); - if (save_diagnostic_ccf) - SAVE_IMG(cccMap, 'ccfMap.mrc'); - end - tmpOut(:,:,iFid) = cccMap; - - - [maxVal,maxMap] = max(cccMap(:)); - defocusCCC{iPrj}(deltaCTF,iFid) = maxVal; - - - % It might be better to reorder the search, to reduce the - % number of times this loop is executed (assumming the value is - % closer to the center of the defocusVector) - if maxVal > bestScore - bestScore = maxVal; - bestCTF = deltaCTF; - if ~(calcCTF) - [mMx, mMy] = ind2sub(size(cccMap), maxMap); - - try - - % It would be good to know why this is out of bounds - % sometimes. FIXME - - cccMap = cccMap(mMx-COM:mMx+COM, mMy-COM:mMy+COM); - cccMap = cccMap - min(cccMap(:)); - - comMapX = sum(sum(bx.*cccMap))./sum(cccMap(:)); - comMapY = sum(sum(by.*cccMap))./sum(cccMap(:)); - - - % peak in Map is where query is relative to ref, dXY then is the shift - % needed to move the predicted position to the measured. - % Data moved from a position of estimated_global_offset, so add this to dXY - - dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estimated_global_offset + [sx,sy]; - catch - dXY = estimated_global_offset + [sx,sy]; % TODO double check me - end - - end - end - end % End of loop over defocus values. - - - if (calcCTF) - - [~,imDefC] = max(defocusCCC{iPrj}(:,iFid),[],1); - dCTF = imDefC; - % fprintf('New best score %3.6f for defocus shift %3.3eAng\n', bestScore, defShiftVect(dCTF)); - - dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixel_size]),'fwd'); - refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-5,400,lowPassCutoff,pixel_size])); - - iRefCTF = refFT .* ... + iRefCTF = refFT .* ... mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... - single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... - single(df1 + defShiftVect(dCTF)),single(df2 + defShiftVect(dCTF)),single(dfA),single(TLT(iPrj,18))); - % Renormalize - dataFT = dataFT ./ (sum(abs(dataFT(:)).^2)./numel(dataFT)); - iRefCTF = iRefCTF ./ (sum(abs(iRefCTF(:)).^2)./numel(iRefCTF)); - - cccMap = dataFT .* iRefCTF; - if (use_PCF) - cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.1); - - end - cccMap = peakMask.*real(bhF.invFFT(cccMap)); - - if (save_diagnostic_ccf) - SAVE_IMG(cccMap, 'ccfMap_2.mrc'); - end - [~,maxMap] = max(cccMap(:)); - - [mMx, mMy] = ind2sub(size(cccMap), maxMap); + single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... + single(df1),single(df2 ),single(dfA),single(TLT(iPrj,18))); + % try + iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')); + cccMap = dataFT .* iRefCTF; + + if (use_PCF) + cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.001); + end + + cccMap = peakMask.*real(bhF.invFFT(cccMap)); + + [maxVal,maxMap] = max(cccMap(:)); + + [mMx, mMy] = ind2sub(size(cccMap), maxMap); + try + % It would be good to know why this is out of bounds + % sometimes. FIXME cccMap = cccMap(mMx-COM:mMx+COM, mMy-COM:mMy+COM); - cccMap = cccMap - min(cccMap(:)); comMapX = sum(sum(bx.*cccMap))./sum(cccMap(:)); comMapY = sum(sum(by.*cccMap))./sum(cccMap(:)); - % peak in Map is where query is relative to ref, dXY then is the shift % needed to move the predicted position to the measured. % Data moved from a position of estimated_global_offset, so add this to dXY - - dXY = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2)+ estimated_global_offset + [sx,sy]; + particle_shift = [mMx,mMy]+[comMapX,comMapY] - ctfOrigin(1:2) + [sx,sy]; + catch + particle_shift = [sx,sy]; % TODO double check me end - fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), dXY, wrkFid(iFid,5)); - if (save_diagnostic_ccf) - error('save_diagnostic_ccf'); + + % Re-calculate the scores with the shifted data + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-5,400,lowPassCutoff,pixel_size]),'fwd'); + dataFT = bhF.shiftStretch(dataFT, -1.*particle_shift, 1.0, false); + dataFT = dataFT ./ sqrt(2.*sum(abs(dataFT(1:end-bhF.invTrim,:)).^2,'all')); + + cccMap = dataFT .* iRefCTF; + cccMap = peakMask.*real(bhF.invFFT(cccMap)); + [maxVal_shift,~] = max(cccMap(:)); + % If the shift makes things worse, override and keep just the global and tile shifts + re_shift_for_defocus = false; + + dXY = particle_shift + estimated_global_offset; + if (maxVal_shift - maxVal < 0) + re_shift_for_defocus = true; + dXY = [sx,sy] + estimated_global_offset; end + if (calcCTF) + refFT = conj(bhF.fwdFFT(refTile,1,1,[1e-8,40,3.*pixel_size,pixel_size])); + dataFT = bhF.swapPhase(bhF.fwdFFT(dataTile,1,1,[1e-8,40,3.*pixel_size,pixel_size]),'fwd'); + + if (re_shift_for_defocus) + dataFT = bhF.shiftStretch(dataFT, -1.*[sx,sy], 1.0, false); + else + dataFT = bhF.shiftStretch(dataFT, -1.*particle_shift, 1.0, false); + end + dataFT = dataFT ./ sqrt(2.*sum(abs(dataFT(1:end-bhF.invTrim,:)).^2,'all')); + + % ctfRange = 10000; + % ctfInc = 100; + + n_steps = 3; + + bestScore = -inf; + bestCTF = 0; + i_ctf_range = ctfRange; + for i_defocus_step = 1:n_steps + if (i_defocus_step > 1) + i_ctf_range = ctfRange ./ (5*(i_defocus_step-1)); + end + ctfInc = max(1, floor(i_ctf_range./10)); + nToCheck = max(1, floor(i_ctf_range./ctfInc)); + defShiftVect = ctfInc.*[-nToCheck:nToCheck] + bestCTF; + for deltaCTF = defShiftVect + + iCTF = mexCTF(true,false,int16(CTFSIZE(1)),int16(CTFSIZE(2)),single(samplingRate*TLT(iPrj,16)*10^10), ... + single(TLT(iPrj,18)*10^10),single(TLT(iPrj,17)*10^3),... + single(df1 + deltaCTF),single(df2 + deltaCTF),single(dfA),single(TLT(iPrj,18))); + iRefCTF = refFT .* iCTF; + + % try + iRefCTF = iRefCTF ./ sqrt(2.*sum(abs(iRefCTF(1:end-bhF.invTrim,:)).^2,'all')) ./ (sum(abs(iCTF).^2,'all') ./ numel(iCTF))^2; + cccMap = dataFT .* iRefCTF; + + if (use_PCF) + cccMap = cccMap .* cccMap ./ (abs(cccMap) + 0.001); + end + + cccMap = peakMask.*real(bhF.invFFT(cccMap)); + + [maxVal,maxMap] = max(cccMap(:)); + + % It might be better to reorder the search, to reduce the + % number of times this loop is executed (assumming the value is + % closer to the center of the defocusVector) + if maxVal > bestScore + bestScore = maxVal; + bestCTF = deltaCTF; + end + + end % End of loop over defocus values. + end % End loop stepping down search range + defocusCCC{iPrj}(iFid) = bestScore; + defocusVAL{iPrj}(iFid) = bestCTF; + end % End of calcCTF block + + + + fprintf(coordOUT,'%d %d %0.4f %0.4f %d\n', wrkFid(iFid,1:2), dXY, wrkFid(iFid,5)); end % end of loop over fiducials if (calcCTF) - [~,imDefC] = max(defocusCCC{iPrj},[],1); - expectedDefocus = mean(defShiftVect(imDefC)); + score_weights = gather(defocusCCC{iPrj}); + score_weights = score_weights ./ sum(score_weights); + expectedDefocus = sum(gather(defocusVAL{iPrj}).*score_weights); defocusShifts{iPrj} = expectedDefocus; - fprintf('prj %d delDef %3.3e\n',expectedDefocus); + fprintf('prj %d delDef %3.3e\n', iPrj, expectedDefocus); end evalMaskCell{iPrj} = uint8(evalMask); evalMask = []; fclose(coordOUT); end % end of the parfor loop + if ( calcCTF ) save(sprintf('%smapBack%d/%s%s.defShiftsMat',mbOUT{1:3},outCTF),'defocusShifts'); defShifts = fopen(sprintf('%smapBack%d/%s%s.defShifts',mbOUT{1:3},outCTF),'w'); defCCC = sprintf('%smapBack%d/%s%s_defCCC.mat',mbOUT{1:3},outCTF); - save(defCCC,'defocusCCC','expectedDefocusPerFiducial'); + save(defCCC,'defocusCCC'); for iPrj = 1:nPrjs fprintf(defShifts,'%6.6e\n',defocusShifts{iPrj}); end @@ -1813,225 +1865,17 @@ mbOutAlt{2:3},... mbOutAlt{2:3}); fclose(aliCom_rerun); -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - % % TODO: It looks like the output model file (3dmod) is the solved positions, - % % but is saved at a pixel size of 1. Scaling by the sampling rate in all dimensions - % % and then adding the origin (only for Z) places the coordinates back into the bin6 model - % % I think we could get shifts from this - % % TODO: could use ImageOriginXandY to accound for a diffence in origin due to binning - % % as ImageSizeXandY given as binned size*sampling rate, which may not equal full size - % % fullTiltSizeXandY,... - % % unsampled_pixel_size,unsampled_pixel_size,... - % % 'ImageSizeXandY %d,%d\n',... - % % 'ImagePixelSizeXandY %f,%f\n',... - % fprintf(aliCom,['#!/bin/bash\n\n',... - % '#iTiltSeries %d\n',... - % 'tiltalign -StandardInput << EOF\n',... - % 'ModelFile %smapBack%d/%s_fit-full.fid\n',... - % 'ImagesAreBinned 1\n',... - % 'OutputModelFile %smapBack%d/%s%s.3dmod\n',... - % 'OutputResidualFile %smapBack%d/%s%s.resid\n',... - % 'OutputFidXYZFile %smapBack%d/%s%s.xyz\n',... - % 'OutputTiltFile %smapBack%d/%s%s.tlt\n',... - % 'OutputXAxisTiltFile %smapBack%d/%s%s.xtilt\n',... - % 'OutputTransformFile %smapBack%d/%s%s.tltxf\n',... - % 'RotationAngle 0.00\n',... % assumed to be rotated already - % 'TiltFile %s\n',... - % 'SurfacesToAnalyze %d\n',... - % 'RotOption %d\n',... % def solve all rotations - % 'RotDefaultGrouping 3\n',... % if rot option --> 5 use def group size - % 'TiltOption %d\n',... % Tilts are harder use automapping - % 'TiltDefaultGrouping %d\n',... - % 'MagOption %d\n',... % def solve all mags - % 'MagDefaultGrouping %d\n',... - % 'XStretchOption 0\n',... - % 'SkewOption 0\n',... - % 'BeamTiltOption 0\n',... - % 'XTiltOption 0\n',... - % 'ResidualReportCriterion 0.001\n',... - % 'RobustFitting\n',... - % 'KFactorScaling %3.3f\n',... - % 'LocalAlignments\n',... - % 'LocalRotOption %d\n',... - % 'LocalRotDefaultGrouping %d\n',... - % 'LocalTiltOption %d\n',... - % 'LocalTiltDefaultGrouping %d\n',... - % 'LocalMagOption %d\n',... - % 'LocalMagDefaultGrouping %d\n',... - % 'OutputLocalFile %smapBack%d/%s%s.local\n',... - % 'TargetPatchSizeXandY %d,%d\n', ... - % 'MinFidsTotalAndEachSurface %d,%d\n',... - % 'MinSizeOrOverlapXandY %f,%f\n',... - % '%s\n',... - % '%s\n',... - % '%s\n',... - % 'EOF'],... - % iTiltSeries,... - % mbOutAlt{1:3},... % for ModelFile - % mbOutAlt{1:3},... - % outCTF,... - % mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,... - % mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF,mbOutAlt{1:3},outCTF, ... - % tilt_script_name,... - % n_surfaces, ... - % emc.rot_option_global, ... - % emc.tilt_option_global, ... - % emc.tilt_default_grouping_global, ... - % emc.mag_option_global, ... - % emc.mag_default_grouping_global, ... - % emc.k_factor_scaling, ... - % emc.rot_option_local, ... - % emc.rot_default_grouping_local, ... - % emc.tilt_option_local, ... - % emc.tilt_default_grouping_local, ... - % emc.mag_option_local, ... - % emc.mag_default_grouping_local, ... - % mbOutAlt{1:3},outCTF,targetPatchSize, ... - % targetPatchSize,... - % nFiducialsPerPatch, ... - % floor(nFiducialsPerPatch/3),... - % emc.min_overlap,emc.min_overlap,... - % final_line1,final_line2,final_line3); - % % % % Assume that any backlash was solved well enough that there are no major - % % % % discontinuities in the coarse alignment. Mag and rot are solved/ tilt in - % % % % the global solution anyhow, so this shouldn't be a bit deal. - % % % % 'SeparateGroup 1-%d\n',... - % % % % iViewGroup, - % % fprintf(aliCom,'\n\ngrep -A %d " At minimum tilt" ./mapBack%d/%s_ta.log > ./mapBack%d/tmp.log',nPrjs+2,mbOUT{1:3},mbOUT{1:3}); - % % fprintf(aliCom,'\nawk ''{if(NR >3) print $5}'' ./mapBack%d/tmp.log > mapBack%d/%s.mag',mbOUT{1:3},mbOUT{1:3}); - -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - % system(sprintf('chmod a=wrx %smapBack%d/%s.align',mbOUT{1:3})); - - if (is_first_run) - % if ( multi_node_run ) - % fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'w'); - % fprintf(fOUT,['%smapBack%d/%s.align > ',... - % '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); - % else - % fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}),'w'); - % fprintf(fOUT,['#!/bin/bash\n\n%smapBack%d/%s.align > ',... - % '%smapBack%d/%s.align_ta.log &\n'],mbOutAlt{1:3},mbOutAlt{1:3}); - % end - if ( multi_node_run ) - % These will be aggregated in the main runAlignments.sh - fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'w'); - else - fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}),'w'); - fprintf(fOUT,'#!/bin/bash\n\n'); - end - % fprintf(fOUT,'cat %s | /scratch/etna/master_align.sh `xargs` &\n',aliCom_name); - fprintf(fOUT,'%s\n',aliCom_name_rerun); - - % Since we send to the background in a shell, makes sure the - % function waits on children. - %if (iTiltSeries == nTiltSeries) - % fprintf(fOUT,'\nwait\n'); - %end - fclose(fOUT); - is_first_run = false; - else - if ( multi_node_run ) - fOUT = fopen(sprintf('%smapBack%d/runAlignments_%d_%d.sh',mbOUT{1:2},tiltStart,nTiltSeries),'a'); - else - fOUT = fopen(sprintf('%smapBack%d/runAlignments.sh',mbOUT{1:2}), 'a'); - end - % fprintf(fOUT,['%smapBack%d/%s.align > ',... - % '%smapBack%d/%s.align_ta.log &\n'], ... - % mbOutAlt{1:3},mbOutAlt{1:3}); - fprintf(fOUT,'%s\n',aliCom_name_rerun); - - % Since we send to the background in a shell, makes sure the - % function waits on children. - %if (iTiltSeries == nTiltSeries) - % fprintf(fOUT,'\nwait\n'); - %end - fclose(fOUT); - end - % if (emc.run_tomocpr_alignments) - - % end + - %system(sprintf('./mapBack/%s.align > ./mapBack/%s_ta.log',TN,TN)); - - %%%%%%%%% There is still sometimes a shift in Z, fit slope of the X shifts - %%%%%%%%% and make cutoff compared to zshift to re-run the alignment with - %%%%%%%%% this value entered for AxisZShift. - - % Until I hear from DAVID get the mag from the tilt log - %%%system(sprintf('grep -A %d " At minimum tilt" ./mapBack/%s_ta.log > tmp.log',nPrjs+2,TN)); - %%%system(sprintf('awk ''{if(NR >3) print $5}'' tmp.log > mapBack/%s.mag',TN)); - %%%end %uf cibdutuib + % This is an partial run + system(sprintf('mkdir -p cache/mapBack%d', mbOUT{2})); + system(sprintf('mv %smapBack%d/%s* cache/mapBack%d/', mbOUT{1:2},mbOutAlt{3}, mbOUT{2})); end % loop over tilts -if ( flgRunAlignments ) - - mainFile = sprintf('cache/mapBack%d/runAlignments.sh',mbOUT{2}); - altFiles = sprintf('cache/mapBack%d/runAlignments_*.sh',mbOUT{2}); - % Only possible as [cycle, 0, 0] - if (multi_node_run) - system(sprintf('rm %s && touch %s',mainFile,mainFile)); - fprintf('Combining Results from alt and main\n'); - system(sprintf('cat %s >> %s',altFiles,mainFile)); - end - - % fOUT = fopen(mainFile,'a'); - % fprintf(fOUT,'\nwait\n'); - % fclose(fOUT); - - % system(sprintf('chmod a=wrx %smapBack%d/runAlignments.sh', mbOUT{1:2})); - % system(sprintf('%smapBack%d/runAlignments.sh', mbOUT{1:2})); - - % BH_multi_parallelWorkers will return at most nthreads - system(sprintf("cat cache/mapBack%d/runAlignments.sh | parallel -j%d 'cat {} | /scratch/etna/master_align.sh `xargs`'", mbOUT{1:2}, floor(BH_multi_parallelWorkers(256)/2))) -end - -if ( conserveDiskSpace ) - system(sprintf('rm %smapBack%d/%s_mapBack.st', mbOUT{1:3})); -end - - -if (flgRunAlignments) - system(sprintf('mv %smapBack%d mapBack%d', mbOUT{1:2}, mbOUT{2})); -else - if (multi_node_run) - % This is an partial run - system(sprintf('mkdir -p cache/mapBack%d', mbOUT{2})); - system(sprintf('mv %smapBack%d/* cache/mapBack%d', mbOUT{1:2}, mbOUT{2})); - else - error('This should not happen'); - end -end - - -% if (flgCleanCache) -% % Double check that this exists to avoid data loss. -% checkDir = dir(tmpCache); -% if isempty(checkDir) -% fprintf('not removing the temp cache because it did not eval with dir\n'); -% else -% cleanItUp = sprintf('rm %s/*',tmpCache); -% system(cleanItUp); -% end -% end -% Since we've updated (potentially) mapBackRePrjSize, save the new metaData. -if (flgRunAlignments) - subTomoMeta.currentTomoCPR = subTomoMeta.currentTomoCPR + 1; - - if isfield(subTomoMeta,'tomoCPR_run_in_cycle') - subTomoMeta.('tomoCPR_run_in_cycle') = cat(1,subTomoMeta.('tomoCPR_run_in_cycle'),... - [subTomoMeta.currentTomoCPR,CYCLE]); - else - subTomoMeta.('tomoCPR_run_in_cycle') = [subTomoMeta.currentTomoCPR,CYCLE]; - end - - save(emc.('subTomoMeta'), 'subTomoMeta'); -end end diff --git a/synthetic/BH_to_cisTEM_mapBack.m b/synthetic/BH_to_cisTEM_mapBack.m index 9e636e8b..383aabeb 100644 --- a/synthetic/BH_to_cisTEM_mapBack.m +++ b/synthetic/BH_to_cisTEM_mapBack.m @@ -1,4 +1,4 @@ -function [ ] = BH_to_cisTEM_mapBack(PARAMETER_FILE, CYCLE, output_prefix, symmetry, MAX_EXPOSURE, mapBackIter) +function [ ] = BH_to_cisTEM_mapBack(PARAMETER_FILE, CYCLE, output_prefix, symmetry, MAX_EXPOSURE, classIDX) % Map back and align using the subtomograms as fiducial markers. @@ -16,7 +16,7 @@ emc = BH_parseParameterFile(PARAMETER_FILE); -mapBackIter = EMC_str2double(mapBackIter); +classIDX = EMC_str2double(classIDX) MAX_EXPOSURE = EMC_str2double(MAX_EXPOSURE) if isnan(MAX_EXPOSURE) error('MAX_EXPOSURE is nan - if running from an interactive matlab session, did you enter as a string?'); @@ -95,9 +95,10 @@ load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); -if (mapBackIter == -1) - mapBackIter = subTomoMeta.currentTomoCPR; +if (classIDX == -1) + classIDX = 0; end +mapBackIter = subTomoMeta.currentTomoCPR; % TODO: use these to add an optional defocus fitting step @@ -166,7 +167,8 @@ if (skip_to_the_end_and_run) continue; end - + + if (useFixedNotAliStack) tilt_filestem = tilt_series_filenames{iTiltSeries}; tilt_filepath = sprintf('%sfixedStacks/%s.fixed', CWD, tilt_series_filenames{iTiltSeries}); @@ -186,6 +188,8 @@ skip_this_tilt_series_because_it_is_empty = false(n_tomos_this_tilt_series,1); % tomoList = fieldnames(subTomoMeta.mapBackGeometry.tomoName); + % FIXME: switch to the included tilts, but also add a check there is > 1 subtomo + % First determine why things are failing with only one subtomo as this could point to an off by one error, which may have significant implications for the alignment. tomoList = {}; n_active_tomos = 0; fn = fieldnames(subTomoMeta.mapBackGeometry.tomoName); @@ -193,9 +197,16 @@ if strcmp(subTomoMeta.mapBackGeometry.tomoName.(fn{iTomo}).tiltName, tilt_series_filenames{iTiltSeries}) % This is dumb, fix it to be explicit. if (subTomoMeta.mapBackGeometry.tomoCoords.(fn{iTomo}).is_active) - tomoList{n_active_tomos+1} = fn{iTomo}; - % Only increment if values found. - n_active_tomos = n_active_tomos + 1; + if (classIDX == 0) + n_subtomos = sum(geometry.(fn{iTomo})(:,26) ~= -9999); + else + n_subtomos = sum(geometry.(fn{iTomo})(:,26) == classIDX); + end + if (n_subtomos > 0) + tomoList{n_active_tomos+1} = fn{iTomo}; + % Only increment if values found. + n_active_tomos = n_active_tomos + 1; + end end end end @@ -388,8 +399,12 @@ positionList = geometry.(tomoList{iTomo}); - - positionList = positionList(positionList(:,26) ~= -9999,:); + if (classIDX == 0) + positionList = positionList(positionList(:,26) ~= -9999,:); + else + positionList = positionList(positionList(:,26) == classIDX,:); + end + nFidsTotal = nFidsTotal + size(positionList,1); tiltHeader = getHeader(MRCImage(tilt_filepath,0)); @@ -665,7 +680,7 @@ STACK = OPEN_IMG('single',tilt_filepath); for iPrj = 1:nPrjs - + n_included_this_prj = 0; if (abs(TLT(iPrj,11)) > MAX_EXPOSURE) continue; end @@ -707,6 +722,7 @@ particle_was_skipped = false; if ( x_start > 0 && y_start > 0 && x_start + tileSize(1) - 1 < sTX && y_start + tileSize(2) - 1 < sTY ) + n_included_this_prj = n_included_this_prj + 1; output_particle_stack(:,:,iGpuDataCounter) = STACK(x_start:x_start+tileSize(1) - 1,y_start:y_start+tileSize(2) - 1,TLT(iPrj,1)); % The trasformation of the particle is e1,e2,e3,esym into it's postion in the tomogram frame, then @@ -785,16 +801,20 @@ end % end of prj loop - % Trim the stack to account for windowing skips - output_particle_stack = gather(output_particle_stack(:,:,1:iGpuDataCounter - 1)); - tmp_stack_filename = sprintf('%s/%s_%d.mrc',mbOUT{1:2},iCell); - SAVE_IMG(output_particle_stack, tmp_stack_filename, pixelSize); + n_included_this_prj - fprintf(newstack_file_handle, '%s\n',tmp_stack_filename); - fprintf(newstack_file_handle, '0-%d\n',iGpuDataCounter-2); + if (n_included_this_prj > 0) + % Trim the stack to account for windowing skips + output_particle_stack = gather(output_particle_stack(:,:,1:iGpuDataCounter - 1)); + tmp_stack_filename = sprintf('%s/%s_%d.mrc',mbOUT{1:2},iCell); + SAVE_IMG(output_particle_stack, tmp_stack_filename, pixelSize); - % output_cell{iCell}= gather(output_particle_stack); - iCell = iCell + 1; + fprintf(newstack_file_handle, '%s\n',tmp_stack_filename); + fprintf(newstack_file_handle, '0-%d\n',iGpuDataCounter-2); + + % output_cell{iCell}= gather(output_particle_stack); + iCell = iCell + 1; + end end % end of the loop over tilt series @@ -827,7 +847,7 @@ leftover_threads = maxThreads - n_threads_per_proc(1)*n_recon_procs; thread_counter = 1; while (leftover_threads > 0) - this_proc = mod(thread_counder,4)+1; + this_proc = mod(thread_counter,4)+1; n_threads_per_proc(this_proc) = n_threads_per_proc(this_proc) + 1; leftover_threads = leftover_threads - 1; end @@ -841,9 +861,9 @@ stack_boundaries(end) = outputNumberOfSlices+1; end -do_initial = false +do_initial = true -if (do_initial) % revert +if (do_initial) % % %%%%%%%%%%%%%%%%%%%%%%%%% % Initial reconstruction % %%%%%%%%%%%%%%%%%%%%%%%%%%% @@ -916,7 +936,7 @@ break; end end - if all_founddo_initial + if all_found break; else fprintf('Waiting for reconstructions to finish...\n'); @@ -961,8 +981,8 @@ % Get the FSC cutoff for refinement fsc = importdata(sprintf('%s_stats.txt',output_prefix),' ',12); -fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) - +% fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)); +fsc_cutoff = fsc.data(find(fsc.data(:,4) < 0.5,1)) %%%%%%%%%%%%%%%%%%%%%%%%% % Refine %%%%%%%%%%%%%%%%%%%%%%%%%%%%r @@ -1149,7 +1169,8 @@ % Get the FSC cutoff for refinement fsc = importdata(sprintf('%s_stats_refined.txt',output_prefix),' ',12); -fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) +% fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) +fsc_cutoff = fsc.data(find(fsc.data(:,4) < 0.5,1)) % Use to decide whether to do more angular refinement fsc_last = fsc_cutoff @@ -1337,15 +1358,16 @@ system(sprintf('rm %s/%sdump_?_*.dat',tmpCache,output_prefix)); -end % revert do _intial +end % do _intial % Get the FSC cutoff for refinement fsc = importdata(sprintf('%s_stats_refined2.txt',output_prefix),' ',12); -fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) +% fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) +fsc_cutoff = fsc.data(find(fsc.data(:,4) < 0.5,1)) fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) -if ~(do_initial) % revert +if ~(do_initial) % fsc_last = fsc_cutoff / 0.94; end n_max_refinements = 7; @@ -1564,7 +1586,8 @@ % Get the FSC cutoff for refinement fsc = importdata(sprintf('%s_stats_refined%d.txt',output_prefix, i_refine),' ',12); -fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) +% fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) +fsc_cutoff = fsc.data(find(fsc.data(:,4) < 0.5,1)) fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) @@ -1759,7 +1782,8 @@ system(sprintf('rm %s/%sdump_?_*.dat',tmpCache,output_prefix)); fsc = importdata(sprintf('%s_stats_refined3.txt',output_prefix),' ',12); - fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) + % fsc_cutoff = 0.5 * (fsc.data(find(fsc.data(:,5) < 0.5,1),2) + fsc.data(find(fsc.data(:,4) < 0.5,1),2)) + fsc_cutoff = fsc.data(find(fsc.data(:,4) < 0.5,1)) fsc_res = fsc.data(find(fsc.data(:,5) < 0.143,1),2) end % defocus refine loop diff --git a/testScripts/EMC_assert_string_value.m b/testScripts/EMC_assert_string_value.m new file mode 100644 index 00000000..a6083e60 --- /dev/null +++ b/testScripts/EMC_assert_string_value.m @@ -0,0 +1,29 @@ +function EMC_assert_string_value(input_value, valid_string_cell, is_case_sensitive) + + value_is_valid = false; + if ~isa(valid_string_cell, 'cell') + error('EMC_assert_string_value: input_string_cell is not a cell array'); + end + + for i = 1:length(valid_string_cell) + if ~ischar(valid_string_cell{i}) + error('EMC_assert_string_value: valid_string_cell{%d} is not a string', i); + end + if (is_case_sensitive) + if (strcmp(input_value, valid_string_cell{i})) + value_is_valid = true; + break; + end + else + if (strcmpi(input_value, valid_string_cell{i})) + value_is_valid = true; + break; + end + end + + end + if ~value_is_valid + error('EMC_assert_string_value: (%s) input_value is not a valid string', input_value); + end + +end \ No newline at end of file diff --git a/testScripts/adamOptimizer.m b/testScripts/adamOptimizer.m new file mode 100644 index 00000000..5e7b83fc --- /dev/null +++ b/testScripts/adamOptimizer.m @@ -0,0 +1,64 @@ +classdef adamOptimizer < handle + + properties (Access = private) + alpha = 0.001; % Learning rate + beta1 = 0.9; % Exponential decay rate for the first moment estimates + beta2 = 0.999; % Exponential decay rate for the second moment estimates + epsilon = 1e-8; % Small value to prevent division by zero + m; % First moment vector + v; % Second moment vector + t = 0; % Time step + initial_parameters = []; + current_parameters = []; % This can be used to store the current parameters if needed + end + + methods + + function obj = adamOptimizer(initial_parameters) + % Check that the initial parameters are >= 1 and one dimensional + if nargin < 1 || isempty(initial_parameters) || ~isvector(initial_parameters) + error('Initial parameters must be a non-empty vector.'); + end + if length(initial_parameters) < 1 + error('Initial parameters must have at least one element.'); + end + % Constructor to initialize the first and second moment vectors + obj.m = zeros(length(initial_parameters), 1); % Initialize first moment vector + obj.v = zeros(length(initial_parameters), 1); % Initialize second moment vector + + % Store the initial parameters + obj.initial_parameters = initial_parameters(:); % Ensure it's a column vector + obj.current_parameters = obj.initial_parameters; % Initialize current parameters + end + + function update(obj, gradient) + % Increment time step + obj.t = obj.t + 1; + + % Update biased first moment estimate + obj.m = obj.beta1 * obj.m + (1 - obj.beta1) .* gradient; + + % Update biased second raw moment estimate + obj.v = obj.beta2 * obj.v + (1 - obj.beta2) .* (gradient .^ 2); + + % Compute bias-corrected first moment estimate + m_hat = obj.m / (1 - obj.beta1 .^ obj.t); + + % Compute bias-corrected second raw moment estimate + v_hat = obj.v / (1 - obj.beta2 .^ obj.t); + + % Update parameters using the Adam update rule + obj.current_parameters = obj.current_parameters - obj.alpha .* m_hat ./ (sqrt(v_hat) + obj.epsilon); + + % Apply the parameter update (this would be applied to the model parameters in practice) + end + + function params = get_current_parameters(obj) + % Getter method to retrieve the current parameters + params = obj.current_parameters; + end + + + end + +end \ No newline at end of file diff --git a/testScripts/emClarity.m b/testScripts/emClarity.m index 1a942e9b..aee81129 100755 --- a/testScripts/emClarity.m +++ b/testScripts/emClarity.m @@ -4,7 +4,6 @@ % Disable warnings - warning off cudaStart=''; % FIXME: This only applies to ctf estimate, and it looks like there is no version 2 !? @@ -162,6 +161,8 @@ multiGPUs = 0; case 'segment' multiGPUs = 0; + case 'getActiveTilts' + multiGPUs = 0; otherwise emc = emC_testParse(varargin{2}); end @@ -189,6 +190,7 @@ '\n\t\t for more details, emClarity help\n',... '\ncheck - system check for dependencies\n',... '\nsegment - define subregions to reconstruct\n',... + '\ngetActiveTilts - get the number of active tilt-series\n',... '\ninit - create a new project from template matching results.\n',... '\nautoAlign - align tilt-serie\n',... '\navg - average subtomograms\n',... @@ -210,6 +212,7 @@ '\nreconstruct - reconstruct a volume from a set of subtomograms\n',... '\nremoveDuplicates - remove subtomos that have migrated to the same position\n',... '\nexperimental - experimental options\n',... + '\nmontage - unstack and/or rotate the elements of a montage about x\n',... '\nremoveNeighbors - clean templateSearch results based on lattice constraints\n']); % Currently disabled options. Multi-reference alignment @@ -233,6 +236,17 @@ else recScript(varargin{2}); end + case 'getActiveTilts' + if emcProgramHelp || ... + length(varargin) ~= 2 + fprintf(['\nUsage: emClarity getActiveTilts param.m\n']); + else + emc = emC_testParse(varargin{2}); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); + [tiltNameList, nTiltSeries] = BH_returnIncludedTilts( subTomoMeta.mapBackGeometry ); + fprintf('%d\n',nTiltSeries); + return + end case 'init' if emcProgramHelp || ... length(varargin) < 2 && length(varargin)> 5 @@ -311,6 +325,7 @@ '\n\nFor a shape based mask\n', ... 'fileNameIN.mrc,fileNameOUT.mrc, pixelSize (Ang)\n']); + else switch length(varargin) case 5 @@ -421,11 +436,11 @@ length(varargin) ~= 4 && length(varargin) ~= 5 fprintf(['\nparam.m\n',... 'cycle number\n',... - 'randomSubset\n']); - % 'use focused mask\n',... - % ' 1 from standard devation\n',... - % ' 2 from variance\n',... - % ' 3 user supplied (not recommended)\n']); + 'randomSubset\n',... + 'use focused mask\n',... + ' 1 from standard devation\n',... + ' 2 from variance\n',... + ' 3 user supplied (not recommended)\n']); else emC_testParse(varargin{2}); @@ -439,7 +454,7 @@ else maskVal = 0; end - + if (maskVal) % re-run on full or randomsubset now using variance or stddev mask BH_pcaPub(varargin{2}, varargin{3}, sprintf('%d',-1.*maskVal)) @@ -525,18 +540,23 @@ fprintf('In tomoCPR the MCR is %s\n',getenv('MCR_CACHE_ROOT')); if emcProgramHelp || ... - ( length(varargin) < 3 || length(varargin) > 4 ) + ( length(varargin) < 4 || length(varargin) > 5 ) fprintf(['\nparam.m\n',... 'cycle number\n',... - 'nTiltStart\n']); + 'stage of alignment [most recent stage finished for geometry metadat, eg - Avg, RawAlignment, Cluster_cls]\n',... + '\n']); else emC_testParse(varargin{2}); - if length(varargin) == 4 + if length(varargin) == 5 tiltStart = EMC_str2double(varargin{4}); else tiltStart = 1; end - BH_synthetic_mapBack(varargin{2}, varargin{3}, tiltStart); + % Check that the stage of alignment is valid + if ~ismember(varargin{4},{'Avg','RawAlignment','Cluster_cls'}) + error('Stage of alignment must be one of Avg, RawAlignment, or Cluster_cls'); + end + BH_synthetic_mapBack(varargin{2}, varargin{3}, varargin{4},tiltStart); end case 'removeDuplicates' if emcProgramHelp || ... @@ -622,15 +642,145 @@ end + case 'montage' + if emcProgramHelp || ... + length(varargin) ~= 7 + fprintf(['parameterfile\n',... + 'cycle #\n',... + 'stage of alignment [RawAlignment or Cluster_cls]\n', ... + 'class number\n',... + 'operation [unstack or angles to rotate volume by [ZXZ]]\n', ... + 'halfset [odd, eve, or std (combine)]\n ']); + else + % FIMXE: move all this to a different file + % FIXME: eve/odd + % FIXME: unstack (add counts too) + % FIXME: set pixel size + % Check the input arguments + EMC_assert_string_value(varargin{7}, {'odd', 'eve', 'std'}, false); + halfset = 0; % default is combine + if strcmpi(varargin{7},'eve') + halfset = 2; + elseif strcmpi(varargin{7},'odd') + halfset = 1; + end + if halfset ~= 0 + error('partial implementation only good for std right now') + end + operation_val = varargin{6}; + if strcmpi(operation_val,'unstack') + operation = 'unstack'; + else + operation = 'rotx'; + operation_val = EMC_str2double(operation_val); + EMC_assert_numeric(operation_val, 3); + end + + n_classes = EMC_str2double(varargin{5}); + EMC_assert_numeric(n_classes, 1, [0, 1000]); + + EMC_assert_string_value(varargin{4}, {'RawAlignment', 'Cluster_cls'}, false) + if strcmpi(varargin{4},'RawAlignment') + prfx = 'Ref'; + else + prfx = 'Cls'; + end + cycle = EMC_str2double(varargin{3}); + EMC_assert_numeric(cycle, 1, [0, 1000]); + + % Read in the parameter file and subTomoMeta + cycleNumber = sprintf('cycle%0.3u', cycle); + load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); + emc = BH_parseParameterFile(varargin{2}); + % Make sure the cycle has been run + if ~isfield(subTomoMeta, cycleNumber) + error('Cycle %s has not been run yet', cycleNumber); + end + % Make sure the class locations information is present + if (halfset == 0 || halfset == 1) + fname = sprintf('class_%d_Locations_%s_%s', n_classes, prfx, 'ODD'); + if ~isfield(subTomoMeta.(cycleNumber),fname) + error('Class locations (%s) for odd half-set are not present', fname); + end + end + if (halfset == 0 || halfset == 2) + fname = sprintf('class_%d_Locations_%s_%s', n_classes, prfx, 'EVE'); + if ~isfield(subTomoMeta.(cycleNumber),fname) + error('Class locations (%s) for even half-set are not present', fname); + end + end + % Eve/Odd should always have the same size windows, grab the first + % Field is a cell, 1 file names of the references, 2 locations in the image (6 indices), 3 number added to the averages. + window_size = subTomoMeta.(cycleNumber).(fname){2}{1}(2:2:end); + odd_stack = []; + eve_stack = []; + if (halfset == 0 || halfset == 1) + fname = sprintf('class_%d_Locations_%s_%s', n_classes, prfx, 'ODD'); + odd_stack = BH_unStackMontage4d(1:length(subTomoMeta.(cycleNumber).(fname){2}), ... + subTomoMeta.(cycleNumber).(fname){1}, ... + subTomoMeta.(cycleNumber).(fname){2}, ... + window_size); + end + if (halfset == 0 || halfset == 2) + fname = sprintf('class_%d_Locations_%s_%s', n_classes, prfx, 'EVE'); + eve_stack = BH_unStackMontage4d(1:length(subTomoMeta.(cycleNumber).(fname){2}), ... + subTomoMeta.(cycleNumber).(fname){1}, ... + subTomoMeta.(cycleNumber).(fname){2}, ... + window_size); + end + if strcmpi(operation,'unstack') + error('not setup yet') + elseif strcmpi(operation,'rotx') + if (halfset == 0 || halfset == 1) + for i = 1:length(odd_stack) + [~,img] = interpolator(gpuArray(odd_stack{i}),operation_val,[0,0,0], 'Bah' , 'forward', 'C1', false); + odd_stack{i} = gather(img); + end + end + if (halfset == 0 || halfset == 2) + for i = 1:length(eve_stack) + [~,img] = interpolator(gpuArray(eve_stack{i}),operation_val,[0,0,0], 'Bah' , 'forward', 'C1', false); + eve_stack{i} = gather(img); + end + end + % Combine them if we want to + switch halfset + case 0 + % Combine the two half-sets + for i = 1:length(odd_stack) + odd_stack{i} = odd_stack{i} + eve_stack{i}; + end + montOUT = BH_montage4d(odd_stack,''); + fname = strjoin(strsplit(subTomoMeta.(cycleNumber).(sprintf('class_%d_Locations_%s_%s',n_classes, prfx,'ODD')){1},'_ODD.mrc'),sprintf('_rot_%2.2f_%2.2f_%2.2f_STD_rotx.mrc',operation_val(1),operation_val(2),operation_val(3))); + % TODO pixels size + SAVE_IMG(montOUT, fname); %,pixelSize); + + case 1 + % Save the odd half-set + for i = 1:length(odd_stack) + odd_stack{i} = odd_stack{i}; + end + case 2 + % Save the even half-set + for i = 1:length(odd_stack) + eve_stack{i} = eve_stack{i}; + end + otherwise + error('halfset must be 0, 1, or 2') + end + % Save the stack + end + end + case 'reconstruct' if emcProgramHelp || ... length(varargin) ~= 6 && length(varargin) ~= 7 - fprintf(['paramterfile\n',... + fprintf(['parameterfile\n',... 'cycle #\n',... 'output prefix\n', ... 'symmetry (C1)\n',... 'max exposure (e/A^2)\n', ... - 'mapBackIter']); + 'classIDX']); else if (length(varargin) == 7) BH_to_cisTEM_mapBack(varargin{2},varargin{3},varargin{4},varargin{5},varargin{6}, varargin{7}); @@ -798,8 +948,6 @@ bh_global_kFactorScaling = 1.0; end - - try bh_global_vol_est_scaling = emc.('setParticleVolumeScaling'); catch diff --git a/testScripts/emc_to_cpp.md b/testScripts/emc_to_cpp.md new file mode 100644 index 00000000..b7b90852 --- /dev/null +++ b/testScripts/emc_to_cpp.md @@ -0,0 +1,94 @@ +# MATLAB to C++ Conversion Plan: emClarity Project + +## 1. Project Overview & Goals + +* **Project**: emClarity MATLAB project. +* **Focus of this Plan**: Conversion of the main entry point script (assumed to be `emClarity.m` or a similar top-level script) and subsequently its sub-programs. +* **Overall Goal**: Convert the emClarity project from MATLAB to C++ to potentially improve performance, reduce licensing dependencies, and facilitate integration with other C++ tools. +* **Strategy**: Phased conversion, starting with the main program and progressively converting its core sub-modules. + +## 2. Analysis of the Main Program (`emClarity.m`) + +*(This section will require details about `emClarity.m`)* + +* **Purpose**: + * Acts as the primary user interface or entry point. + * Parses user commands/arguments. + * Dispatches tasks to various sub-programs or functions within the emClarity suite. +* **Key Functionalities**: + * Argument parsing (e.g., identifying which sub-program to run and its parameters). + * Calling/dispatching to other MATLAB scripts/functions (e.g., `BH_synthetic_mapBack.m`, alignment routines, classification, etc.). + * Global setup or environment configuration. + * Help system / usage information. +* **Inputs**: + * Command-line arguments or function parameters. + * Parameter files. +* **Outputs**: + * Messages to the console. + * Calls to sub-programs which produce their own outputs. + +## 3. Dependency Analysis (from `emClarity.m` perspective) + +* **Primary Dependencies**: + * List of core MATLAB scripts/functions directly called by `emClarity.m` (e.g., `BH_align_subTomos`, `BH_tomoCPR_cluster`, `BH_synthetic_mapBack`, etc.). + * Parameter parsing functions (e.g., `BH_parseParameterFile`). +* **MATLAB Toolboxes**: + * Identify any toolboxes used directly by `emClarity.m` or its immediate utility functions. +* **Shared Utility Functions**: + * Common helper functions used for path management, string operations, etc. + +## 4. C++ Conversion Strategy for `emClarity.m` + +* **C++ Main Executable**: + * The C++ equivalent of `emClarity.m` will be a `main()` function in a C++ executable. +* **Argument Parsing**: + * Implement robust command-line argument parsing. + * Libraries: `cxxopts`, `CLI11`, `Boost.Program_options`, or a custom solution. + * The goal is to replicate the way `emClarity.m` receives and interprets user commands. +* **Dispatching Logic**: + * Based on parsed arguments, the C++ `main` function will call the C++ equivalents of the MATLAB sub-programs. + * This might involve a series of `if/else if` statements or a map of command strings to function pointers/functors. +* **Parameter Handling**: + * The C++ version will need to read and parse parameter files, similar to `BH_parseParameterFile.m`. This module should be one of the first to be converted. +* **Build System**: + * CMake is recommended for managing the C++ project, its modules, and dependencies. + +## 5. Modular Conversion Approach & Prioritization + +1. **Core Utilities & Parameter Parsing**: + * Convert `BH_parseParameterFile.m` to a C++ equivalent. + * Convert essential utility functions (string manipulation, file system interaction). +2. **Main Dispatcher (`emClarity.cpp`)**: + * Develop the C++ `main` function with argument parsing and basic dispatching stubs. +3. **Sub-Program Conversion (Phased)**: + * Identify a key sub-program (e.g., `BH_synthetic_mapBack.m` as previously discussed, or another core module). + * Create a detailed conversion plan for that sub-program (similar to the one we started for `BH_synthetic_mapBack.m`). + * Convert and test the sub-program in C++. + * Integrate the converted C++ sub-program with the C++ main dispatcher. + * Repeat for other sub-programs. + * **Priority List (Example - to be refined based on `emClarity.m` structure)**: + * Parameter parsing (`BH_parseParameterFile`) + * `BH_synthetic_mapBack` (synthetic data generation and basic alignment) + * Core alignment routines + * Classification routines + * Other reconstruction/processing modules + +## 6. C++ Libraries and Tools + +* **Standard Libraries**: C++17 or newer for ``, ``, ``, etc. +* **Image Processing**: OpenCV, ITK, or existing custom C++ image libraries. +* **MRC I/O**: `libmrc` or custom implementation. +* **FFT**: FFTW (CPU), cuFFT (GPU). +* **Linear Algebra**: Eigen. +* **Parallelism**: OpenMP, C++ ``, TBB. +* **Logging**: spdlog, glog. + +## 7. Testing and Validation Strategy + +* **Unit Tests**: For individual C++ functions and classes. +* **Integration Tests**: For modules and the interaction between the main dispatcher and sub-programs. +* **Comparison with MATLAB**: Output data (images, metadata files, logs) from C++ versions should be compared against outputs from the original MATLAB code using the same input datasets. Numerical precision differences should be anticipated and managed. + +--- + +To proceed with detailing this plan, especially sections 2 and 3, please provide the content of `emClarity.m` or describe its main functionalities and how it calls other scripts. \ No newline at end of file diff --git a/testScripts/fourierTransformer.m b/testScripts/fourierTransformer.m index b9fa4347..20e326c1 100644 --- a/testScripts/fourierTransformer.m +++ b/testScripts/fourierTransformer.m @@ -213,9 +213,9 @@ function delete(obj) end function [inputVol] = shiftStretch(obj, inputVol, shiftXY, Mag, isCentered) - + [ dU, dV ] = BH_multi_gridCoordinates(obj.inputSize,'Cartesian','GPU', ... - {'none'},1,isCentered,0,{'halfgrid'}); + {'none'},1,isCentered,0,{'halfgrid'}); inputVol = inputVol .* (Mag.^-2.*exp(-2i.*pi.*(dU.*shiftXY(1)+dV.*shiftXY(2)))); clear dU dV @@ -342,21 +342,21 @@ function makeBandPass(obj, sizeInput, bpValsNew) bpValsNew(2),... bpValsNew(3),... 'GPU', ... - bpValsNew(4)); - switch ndims(obj.bandpass) - case 3 - obj.bandpass = obj.bandpass(1:obj.halfDimSize,:,:); - case 2 - obj.bandpass = obj.bandpass(1:obj.halfDimSize,:); - case 1 - obj.bandpass = obj.bandpass(1:obj.halfDimSize); + bpValsNew(4), 'halfGrid'); + + % switch ndims(obj.bandpass) + % case 3 + % obj.bandpass = obj.bandpass(1:obj.halfDimSize,:,:); + % case 2 + % obj.bandpass = obj.bandpass(1:obj.halfDimSize,:); + % case 1 + % obj.bandpass = obj.bandpass(1:obj.halfDimSize); + % end + + % Update the properties + obj.bpDoesNotExist = false; + obj.bpVals = bpValsNew; end - - % Update the properties - obj.bpDoesNotExist = false; - obj.bpVals = bpValsNew; - end - end diff --git a/testScripts/logFile/emClarity.logfile b/testScripts/logFile/emClarity.logfile new file mode 100644 index 00000000..67402490 --- /dev/null +++ b/testScripts/logFile/emClarity.logfile @@ -0,0 +1,32 @@ + + *************************************** + +emClarity version 57a0d00 +run starting on 03-Apr-2025 07:12:20 +cmd int matlab -nosplash -nodisplay + + *************************************** + +Error using textscan +Invalid file identifier. Use fopen to generate a valid file identifier. +Error in BH_parseParameterFile (line 7) + +Error in emClarity>emC_testParse (line 679) + +Error in emClarity (line 168) + + *************************************** + +emClarity version 57a0d00 +run starting on 03-Apr-2025 07:30:44 +cmd int matlab + + *************************************** + +Error using textscan +Invalid file identifier. Use fopen to generate a valid file identifier. +Error in BH_parseParameterFile (line 7) + +Error in emClarity>emC_testParse (line 679) + +Error in emClarity (line 168) diff --git a/testScripts/mCompile.sh b/testScripts/mCompile.sh index 16746437..962b8bb5 100755 --- a/testScripts/mCompile.sh +++ b/testScripts/mCompile.sh @@ -44,8 +44,8 @@ outName="$(basename ${mFile} .m)${post}" # bugs line. e.g. buggs=5testingFeature major=1 minor=8 -bugs=2 -nightly=2 +bugs=3 +nightly=0 binaryOutName="${major}_${minor}_${bugs}_${nightly}" scriptOutName="${major}_${minor}_${bugs}_${nightly}_v23a" diff --git a/testScripts/optimize_gmm_weights.m b/testScripts/optimize_gmm_weights.m new file mode 100644 index 00000000..6654fd7f --- /dev/null +++ b/testScripts/optimize_gmm_weights.m @@ -0,0 +1,106 @@ +% function [] = optimize_gmm_weights(starting_gmm, data_vector) + + starting_gmm = GMM; + data_vector = orig_vector; +% Make sure the data vector size matches expected +if (size(data_vector, 2) ~= size(starting_gmm.mu, 2)) + error('Data vector size does not match GMM parameters'); +end + +n_batches = 4; + +randomized_idx = randperm(size(data_vector,1)); +bin_size = floor(1/n_batches * size(data_vector,1)); + +validation_idx = randomized_idx(1:bin_size); +test_idx = randomized_idx(bin_size+1:end); + +n_epochs = 1; + + +n_cores = 16; +% try +% EMC_parpool(n_cores); +% catch +% delete(gcp('nocreate')); +% pause(3) +% EMC_parpool(n_cores); +% end + +fprintf('Starting log likelihood: %f\n',starting_gmm.NegativeLogLikelihood); + + + +optimizer = adamOptimizer(ones(starting_gmm.NumVariables,1)); +current_gmm = starting_gmm; +for i_epoch = 1:n_epochs + for i_batch = 1:n_batches + if (i_batch < n_batches) + batch_idx = test_idx((i_batch-1)*bin_size+1:i_batch*bin_size); + else + batch_idx = test_idx((i_batch-1)*bin_size+1:end); + end + + + + weights = [0.9 .* optimizer.get_current_parameters()' ; ... + 1.1 .* optimizer.get_current_parameters()']; + loss = cell(2,starting_gmm.NumVariables); + + + starting_values = struct('mu',current_gmm.mu,'Sigma',current_gmm.Sigma,'ComponentProportion',current_gmm.PComponents); + for i_weight = 1:starting_gmm.NumVariables + for lowhigh = 1:2 + gm = gmdistribution(current_gmm.mu,current_gmm.Sigma, current_gmm.PComponents); + fit_vector = data_vector(batch_idx,:); + p1 = mahal(gm, fit_vector); + fit_vector(:,i_weight) = fit_vector(:,i_weight) .* weights(lowhigh, i_weight); + p2 = mahal(gm, fit_vector); + error('asdf') + + + i_gmm = fitgmdist(fit_vector, ... + starting_gmm.NumComponents, ... + 'Regularize', starting_gmm.RegularizationValue, ... + 'Replicates', 1, ... + 'CovarianceType', starting_gmm.CovarianceType, ... + 'SharedCovariance', starting_gmm.SharedCovariance, ... % covariance can vary between clusters + 'Start', starting_values, ... + 'Options', statset('UseParallel', 0, 'MaxIter', 1) ); % We just want to estimate the gradient, so only do one iteration + loss{lowhigh,i_weight} = i_gmm.NegativeLogLikelihood; + fprintf('for i_weight %d and lowhisgh %d loss is %f\n', i_weight, lowhigh, i_gmm.NegativeLogLikelihood); + end + end + + loss_v = 0.* weights; + for i_weight = 1:starting_gmm.NumVariables + for lowhigh = 1:2 + loss_v(lowhigh, i_weight) = loss{lowhigh, i_weight}; + end + end + gradient = -diff(loss_v) ./ diff(weights); + optimizer.update(gradient'); + + optimizer.get_current_parameters' + + + + end % i_batch + error('batch') +end +% Initialize an adam optimizer to fit the weights +% Loop over each epoch + % Loop over each batch, probably also 20% + % Calculate a finite difference gradient for each weight, timing could be problematic and will probably play into the batch size + % Probably need something to handle removing values where weights are pushed to zero (or so far outsize the average weight) + % Maybe this doesn't make a difference though if all + % Taking the existing mu/sigma and S = struct('mu',Mu,'Sigma',Sigma,'ComponentProportion',PComponents); + + % GMModel3 = fitgmdist(X,3,'Start',S); + + % Update the values with the adam optimizer + + % Calculate the score on the validation dataset + + +% end \ No newline at end of file diff --git a/testScripts/python/metaData/emc_parse_parameter_file.py b/testScripts/python/metaData/emc_parse_parameter_file.py new file mode 100644 index 00000000..3f2bd22b --- /dev/null +++ b/testScripts/python/metaData/emc_parse_parameter_file.py @@ -0,0 +1,212 @@ +import ast +import os +from emc_type_traits import EmcTypeTraits + +class EmcParameterFile: + def __init__(self, parameter_file_path): + self.parameter_file_path = parameter_file_path + self.type_checker = EmcTypeTraits() + self.string_keys = { + 'subtomometa', 'ali_mtype', 'cls_mtype', 'raw_mtype', 'fsc_mtype', + 'pca_distmeasure', 'kms_mtype', 'flgprecision', 'tmp_xcfscale', + 'fastscratchdisk', 'tmp_erasemasktype', 'startingdirection', 'peak_mtype', 'symmetry', + 'gmm_covariance_type', 'distance_metric' + } + self.emc_params = self._parse_parameter_file() + + def _parse_parameter_file(self): + emc_params = {} + try: + with open(self.parameter_file_path, 'r') as f: + lines = f.readlines() + except FileNotFoundError: + print(f"Error: Parameter file not found at {self.parameter_file_path}") + return None + + raw_lines = [] + for line_content in lines: + cleaned_line = line_content.split('%', 1)[0].strip() + if cleaned_line: + raw_lines.append(cleaned_line) + + last_parsed_parameter = 'none' + for line_content in raw_lines: + parts = line_content.split('=', 1) + if len(parts) == 2: + name = parts[0].strip() + value_str = parts[1].strip() + if not name: + print(f"Warning: Empty parameter name in line: '{line_content}'") + continue + if name.lower() in self.string_keys: + emc_params[name] = value_str + else: + try: + emc_params[name] = ast.literal_eval(value_str) + except (ValueError, SyntaxError): + print(f"Warning: Could not parse value for '{name}' ('{value_str}') as a standard Python literal. Storing as string.") + emc_params[name] = value_str + last_parsed_parameter = name + else: + print(f"Last successfully parsed parameter: {last_parsed_parameter}") + print(f"Error: Could not split Name=Value pair for line: '{line_content}'") + + # --- Assertions, Defaults, and Derived Parameters --- + # fastScratchDisk + if 'fastScratchDisk' in emc_params: + if str(emc_params['fastScratchDisk']).lower() == 'ram': + emc_cache_mem_str = os.getenv('EMC_CACHE_MEM') + if not emc_cache_mem_str: + print('Did not find environment variable EMC_CACHE_MEM. Skipping RAM disk for fastScratchDisk.') + emc_params['fastScratchDisk'] = '' + else: + try: + emc_cache_mem_val = float(emc_cache_mem_str) + if emc_cache_mem_val < 32: + print(f'EMC_CACHE_MEM ({emc_cache_mem_val}GB) is less than 32GB. Not using RAM disk.') + emc_params['fastScratchDisk'] = '' + else: + mcr_cache_root = os.getenv('MCR_CACHE_ROOT') + if mcr_cache_root: + emc_params['fastScratchDisk'] = mcr_cache_root + else: + print('MCR_CACHE_ROOT environment variable not set. Cannot use RAM disk.') + emc_params['fastScratchDisk'] = '' + except ValueError: + print(f"Warning: Could not convert EMC_CACHE_MEM ('{emc_cache_mem_str}') to float.") + emc_params['fastScratchDisk'] = '' + else: + emc_params['fastScratchDisk'] = '' + + # nGPUs + if 'nGPUs' in emc_params: + self.type_checker.assert_numeric(emc_params['nGPUs'], 1, [1, 1000], param_name='nGPUs') + else: + raise ValueError('nGPUs is a required parameter') + + # nCpuCores + if 'nCpuCores' in emc_params: + self.type_checker.assert_numeric(emc_params['nCpuCores'], 1, [1, 1000], param_name='nCpuCores') + else: + raise ValueError('nCpuCores is a required parameter') + + # symmetry + symmetry_has_been_checked = False + if 'symmetry' not in emc_params: + raise ValueError('You must now specify a symmetry=X parameter, where symmetry E (C1,C2..CX,O,I)') + # TODO: Add asserts on allowed values for symmetry parameter using type_checker.assert_string_value + symmetry_has_been_checked = True + + # PIXEL_SIZE + if 'PIXEL_SIZE' in emc_params: + self.type_checker.assert_numeric(emc_params['PIXEL_SIZE'], 1, [0, 100e-10], param_name='PIXEL_SIZE') + emc_params['pixel_size_si'] = emc_params['PIXEL_SIZE'] + emc_params['pixel_size_angstroms'] = emc_params['PIXEL_SIZE'] * 10**10 + else: + raise ValueError('PIXEL_SIZE is a required parameter') + + # Cs + if 'Cs' in emc_params: + self.type_checker.assert_numeric(emc_params['Cs'], 1, [0, 10e-3], param_name='Cs') + else: + raise ValueError('Cs is a required parameter') + + # VOLTAGE + if 'VOLTAGE' in emc_params: + self.type_checker.assert_numeric(emc_params['VOLTAGE'], 1, [20e3, 1000e3], param_name='VOLTAGE') + else: + raise ValueError('VOLTAGE is a required parameter') + + # AMPCONT + if 'AMPCONT' in emc_params: + self.type_checker.assert_numeric(emc_params['AMPCONT'], 1, [0.0, 1.0], param_name='AMPCONT') + if emc_params.get('Cs') == 0: + emc_params['Cs'] = 1e-10 + else: + raise ValueError('AMPCONT is a required parameter') + + # --- Optional Parameters & Defaults (examples) --- + if 'nPeaks' in emc_params: + self.type_checker.assert_numeric(emc_params['nPeaks'], 1, param_name='nPeaks') + else: + emc_params['nPeaks'] = 1 + + if 'CUTPADDING' in emc_params: + self.type_checker.assert_numeric(emc_params['CUTPADDING'], 1, param_name='CUTPADDING') + else: + emc_params['CUTPADDING'] = 20 + + # Deprecated substitutions examples + emc_params = self.type_checker.assert_deprecated_substitution(emc_params, 'ccc_cutoff', 'flgCCCcutoff') + if 'flgCCCcutoff' in emc_params: + self.type_checker.assert_numeric(emc_params['flgCCCcutoff'], 1, param_name='flgCCCcutoff') + else: + emc_params['flgCCCcutoff'] = 0.0 + + emc_params = self.type_checker.assert_deprecated_substitution(emc_params, 'projectVolumes', 'flgProjectVolumes') + if 'flgProjectVolumes' in emc_params: + self.type_checker.assert_boolean(emc_params['flgProjectVolumes'], param_name='flgProjectVolumes') + else: + emc_params['flgProjectVolumes'] = False + + emc_params = self.type_checker.assert_deprecated_substitution(emc_params, 'limit_to_one_core', 'flgLimitToOneProcess') + if 'flgLimitToOneProcess' in emc_params: + self.type_checker.assert_boolean(emc_params['flgLimitToOneProcess'], param_name='flgLimitToOneProcess') + else: + emc_params['flgLimitToOneProcess'] = False + + if emc_params.get('flgLimitToOneProcess') == True: + emc_params['nCpuCores'] = 1 + + if 'force_no_symmetry' in emc_params: + self.type_checker.assert_boolean(emc_params['force_no_symmetry'], param_name='force_no_symmetry') + if not symmetry_has_been_checked: + raise RuntimeError('force_no_symmetry must be after symmetry check') + if emc_params['force_no_symmetry']: + emc_params['symmetry'] = 'C1' + else: + emc_params['force_no_symmetry'] = True + if emc_params['force_no_symmetry'] and symmetry_has_been_checked: + emc_params['symmetry'] = 'C1' + + print("Info: Parameter parsing complete. Note that many assertions, defaults, and derived parameter calculations from the original MATLAB script may be simplified or pending full implementation in this Python version.") + return emc_params + + def get(self, key, default=None): + return self.emc_params.get(key, default) + + def as_dict(self): + return dict(self.emc_params) + +if __name__ == '__main__': + param_file = "param0.m" + if not os.path.exists(param_file) and not os.path.exists("../../../docs/exampleParametersAndRunScript/param0.m"): + print(f"Test file '{param_file}' not found. Creating a dummy one for demonstration.") + with open(param_file, "w") as f: + f.write("subTomoMeta=emClarity_tutorial % project name\n") + f.write("fastScratchDisk=/scratch/user\n") + f.write("nGPUs=2\n") + f.write("nCpuCores=12\n") + f.write("PIXEL_SIZE=2.0e-10\n") + f.write("Cs=2.7e-3\n") + f.write("VOLTAGE=300e3\n") + f.write("AMPCONT=0.07\n") + f.write("symmetry=C1\n") + f.write("particleRadius=[100,100,100]\n") + f.write("someBooleanFlag=True\n") + f.write("anotherBoolean=0\n") + f.write("limit_to_one_core=false\n") + f.write("force_no_symmetry=false\n") + elif not os.path.exists(param_file) and os.path.exists("../../../docs/exampleParametersAndRunScript/param0.m"): + param_file = "../../../docs/exampleParametersAndRunScript/param0.m" + + + print(f"Parsing '{param_file}'...") + emc_file = EmcParameterFile(param_file) + parameters = emc_file.as_dict() + + if parameters: + print("\n--- Parsed Parameters ---") + for key, value in parameters.items(): + print(f"{key}: {value} (type: {type(value).__name__})") + print("--- End of Parameters ---") diff --git a/testScripts/python/metaData/emc_type_traits.py b/testScripts/python/metaData/emc_type_traits.py new file mode 100644 index 00000000..f868d8c3 --- /dev/null +++ b/testScripts/python/metaData/emc_type_traits.py @@ -0,0 +1,59 @@ +class EmcTypeTraits: + def assert_numeric(self, value, expected_length=None, value_range=None, param_name="Parameter"): + """Checks if a value is numeric, optionally checks length and range.""" + is_numeric = isinstance(value, (int, float)) + is_list_of_numeric = isinstance(value, list) and all(isinstance(x, (int, float)) for x in value) + + if not (is_numeric or is_list_of_numeric): + raise ValueError(f"{param_name} is not numeric: {value}") + + if expected_length is not None: + current_length = 1 if is_numeric else len(value) + if current_length != expected_length: + raise ValueError(f"{param_name} '{value}' has length {current_length}, expected {expected_length}") + + if value_range is not None: + values_to_check = value if isinstance(value, list) else [value] + for v_item in values_to_check: + if not (value_range[0] <= v_item <= value_range[1]): + raise ValueError(f"{param_name} value {v_item} out of range {value_range}") + return True + + def assert_boolean(self, value, param_name="Parameter"): + """Checks if a value is boolean-like (True, False, 0, 1).""" + if not isinstance(value, (bool, int, float)) or (isinstance(value, (int, float)) and value not in (0, 1)): + # ast.literal_eval might turn "0" into int 0, "1" into int 1. + # True/False are bool. + # Allow 0/1 as bool-like after ast.literal_eval + # If strict bool is needed, this check would be: if not isinstance(value, bool): + pass # Current logic allows 0/1 + # raise ValueError(f"{param_name} is not boolean-like: {value}") + return True + + def assert_string_value(self, value, allowed_values, case_sensitive=False, param_name="Parameter"): + """Checks if a string value is one of the allowed values.""" + val_to_check = value if case_sensitive else str(value).lower() + allowed_to_check = allowed_values if case_sensitive else [str(x).lower() for x in allowed_values] + if val_to_check not in allowed_to_check: + raise ValueError(f"{param_name} value '{value}' not in allowed set: {allowed_values}") + return True + + def assert_deprecated_substitution(self, params, old_name, new_name): + """Handles deprecated parameter names, substituting with new names if appropriate.""" + if old_name in params: + old_value = params.pop(old_name) # Remove old_name first + if new_name not in params or params.get(new_name) is None: + print(f"Info: Parameter '{old_name}' is deprecated. Using its value for '{new_name}'.") + params[new_name] = old_value + # If new_name exists and has a value, MATLAB code prefers old_value if new_value is None. + # The original python translation was: + # elif params.get(new_name) is None and params.get(old_name) is not None : + # This condition is tricky after popping old_name. + # Let's simplify: if new_name is already there with a non-None value, we keep it. + # Otherwise, the value from old_name is used. + # The MATLAB code implies if new_name is present (even if None), old_name's value might be preferred. + # The current logic: if new_name is not in params OR new_name is in params but its value is None, use old_value. + # If new_name is in params and has a non-None value, the old_value is effectively ignored (already popped). + elif params.get(new_name) is not None: + print(f"Info: Parameter '{old_name}' is deprecated. '{new_name}' already exists with value '{params[new_name]}'. Value from '{old_name}' ('{old_value}') is ignored.") + return params \ No newline at end of file diff --git a/testScripts/python/metaData/test_emc_parse_parameter_file.py b/testScripts/python/metaData/test_emc_parse_parameter_file.py new file mode 100644 index 00000000..9365060c --- /dev/null +++ b/testScripts/python/metaData/test_emc_parse_parameter_file.py @@ -0,0 +1,48 @@ +import unittest +import os +from emc_parse_parameter_file import EmcParameterFile + +class TestEmcParameterFile(unittest.TestCase): + def setUp(self): + # Create a temporary parameter file for testing + self.test_file = "test_param.m" + with open(self.test_file, "w") as f: + f.write("subTomoMeta=emClarity_tutorial\n") + f.write("fastScratchDisk=/scratch/test\n") + f.write("nGPUs=2\n") + f.write("nCpuCores=8\n") + f.write("PIXEL_SIZE=2.0e-10\n") + f.write("Cs=2.7e-3\n") + f.write("VOLTAGE=300e3\n") + f.write("AMPCONT=0.07\n") + f.write("symmetry=C1\n") + f.write("CUTPADDING=15\n") + f.write("nPeaks=3\n") + + def tearDown(self): + if os.path.exists(self.test_file): + os.remove(self.test_file) + + def test_parse_and_access(self): + emc_file = EmcParameterFile(self.test_file) + params = emc_file.as_dict() + self.assertEqual(params['subTomoMeta'], "emClarity_tutorial") + self.assertEqual(params['nGPUs'], 2) + self.assertEqual(params['nCpuCores'], 8) + self.assertEqual(params['PIXEL_SIZE'], 2.0e-10) + self.assertEqual(params['symmetry'], "C1") + self.assertEqual(params['CUTPADDING'], 15) + self.assertEqual(params['nPeaks'], 3) + # Test default + self.assertEqual(params['flgCCCcutoff'], 0.0) + self.assertEqual(params['flgProjectVolumes'], False) + + def test_missing_required(self): + with open("bad_param.m", "w") as f: + f.write("nGPUs=2\n") + with self.assertRaises(ValueError): + EmcParameterFile("bad_param.m") + os.remove("bad_param.m") + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/testScripts/python/metaData/test_emc_type_traits.py b/testScripts/python/metaData/test_emc_type_traits.py new file mode 100644 index 00000000..a56fc0d4 --- /dev/null +++ b/testScripts/python/metaData/test_emc_type_traits.py @@ -0,0 +1,52 @@ +import unittest +from emc_type_traits import EmcTypeTraits + +class TestEmcTypeTraits(unittest.TestCase): + def setUp(self): + self.traits = EmcTypeTraits() + + def test_assert_numeric_scalar(self): + self.assertTrue(self.traits.assert_numeric(5)) + self.assertTrue(self.traits.assert_numeric(3.14)) + with self.assertRaises(ValueError): + self.traits.assert_numeric("not_a_number") + + def test_assert_numeric_list(self): + self.assertTrue(self.traits.assert_numeric([1, 2, 3])) + with self.assertRaises(ValueError): + self.traits.assert_numeric([1, "bad", 3]) + + def test_assert_numeric_length(self): + self.assertTrue(self.traits.assert_numeric([1, 2], expected_length=2)) + with self.assertRaises(ValueError): + self.traits.assert_numeric([1, 2], expected_length=3) + + def test_assert_numeric_range(self): + self.assertTrue(self.traits.assert_numeric(5, value_range=[0, 10])) + with self.assertRaises(ValueError): + self.traits.assert_numeric(15, value_range=[0, 10]) + + def test_assert_boolean(self): + self.assertTrue(self.traits.assert_boolean(True)) + self.assertTrue(self.traits.assert_boolean(False)) + self.assertTrue(self.traits.assert_boolean(1)) + self.assertTrue(self.traits.assert_boolean(0)) + # Should not raise, but if you want strict bool, uncomment below + # with self.assertRaises(ValueError): + # self.traits.assert_boolean("not_bool") + + def test_assert_string_value(self): + self.assertTrue(self.traits.assert_string_value("A", ["A", "B"])) + self.assertTrue(self.traits.assert_string_value("a", ["A", "B"])) + with self.assertRaises(ValueError): + self.traits.assert_string_value("C", ["A", "B"]) + + def test_assert_deprecated_substitution(self): + params = {"old": 1} + result = self.traits.assert_deprecated_substitution(params, "old", "new") + self.assertIn("new", result) + self.assertNotIn("old", result) + self.assertEqual(result["new"], 1) + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/testScripts/test_adamOptimizer.m b/testScripts/test_adamOptimizer.m new file mode 100644 index 00000000..675d9a18 --- /dev/null +++ b/testScripts/test_adamOptimizer.m @@ -0,0 +1,96 @@ +% in this script fit a linear polynomial to a set of data points using the adamOptimizer class +% First, we will create some sample data points +% Sample data points +x = linspace(0, 10, 100)'; % 100 data points from 0 to 10 +y_true = 2 * x + 3; % True linear relationship (y = 2x + 3) +noise = randn(size(x)); % Add some noise to the data +y = y_true + 1.5 * noise; % Noisy observations + +% plot the data points +figure; +scatter(x, y, 'b.'); +%show the figure +title('Sample Data Points'); +xlabel('x'); +ylabel('y'); +grid on; + +% Initialize parameters for the linear model (y = mx + b) +initial_parameters = [0; 0]; % Start with m=0, b=0 (slope and intercept) +% Create an instance of the adamOptimizer +optimizer = adamOptimizer(initial_parameters); + +% Define the number of iterations for optimization +num_iterations = 10000; +loss = zeros(num_iterations, 1); % Preallocate loss array for optional debugging +num_complete_iterations = 0; % Initialize the number of complete iterations +% Perform the optimization +for i = 1:num_iterations + % Compute the gradient of the loss + [ gradient, loss_i ] = compute_gradient(x, y, optimizer.get_current_parameters); + loss(i) = loss_i; % Store the loss for debugging purposes (optional) + num_complete_iterations = num_complete_iterations + 1; % Increment the count of complete iterations + % If iteration > 5 compare current loss to previous loss to check for convergence + if i > 5 && abs(loss(i) - loss(i-1)) < 1e-6 + fprintf('Converged after %d iterations.\n', i); + break; % Stop early if the loss has converged + end + % Update the optimizer with the computed gradient + optimizer.update(gradient); +end +% After optimization, retrieve the optimized parameters +optimized_parameters = optimizer.get_current_parameters; +% Display the optimized parameters +fprintf('Optimized parameters:\n'); +fprintf('Slope (m): %.4f\n', optimized_parameters(1)); +fprintf('Intercept (b): %.4f\n', optimized_parameters(2)); +% Plot the results +figure; +scatter(x, y, 'b.', 'DisplayName', 'Data Points'); % Original data points +hold on; +% Plot the fitted line using the optimized parameters +y_pred = optimized_parameters(1) * x + optimized_parameters(2); % Compute the predicted y values +plot(x, y_pred, 'r-', 'LineWidth', 2, 'DisplayName', 'Fitted Line'); % Fitted line +title('Fitted Linear Model using Adam Optimizer'); +xlabel('x'); +ylabel('y'); +legend('show'); +% Show the figure +hold off; + +% Trim the loss to the number of iterations completed (in case of early stopping) +% plot the loss over iterations (optional) +figure; +plot(1:num_complete_iterations, loss(1:num_complete_iterations), 'b-', 'LineWidth', 2); +title('Loss over Iterations'); +xlabel('Iteration'); +ylabel('Mean Squared Error (Loss)'); +grid on; +% This script fits a linear model to the noisy data using the Adam optimizer +% and plots the fitted line along with the original data points. The loss is also plotted to show the convergence. + + +% Define a function to compute the gradient of the loss (mean squared error) +function [gradient, loss] = compute_gradient(x, y, parameters) + % Extract parameters + m = parameters(1); + b = parameters(2); + + % Compute predictions + y_pred = m * x + b; + + % Compute the error + error = y_pred - y; + + % Compute the loss (mean squared error) if needed + loss = mean(error .^ 2); % Mean Squared Error (optional, can be used for debugging) + + % Compute the gradient of the loss with respect to m and b + grad_m = (2 / length(x)) * sum(error .* x); + grad_b = (2 / length(x)) * sum(error); + + % Combine gradients into a single vector + gradient = [grad_m; grad_b]; +end + + diff --git a/transformations/BH_average3d.m b/transformations/BH_average3d.m index 368d571e..c3d623ad 100755 --- a/transformations/BH_average3d.m +++ b/transformations/BH_average3d.m @@ -38,6 +38,11 @@ emc.flgQualityWeight = 0; end +if (emc.flgQualityWeight <=0 && emc.ccc_cutoff > 0.0) + fprintf('\n\t\tWARNING: Quality weighting is on, but the cutoff is set to %2.2f, so no volumes will be used. Overriding ccc_cutoff\n\n',emc.ccc_cutoff); + emc.ccc_cutoff = 0; +end + if (emc.projectVolumes && ~emc.flgCutOutVolumes) emc.flgCutOutVolumes = true; end @@ -54,9 +59,6 @@ end rotConvention = 'Bah'; -if ( emc.doHelical ) - rotConvention = 'Helical'; -end % The weights are only re-estimated for an out of plane search. Until this % happens, they are not valid. @@ -70,6 +72,8 @@ end end + + %%% For general release, I've disabled class average alignment and %%% multi-reference alignment, so set the default to OFF. If either of %%% these features are re-introduced, this will need to be reverted. @@ -126,6 +130,9 @@ className = emc.(sprintf('%s_className','Raw')); samplingRate = emc.('Ali_samplingRate'); + if (~emc.multi_reference_alignment && max(size(classVector{1},2),size(classVector{2},2)) > 1) + error('You have more than one class, but multi-reference alignment is not enabled'); + end if (emc.multi_reference_alignment && (test_multi_ref_diffmap || ~emc.classification)) fprintf('\n\nMutliRef enabled.\n'); pause(2); @@ -147,10 +154,10 @@ % Goal is to re-extract odd-half, applying the xform found in fscGold fieldPrefix = 'Ref'; savePrefix = 'Ref'; - classVector{1} = emc.(sprintf('%s_classes_odd',fieldPrefix)); - classVector{2} = emc.(sprintf('%s_classes_eve',fieldPrefix)); + classVector{1} = emc.(sprintf('%s_classes_odd','Raw')); + classVector{2} = emc.(sprintf('%s_classes_eve','Raw')); - className = emc.(sprintf('%s_className',fieldPrefix)); + className = emc.(sprintf('%s_className','Raw')); samplingRate = emc.('Ali_samplingRate'); flgFinalAvg = 1; @@ -244,6 +251,16 @@ doNotTrim = false; eachTomo = false; flgEstSNR = 0; +% see if we previously had weights from a GMM, this will be overwritten if we are in Cluster_cls stage. +if isfield(subTomoMeta.(cycleRead), 'class_weights') + use_class_weights = subTomoMeta.(cycleRead).class_weights; +else + use_class_weights = false; +end + +fprintf('\n\n\tUsing class weights: %d\n\n', use_class_weights); +pause(3); + switch STAGEofALIGNMENT case 'RawAlignment' if ( CYCLE ) @@ -257,7 +274,17 @@ end case 'FinalAlignment' - geometry = subTomoMeta.(cycleRead).Avg_geometry; + if emc.multi_reference_alignment + if (emc.classification) + geometry = subTomoMeta.(cycleNumber).('ClusterClsGeom') ; + else + geometry = subTomoMeta.(cycleNumber).('ClusterRefGeom') ; + end + else + geometry = subTomoMeta.(cycleRead).Avg_geometry; + + end + if ~(emc.classification) doNotTrim = true; end @@ -282,7 +309,13 @@ geometry = subTomoMeta.(cycleRead).ClusterResults.(cN{1}); end - + + % If we are averaging from new classes, check to see iff we have probability weights form a GMM + if isfield(subTomoMeta.(cycleRead).ClusterResults, 'class_weights') + use_class_weights = subTomoMeta.(cycleRead).ClusterResults.class_weights; + fprintf('\n\n\tUsing class weights from new class assignments: %d\n\n', use_class_weights); + pause(3); + end subTomoMeta.(cycleRead).('KmsSampling') = samplingRate; doNotTrim = true; @@ -371,8 +404,8 @@ % Assuming the mask/window sizes haven't changed since the original average % that was used to calc the FSC -- that would break things here. - imgNAME = sprintf('class_%d_Locations_REF_%s', className, 'EVE'); - weightNAME = sprintf('class_%d_Locations_REF_%s_Wgt', className, 'EVE'); + imgNAME = sprintf('class_%d_Locations_Ref_%s', className, 'EVE'); + weightNAME = sprintf('class_%d_Locations_Ref_%s_Wgt', className, 'EVE'); [ refIMG ] = BH_unStackMontage4d(1, ... subTomoMeta.(cycleNumber).(imgNAME){1}, ... @@ -540,6 +573,7 @@ for iParProc = 1:nParProcesses for iTomo = iterList{iParProc} if (emc.track_stats) + error('The weight in column two is now being taken as a class probability assignent from clustering, this branch is broken.'); geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks) = geometry.(tomoList{iTomo})(:,1:26:26*emc.nPeaks)./geometry.(tomoList{iTomo})(:,2:26:26*emc.nPeaks); end @@ -648,7 +682,7 @@ nIgnored = 0; nSubTomosTotal = 0; avgVolume_tmp = cell(maxClasses,2); - avgWedge_tmp = cell(maxClasses,2); + avgWedge_tmp = cell(maxClasses,4); geometry_tmp = geometry; @@ -656,8 +690,10 @@ for iRow = 1:maxClasses for iCol = 1:2 avgVolume_tmp{iRow, iCol} = zeros(sizeMask, 'single'); + avgWedge_tmp{iRow, iCol}= zeros(sizeCalc, 'single'); + end + for iCol = 3:4 avgWedge_tmp{iRow, iCol}= zeros(sizeCalc, 'single'); - end end @@ -831,6 +867,14 @@ else peakWgt = 1; end + + if (use_class_weights) + EMC_assert_numeric(positionList(iSubTomo,2),1,[0.0,1.0]) + if (positionList(iSubTomo,2) < 0.98) + fprintf('Reweighting subtomo %d with class weight %2.2f\n',iSubTomo,positionList(iSubTomo,2).^2); + end + peakWgt = peakWgt .* positionList(iSubTomo,2).^2; + end % % if (emc.spike_prior) % % peakWgt = peakWgt.*spike_info.(tomoList{iTomo}).('angular_prob')(iSubTomo,1).*spike_info.('normalization_factor'); @@ -1006,7 +1050,7 @@ [~, iWedgeMask] = interpolator(gpuArray(iSF3D), angles, [0,0,0], rotConvention , 'inv', emc.symmetry, true); - + % Scaling prior to applying the quality weight which will result in less total power for downweighted volumes iParticle = iParticle - mean(iParticle(interpMask_tmpBinary)); iParticle = iParticle ./ rms(iParticle(interpMask_tmpBinary)); iParticle = iParticle .* interpMask_tmp; @@ -1032,6 +1076,7 @@ % Flag the particle as ignored positionList(iSubTomo, 26:26:emc.nPeaks*26) = -9999; else + if (test_fuzz) % Find the right row in the weight array @@ -1051,9 +1096,11 @@ for iWeight = 1:length(iProb) avgVolume_tmp{iWeight, positionList(iSubTomo,7)} = ... avgVolume_tmp{iWeight, positionList(iSubTomo,7)} + (iParticle .* iProb(iWeight)); - + % FIXME: for storing sum of CTFS (these are just sum of) avgWedge_tmp{ iWeight, positionList(iSubTomo,7)} = ... avgWedge_tmp{ iWeight, positionList(iSubTomo,7)} + (iWedgeMask .* iProb(iWeight)); + + nExtracted_tmp(iWeight, positionList(iSubTomo,7)) = ... nExtracted_tmp(iWeight, positionList(iSubTomo,7)) + iProb(iWeight); end @@ -1118,7 +1165,8 @@ avgVolume_tmp{ iSnr, 1} = avgVolume_tmp{ iSnr, 1} + ... gather(iTempParticleODD); avgVolume_tmp{ iSnr, 2} = avgVolume_tmp{ iSnr, 2} + ... - gather(iTempParticleEVE); + gather(iTempParticleEVE); + % FIXME: for storing sum of CTFS (these are just sum of) avgWedge_tmp{ iSnr, 1} = avgWedge_tmp{ iSnr, 1} + ... gather(iTempWedgeODD); @@ -1136,6 +1184,10 @@ gather(iTempWedgeODD); avgWedge_tmp{ iClassPos, 2} = avgWedge_tmp{ iClassPos, 2} + ... gather(iTempWedgeEVE); + avgWedge_tmp{ iClassPos, 3} = avgWedge_tmp{ iClassPos, 3} + ... + gather(sqrt(iTempWedgeODD)); + avgWedge_tmp{ iClassPos, 4} = avgWedge_tmp{ iClassPos, 4} + ... + gather(sqrt(iTempWedgeEVE)); end end end % end of loop over classes @@ -1172,7 +1224,8 @@ nExtracted = zeros(maxClasses,2); avgVolume = cell(maxClasses,2); -avgWedge = cell(maxClasses,4); +avgSF3D = cell(maxClasses,2); +avgSF3D_Sq = cell(maxClasses,2); nSubTomosTotal = 0; nIgnored = 0; @@ -1187,13 +1240,15 @@ for iHalf = 1:2-flgFinalAvg if iParProc == 1 avgVolume{iVol, iHalf} = avgResults{iParProc}{iVol, iHalf}; - avgWedge{iVol, iHalf} = wgtResults{iParProc}{iVol, iHalf}; - + avgSF3D_Sq{iVol, iHalf} = wgtResults{iParProc}{iVol, iHalf}; + avgSF3D{iVol, iHalf} = wgtResults{iParProc}{iVol, iHalf+2}; else avgVolume{iVol, iHalf} = avgVolume{iVol, iHalf} + ... avgResults{iParProc}{iVol, iHalf}; - avgWedge{iVol, iHalf} = avgWedge{iVol, iHalf} + ... + avgSF3D_Sq{iVol, iHalf} = avgSF3D_Sq{iVol, iHalf} + ... wgtResults{iParProc}{iVol, iHalf}; + avgSF3D{iVol, iHalf} = avgSF3D{iVol, iHalf} + ... + wgtResults{iParProc}{iVol, iHalf+2}; end end @@ -1251,11 +1306,14 @@ classSum = cell(2,1); classWgtSum = cell(2,1); +classWgtSumSqrt = cell(2,1); if (saveClassSum > -1) classSum{1} = zeros(size(avgVolume{1,1}),'single'); classSum{2} = zeros(size(avgVolume{1,2}),'single'); - classWgtSum{1} = zeros(size(avgWedge{1,1}),'single'); - classWgtSum{2} = zeros(size(avgWedge{1,2}),'single'); + classWgtSum{1} = zeros(size(avgSF3D_Sq{1,1}),'single'); + classWgtSum{2} = zeros(size(avgSF3D_Sq{1,2}),'single'); + classWgtSumSqrt{1} = zeros(size(avgSF3D_Sq{1,3}),'single'); + classWgtSumSqrt{2} = zeros(size(avgSF3D_Sq{1,4}),'single'); end for iClassPos = 1:maxClasses @@ -1294,7 +1352,8 @@ if (saveClassSum > -1) fprintf('adding class %d to %d\n',iClassPos,iGold); classSum{iGold} = classSum{iGold} + classStorage{iClassPos,iGold}; - classWgtSum{iGold} = classWgtSum{iGold} + avgWedge{iClassPos,iGold}; + classWgtSum{iGold} = classWgtSum{iGold} + avgSF3D_Sq{iClassPos,iGold}; + classWgtSumSqrt{iGold} = classWgtSumSqrt{iGold} + avgSF3D{iClassPos,iGold}; end end @@ -1314,7 +1373,7 @@ [classListOut, geometry] = reorder_classes(avgVolume(:,1),avgVolume(:,2),maxClasses, geometry); avgVolume(:,2) = avgVolume(classListOut(:,2), 2); - avgWedge(:,2) = avgWedge(classListOut(:,2), 2); + avgSF3D_Sq(:,2) = avgSF3D_Sq(classListOut(:,2), 2); classMatches = fopen(sprintf('%s_class%d_%s_matches.txt', ... outputPrefix,className,fieldPrefix),'w'); @@ -1367,12 +1426,22 @@ end SAVE_IMG(montOUT, imout, emc.pixel_size_angstroms); %%%%%%%% - [montOUT, imgLocations] = BH_montage4d(avgWedge(:,iGold), ''); + [montOUT, imgLocations] = BH_montage4d(avgSF3D_Sq(:,iGold), ''); imout = sprintf('%s_class%d_%s_%s_Wgt.mrc',outputPrefix, ... className, fieldPrefix, halfSet); classOut = sprintf('class_%d_Locations_%s_%s_Wgt', className,fieldPrefix, halfSet); + % For the weight, instead of imgCounts save the padValues + subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,fscPAD}; + SAVE_IMG(montOUT, imout, emc.pixel_size_angstroms); + + [montOUT, imgLocations] = BH_montage4d(avgSF3D(:,iGold), ''); + + imout = sprintf('%s_class%d_%s_%s_WgtSqrt.mrc',outputPrefix, ... + className, fieldPrefix, halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_WgtSqrt', className,fieldPrefix, halfSet); + % For the weight, instead of imgCounts save the padValues subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,fscPAD}; @@ -1400,6 +1469,14 @@ classOut = sprintf('class_%d_Locations_%s_%s_Wgt', saveClassSum, savePrefix, halfSet); SAVE_IMG(montOUT, imout,emc.pixel_size_angstroms); subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; + + [montOUT, imgLocations] = BH_montage4d(classWgtSumSqrt(iGold), ''); + + imout = sprintf('%s_class%d_%s_%s_WgtSqrt.mrc',outputPrefix, ... + saveClassSum, savePrefix, halfSet); + classOut = sprintf('class_%d_Locations_%s_%s_WgtSqrt', saveClassSum, savePrefix, halfSet); + SAVE_IMG(montOUT, imout,emc.pixel_size_angstroms); + subTomoMeta.(cycleNumber).(classOut) = {imout,imgLocations,imgCounts}; end end @@ -1414,6 +1491,7 @@ subTomoMeta.(cycleNumber).('SymmetryApplied').(STAGEofALIGNMENT) = emc.symmetry; subTomoMeta.(cycleNumber).('ClassVector').(STAGEofALIGNMENT) = classVector; +subTomoMeta.(cycleNumber).('class_weights') = use_class_weights; cycleNumber = gather(cycleNumber); subTomoMeta.('currentCycle') = gather(CYCLE); @@ -1450,6 +1528,9 @@ +refIMG = cell(2,1); +refWGT = cell(2,1); +refWGT_SQRT = cell(2,1); if ~( flgEstSNR ) load(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); @@ -1465,6 +1546,8 @@ className, fieldPrefix, halfSet); wgtIN = sprintf('class_%d_Locations_%s_%s_Wgt', ... className, fieldPrefix, halfSet); + wgtSqrtIN = sprintf('class_%d_Locations_%s_%s_WgtSqrt', ... + className, fieldPrefix, halfSet); [ refIMG{iGold} ] = BH_unStackMontage4d(1:maxClasses, ... subTomoMeta.(cycleNumber).(imgIN){1}, ... @@ -1475,6 +1558,11 @@ subTomoMeta.(cycleNumber).(wgtIN){1},... subTomoMeta.(cycleNumber).(wgtIN){2},... sizeCalc); + + [ refWGT_SQRT{iGold} ] = BH_unStackMontage4d(1:maxClasses, ... + subTomoMeta.(cycleNumber).(wgtSqrtIN){1},... + subTomoMeta.(cycleNumber).(wgtSqrtIN){2},... + sizeCalc); end @@ -1545,6 +1633,7 @@ refTMP = gather(BH_multi_cRef_Vnorm(fscParams, aliParams, mskParams,... {refIMG{1}{iOdd},refIMG{2}{iEve}}, ... {refWGT{1}{iOdd},refWGT{2}{iEve}}, ... + {refWGT_SQRT{1}{iOdd},refWGT_SQRT{2}{iEve}}, ... flgCombine,flgRefCutOff, emc.pixel_size_angstroms, bFactorSend)); if ~(flgFinalAvg) diff --git a/transformations/BH_montage4d_halfGrid.m b/transformations/BH_montage4d_halfGrid.m index 85781498..52f33450 100644 --- a/transformations/BH_montage4d_halfGrid.m +++ b/transformations/BH_montage4d_halfGrid.m @@ -39,7 +39,7 @@ if ischar(IMAGES{1}) - img1 = OPEN_IMG('single', IMAGES{1)); + img1 = OPEN_IMG('single', IMAGES{1}); else img1 = IMAGES{1}; end From 335a3b7be998fc8da18c15c77f964aaa7d0710c0 Mon Sep 17 00:00:00 2001 From: himesb Date: Tue, 26 Aug 2025 10:02:03 -0400 Subject: [PATCH 094/151] feat: Implement comprehensive tilt-series asset management and alignment system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Major Features Added: • Tilt-Series Asset Management (tilt_series_assets.py) - Group-based asset organization with hierarchical tree display - IMOD tool integration (header, extracttilts, imod) for validation - Multi-selection with copy/paste functionality using composite keys - Real-time validation status with visual indicators (✓ ✗ ⚠) - Collapsible groups and responsive UI layouts • Enhanced AutoAlign Workflow (autoalign_widget.py) - Group-based alignment processing with asset validation - Python multiprocessing for parallel execution - Progress tracking with shared memory and queue communication - Selection preservation and comprehensive error handling • Project-Aware State Management - SQLite persistence with composite keys for data integrity - Project notification system for coordinated tab updates - Lazy loading to prevent premature asset access • IMOD Integration Framework - Subprocess management for external tool execution - Background validation without UI blocking - Consistent tool usage with command-line workflows Technical Improvements: • Fixed copy operations creating new entries instead of moving • Implemented composite database keys (group::asset_name) • Added validation status synchronization across components • Enhanced UI responsiveness with background processing • Improved column widths and layout optimization Files Modified: 28 GUI files, ~6000 lines added Core Components: state_manager.py, main.py, tilt_series_assets.py, autoalign_widget.py Documentation: Updated GUI_IMPLEMENTATION_SUMMARY.md, added workflow_analysis.md This transforms the GUI from a minimal framework into a production-ready tilt-series workflow management system with comprehensive IMOD integration." --- .github/copilot-instructions.md | 105 +++ .gitignore | 5 + GUI_IMPLEMENTATION_SUMMARY.md | 184 ++++ README_GUI.md | 84 ++ alignment/BH_alignRaw3d_v2.m | 3 +- alignment/BH_templateSearch3d_2.m | 3 +- ctf/BH_ctf_Correct3d.m | 11 +- ctf/BH_ctf_Updatefft.m | 2 +- docs/gen_param.m | 275 ++++++ emClarity_Tutorial.md | 580 ++++++++++++ gui/PARAMETER_GUIDE.md | 165 ++++ gui/README.md | 134 +++ gui/__init__.py | 14 + gui/autoalign_widget.py | 957 +++++++++++++++++++ gui/commands.py | 426 +++++++++ gui/commands_new.py | 426 +++++++++ gui/config.py | 190 ++++ gui/emclarity-gui.desktop | 10 + gui/emclarity_gui_state.db | Bin 0 -> 94208 bytes gui/launcher.py | 47 + gui/main.py | 896 ++++++++++++++++++ gui/parameter_config.json | 419 +++++++++ gui/parameter_loader.py | 185 ++++ gui/parameters.py | 85 ++ gui/parameters_new.py | 85 ++ gui/profile_widgets.py | 568 ++++++++++++ gui/project_dialog.py | 86 ++ gui/run_gui.sh | 34 + gui/state_manager.py | 464 ++++++++++ gui/temp_methods.py | 29 + gui/test_collapsible.py | 44 + gui/test_components.py | 91 ++ gui/test_gui.py | 66 ++ gui/test_parameters.py | 120 +++ gui/tilt_series_assets.py | 1165 ++++++++++++++++++++++++ gui/widgets.py | 1019 +++++++++++++++++++++ gui/workflow.py | 311 +++++++ gui/workflow_widgets_new.py | 227 +++++ metaData/BH_geometryAnalysis.m | 177 ++-- metaData/BH_geometryInitialize.m | 2 +- metaData/BH_parseParameterFile.m | 23 +- metaData/BH_removeDuplicates.m | 2 +- metaData/BH_skipClassAlignment.m | 2 +- metaData/EMC_checkCacheForFile.m | 101 ++ metaData/EMC_setCacheForFile.m | 126 +++ statistics/BH_clusterPub.m | 2 +- statistics/BH_fscGold_class.m | 2 +- statistics/BH_fscSplit.m | 2 +- statistics/BH_pcaPub.m | 5 +- synthetic/BH_synthetic_mapBack.m | 2 +- testScripts/emClarity.m | 31 +- testScripts/mCompile.sh | 2 +- transformations/BH_average3d.m | 169 +++- transformations/BH_multi_loadOrBuild.m | 5 +- 54 files changed, 10045 insertions(+), 123 deletions(-) create mode 100644 .github/copilot-instructions.md create mode 100644 GUI_IMPLEMENTATION_SUMMARY.md create mode 100644 README_GUI.md create mode 100644 docs/gen_param.m create mode 100644 emClarity_Tutorial.md create mode 100644 gui/PARAMETER_GUIDE.md create mode 100644 gui/README.md create mode 100644 gui/__init__.py create mode 100644 gui/autoalign_widget.py create mode 100644 gui/commands.py create mode 100644 gui/commands_new.py create mode 100644 gui/config.py create mode 100644 gui/emclarity-gui.desktop create mode 100644 gui/emclarity_gui_state.db create mode 100644 gui/launcher.py create mode 100644 gui/main.py create mode 100644 gui/parameter_config.json create mode 100644 gui/parameter_loader.py create mode 100644 gui/parameters.py create mode 100644 gui/parameters_new.py create mode 100644 gui/profile_widgets.py create mode 100644 gui/project_dialog.py create mode 100755 gui/run_gui.sh create mode 100644 gui/state_manager.py create mode 100644 gui/temp_methods.py create mode 100644 gui/test_collapsible.py create mode 100644 gui/test_components.py create mode 100644 gui/test_gui.py create mode 100644 gui/test_parameters.py create mode 100644 gui/tilt_series_assets.py create mode 100644 gui/widgets.py create mode 100644 gui/workflow.py create mode 100644 gui/workflow_widgets_new.py create mode 100644 metaData/EMC_checkCacheForFile.m create mode 100644 metaData/EMC_setCacheForFile.m diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md new file mode 100644 index 00000000..145eaf75 --- /dev/null +++ b/.github/copilot-instructions.md @@ -0,0 +1,105 @@ +# Copilot Rules + +## Project specific goals + +- emClarity is an application written in matlab and also uses mex and mexCuda for high-performance computing tasks, particularly in the field of cryo-electron microscopy (cryo-EM). +- The original application is entirely command line driven, and our goal is to build a simple Pyside6 GUI to facilitate user interaction with the underlying functionality. +- The GUI should provide a user-friendly interface for configuring and running cryo-EM data processing workflows. +- As we develop, we want to clean up and simplify code as well as adding tests to ensure functionality and prevent regressions. + +## Copilot Behavior + +- Copilot should activate and within virtual environment when working with python. +- Copilot should provide concise and relevant code suggestions. +- Copilot should avoid suggesting large blocks of code without context. +- Copilot should prioritize user intent and project context in its suggestions. + +## Copilot code preferences + +- Copilot should generate code that is idiomatic to the programming language being used. +- Copilot should prefer built-in language features and standard libraries over external dependencies. +- Copilot should aim for simplicity and clarity in its code suggestions. +- Copilot should never hard-code variables and instead place them in a relevant configuration file or environment variable. +- Copilot should strive for consistency in naming conventions and code style, and use descriptive names for variables and functions. +- Copilot should not allow default values or other design patterns that could lead to ambiguity or confusion. +- Copilot should prefer to fail fast and descriptively. + + +## Development Environment Notes + +- **Project Location**: `/sa_shared/git/emClarity/` +- **Virtual Environment**: `.venv/` in project root +- **GUI Location**: `gui/` subdirectory +- **Qt Platform**: Use `QT_QPA_PLATFORM=xcb` for stability +- **Branch**: `ctf3d_work` + +## emClarity GUI Development - Key Learnings & Best Practices + +*Generated from GUI development session on August 26, 2025* + +### Key Learnings for Future GUI Development Sessions + +#### 1. **Virtual Environment Context Management** + +**Problem**: Frequently forgot to activate the virtual environment when testing Python imports or running the GUI, leading to import errors and wasted debugging time. + +**Solution Pattern**: +```bash + +# Always use this pattern for Python testing in emClarity +cd /sa_shared/git/emClarity && source .venv/bin/activate && cd gui && python -c "..." + +# For GUI launches +cd /sa_shared/git/emClarity && source .venv/bin/activate && cd gui && QT_QPA_PLATFORM=xcb python main.py & +``` + +**Impact**: This pattern eliminates 80% of "import not found" errors and ensures consistent testing environment. + +--- + +#### 2. **Incremental GUI Testing with State Cleanup** + +**Problem**: Making multiple changes before testing led to complex debugging when things broke. Also, GUI processes would accumulate without proper cleanup. + +**Solution Pattern**: +```bash + +# Always kill existing GUI processes before launching new ones +pkill -f "python main.py" + +# Then launch fresh instance +cd /sa_shared/git/emClarity && source .venv/bin/activate && cd gui && QT_QPA_PLATFORM=xcb python main.py & +``` + +**Impact**: This approach caught errors early (e.g., the toggle_keep_on_top parameter issue, import errors) and prevented GUI state conflicts. + +--- + +#### 3. **File Context Awareness for Complex Edits** + +**Problem**: When making large-scale changes (like the parameter system rewrite), I sometimes lost track of file state and made edits that corrupted files or created inconsistencies. + +**Solution Pattern**: +```python + +# Before major file restructuring, always read current state +read_file(file_path, start_line=1, end_line=50) # Check current structure + +# For complex replacements, verify the exact context +grep_search(pattern, include_pattern=file_path) # Find exact locations + +# After major edits, immediately test key functionality +python -c "from module import Class; test_basic_functionality()" +``` + +**Impact**: This prevented the parameters.py file corruption incident and caught the unit/scaling issues early in development. + +#### 4. What Worked Well in a second session: + +- Iterative development approach with small, focused changes +- Database design with composite keys for robust copy/paste functionality +- IMOD tool integration with subprocess management and real-time validation +- Python multiprocessing implementation with shared memory and queue communication +- Project-aware state management with tab notification system + +--- diff --git a/.gitignore b/.gitignore index dcd8dc51..90639d0d 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,9 @@ testScripts/readme.txt testScripts/requiredMCRProducts.txt testScripts/unresolvedSymbols.txt +# Python virtual environment +.venv/ +venv/ +env/ + __pycache__/ \ No newline at end of file diff --git a/GUI_IMPLEMENTATION_SUMMARY.md b/GUI_IMPLEMENTATION_SUMMARY.md new file mode 100644 index 00000000..0e845d67 --- /dev/null +++ b/GUI_IMPLEMENTATION_SUMMARY.md @@ -0,0 +1,184 @@ +# emClarity GUI - Implementation Summary + +## 🎉 Successfully Implemented + +We have successfully created a modern PySide6-based GUI for emClarity with the following features: + +## Core Components + +### State Management (`state_manager.py`) + +- Centralized state management using SQLite database +- Stores parameters, project settings, user preferences, and tilt-series assets +- Ensures data persistence across sessions with composite key support +- Project-aware data loading and synchronization + +### Main GUI (`main.py`) + +- Central application window with tab-based navigation +- Integrates all workflow components including tilt-series asset management +- Manages project loading and saving with tab notification system +- Coordinates asset validation and alignment workflows + +### Tilt-Series Asset Management (`tilt_series_assets.py`) + +- Comprehensive asset organization with group-based structure +- IMOD tool integration (header, extracttilts, imod commands) +- Multi-selection support with copy/paste functionality +- Real-time validation status tracking and visual indicators +- Hierarchical tree display with collapsible groups + +### AutoAlign Workflow (`autoalign_widget.py`) + +- Group-based alignment processing with asset validation +- Python multiprocessing for parallel alignment execution +- Progress tracking with shared memory and queue communication +- Selection preservation and error handling + +### Parameter Management (`parameters.py`, `parameter_loader.py`) + +- Dynamic parameter loading from JSON configuration +- Widget creation and validation +- Parameter persistence and retrieval + +### ✅ Environment Setup +- **Virtual Environment**: Properly configured Python venv with PySide6 +- **Version Detection**: Automatically detects emClarity 1.8.4.0.v23a from `/sa_shared/software/current_emClarity` +- **Path Configuration**: Correctly sets up all emClarity environment variables +- **Display Support**: Works with X11 forwarding and handles display detection + +### ✅ Modern UI Features +- **High-DPI Support**: Larger fonts (12pt) for better readability on high-resolution monitors +- **Version Display**: Prominently shows current emClarity version in header +- **Responsive Design**: Clean, modern interface with proper spacing +- **Error Handling**: Graceful handling of configuration and display issues + +### ✅ Project Structure +``` +gui/ +├── __init__.py # Package initialization +├── config.py # Environment configuration +├── commands.py # Command definitions (27 commands, 9 categories) +├── main.py # GUI application (minimal working version) +├── launcher.py # Python launcher with error handling +├── run_gui.sh # Shell script launcher +├── test_components.py # Component testing script +├── test_gui.py # GUI testing script +└── emclarity-gui.desktop # Desktop entry file +``` + +## 🚀 Quick Start + +### Option 1: Using the shell launcher (Recommended) +```bash +cd /sa_shared/git/emClarity +./gui/run_gui.sh +``` + +### Option 2: Manual launch +```bash +cd /sa_shared/git/emClarity +source .venv/bin/activate +python gui/main.py +``` + +## 📊 Current Status + +### ✅ Working Components + +- **Tilt-Series Asset Management**: Full-featured asset organization with group structure, IMOD integration, and validation +- **AutoAlign Workflow**: Group-based alignment with multiprocessing and progress tracking +- **Project Management**: Project loading/saving with tab notification and coordinated updates +- **Multi-Selection Support**: Extended selection in tree widgets with copy/paste functionality +- **Real-time Validation**: IMOD tool integration with visual status indicators +- **State Persistence**: SQLite database with composite keys for reliable data storage +- **Parameter Management**: Dynamic widget creation and validation from JSON configuration +- **Environment detection and configuration**: Robust system detection and setup +- **Modern UI Features**: High-DPI support, responsive layouts, collapsible groups + +### 🔄 Completed Major Features + +The GUI now includes comprehensive tilt-series workflow capabilities: + +- **Asset Management System**: Complete with IMOD integration, validation, and group organization +- **Parallel Processing**: Multiprocessing implementation for AutoAlign with progress tracking +- **Data Persistence**: Robust SQLite storage with composite keys and project-aware loading +- **Advanced UI**: Multi-selection, copy/paste, collapsible groups, and responsive layouts + +### 🎯 Future Enhancement Opportunities + +- Integration with additional emClarity workflow steps (CTF estimation, reconstruction) +- Advanced visualization features for tilt-series data +- Batch processing capabilities for multiple datasets +- Export/import functionality for asset configurations + +## 🔬 IMOD Integration + +The GUI includes comprehensive IMOD tool integration for tilt-series validation: + +### Supported IMOD Commands + +- **header**: Extract metadata from tilt-series files (dimensions, pixel size, tilt angles) +- **extracttilts**: Parse tilt angles from rawtlt files or image headers +- **imod**: Launch IMOD viewer for visual inspection of tilt-series data + +### Validation Features + +- **Real-time Status**: Visual indicators (✓ valid, ✗ invalid, ⚠ warning) for each asset +- **Automatic Validation**: Background validation when assets are added or modified +- **Error Reporting**: Detailed error messages for invalid files or missing dependencies +- **Group Validation**: Aggregate validation status for asset groups + +### Integration Benefits + +- **Seamless Workflow**: Direct validation without leaving the GUI +- **Error Prevention**: Catch invalid assets before processing +- **Visual Feedback**: Immediate status updates for user confidence +- **Tool Consistency**: Uses same IMOD tools as command-line workflows + +## 🛠️ Technical Details + +### Dependencies +- Python 3.12+ with PySide6 +- emClarity installation (automatically detected) +- X11 or Wayland display server + +### Environment Variables Set +- `emClarity_ROOT`: Root installation directory +- `EMC_AUTOALIGN`: Alignment tools path +- `EMC_FINDBEADS`: Bead finding tools path +- `BH_CHECKINSTALL`: Installation checker path +- `MATLAB_SHELL`: Shell configuration + +### Command Categories Available +- **System** (4 commands): help, check, benchmark, experimental +- **Project Setup** (2 commands): init, segment +- **Alignment** (2 commands): autoAlign, alignRaw +- **Analysis** (3 commands): avg, fsc, mask +- **CTF** (1 command): ctf (with sub-options) +- **Classification** (2 commands): pca, cluster +- **Template Matching** (2 commands): templateSearch, cleanTemplateSearch +- **Reconstruction** (2 commands): reconstruct, tomoCPR +- **Utility** (9 commands): calcWeights, skip, rescale, geometry, etc. + +## 🎯 Design Philosophy + +The GUI follows the project guidelines from `prompt.md`: +- **Clean and Simple**: Focused on user workflow +- **Non-destructive**: Interfaces with existing binaries +- **Descriptive**: Clear naming and comprehensive help +- **Fail-fast**: Early error detection and reporting +- **Modern**: Contemporary UI patterns with good typography + +## 🧪 Testing + +All components have been tested: +```bash +# Test configuration and commands +python gui/test_components.py + +# Test GUI creation (headless) +QT_QPA_PLATFORM=offscreen python gui/test_gui.py +``` + +The GUI is now ready for use and further development! diff --git a/README_GUI.md b/README_GUI.md new file mode 100644 index 00000000..897f738a --- /dev/null +++ b/README_GUI.md @@ -0,0 +1,84 @@ +# emClarity GUI + +A modern PySide6-based graphical user interface for emClarity, the cryo-electron microscopy processing software. + +## Features + +- **Modern Interface**: Clean, responsive design optimized for high-DPI displays +- **Version Display**: Shows the current emClarity version and installation path +- **Command Categories**: Organized commands by functional groups +- **Parameter Input**: Smart parameter forms with file browsers and validation +- **Real-time Output**: Live command execution with formatted output +- **Environment Monitoring**: Detailed environment and configuration information +- **GPU Awareness**: Clear indication of GPU-required commands + +## Quick Start + +### Option 1: Using the launcher script (Recommended) +```bash +# From the emClarity root directory +./gui/run_gui.sh +``` + +### Option 2: Manual setup +```bash +# Create and activate virtual environment +python3 -m venv .venv +source .venv/bin/activate + +# Install required packages +pip install PySide6 typing_extensions + +# Launch GUI +python gui/launcher.py +``` + +## System Requirements + +- Python 3.8 or later +- Qt5/Qt6 support (automatically installed with PySide6) +- Display server (X11 or Wayland) for GUI operation +- Existing emClarity installation + +## Available Commands + +The GUI provides access to all emClarity commands organized by category: + +### System Commands +- **help**: Show available commands +- **check**: System dependency verification +- **benchmark**: Performance testing +- **experimental**: Access experimental features + +### Project Setup +- **init**: Initialize new project from template matching results +- **segment**: Define reconstruction subregions + +### Alignment +- **autoAlign**: Automatic tilt-series alignment +- **alignRaw**: Align references against individual subtomograms + +### Analysis +- **avg**: Average subtomograms +- **fsc**: Calculate Fourier Shell Correlation +- **mask**: Create and apply masks + +### CTF Processing +- **ctf**: CTF estimation, refinement, and correction + +### Classification +- **pca**: Principal Component Analysis +- **cluster**: Cluster subtomogram populations + +### Template Matching +- **templateSearch**: Global template matching +- **cleanTemplateSearch**: Clean search results + +### Reconstruction +- **reconstruct**: Volume reconstruction +- **tomoCPR**: Tomogram constrained projection refinement + +### Utility Commands +- **calcWeights**, **skip**, **rescale**, **geometry** +- **removeNeighbors**, **removeDuplicates**, **combineProjects** +- **getActiveTilts**, **montage** diff --git a/alignment/BH_alignRaw3d_v2.m b/alignment/BH_alignRaw3d_v2.m index c97f2d0a..c05f5b69 100644 --- a/alignment/BH_alignRaw3d_v2.m +++ b/alignment/BH_alignRaw3d_v2.m @@ -733,7 +733,8 @@ volumeData = []; else do_load = false; - [ volumeData ] = BH_multi_loadOrBuild(tomoList{iTomo}, ... + [ volumeData ] = BH_multi_loadOrBuild(emc.alt_cache, ... + tomoList{iTomo}, ... mapBackIter, ... samplingRate,... iGPUidx, ... diff --git a/alignment/BH_templateSearch3d_2.m b/alignment/BH_templateSearch3d_2.m index 8c7698dd..c39335df 100644 --- a/alignment/BH_templateSearch3d_2.m +++ b/alignment/BH_templateSearch3d_2.m @@ -164,7 +164,8 @@ do_load = true; -[ tomogram ] = BH_multi_loadOrBuild(sprintf('%s_%d',tomoName,tomoIdx), ... +[ tomogram ] = BH_multi_loadOrBuild(emc.alt_cache, ... + sprintf('%s_%d',tomoName,tomoIdx), ... mapBackIter, ... samplingRate,... gpuIDX, ... diff --git a/ctf/BH_ctf_Correct3d.m b/ctf/BH_ctf_Correct3d.m index a0c09845..1938b443 100755 --- a/ctf/BH_ctf_Correct3d.m +++ b/ctf/BH_ctf_Correct3d.m @@ -307,8 +307,9 @@ for iTomo = 1:nTomos % The order of tomo num could be off but only if all are present do we % skip. - checkRecon = sprintf('cache/%s_bin%d%s.rec', tomoList{iTomo}, samplingRate, filtered); - if exist(checkRecon, 'file') + alt_cache = emc.alt_cache; + checkRecon = EMC_checkCacheForFile(alt_cache, sprintf('cache/%s_bin%d%s.rec', tomoList{iTomo}, samplingRate, filtered)); + if isfile(checkRecon) try % Could have a corrupt file testread = MRCImage(checkRecon,0); @@ -388,6 +389,7 @@ if samplingRate > 1 + % For now, we are only using the alt_cache for the tomos fullStack = sprintf('%aliStacks/%s_ali%d.fixed', tiltList{iTilt}, mapBackIter + 1); inputStack = sprintf('cache/%s_ali%d_bin%d.fixed', tiltList{iTilt}, mapBackIter + 1, samplingRate); if ~emc_check_for_valid_image_file(inputStack) @@ -653,10 +655,11 @@ for iTomo = 1:nTomos % Note that bh_global_turn_on_phase_plate could be true for any of the recon_for_stage bools, so it must % be checked first. + alt_cache = emc.alt_cache; if (bh_global_turn_on_phase_plate(1)) - reconNameFull = sprintf('cache/%s_bin%d_filtered.rec', tomoList{iTomo}, samplingRate); + reconNameFull = EMC_setCacheForFile(alt_cache, sprintf('cache/%s_bin%d_filtered.rec', tomoList{iTomo}, samplingRate)); else - reconNameFull = sprintf('cache/%s_bin%d.rec', tomoList{iTomo},samplingRate); + reconNameFull = EMC_setCacheForFile(alt_cache, sprintf('cache/%s_bin%d.rec', tomoList{iTomo},samplingRate)); end fprintf('in ctf3d reconNameFull is %s\n\n',reconNameFull); diff --git a/ctf/BH_ctf_Updatefft.m b/ctf/BH_ctf_Updatefft.m index 45c419b3..2d1429f7 100644 --- a/ctf/BH_ctf_Updatefft.m +++ b/ctf/BH_ctf_Updatefft.m @@ -499,7 +499,7 @@ end end end - save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta'); + save(sprintf('%s.mat', emc.('subTomoMeta')), 'subTomoMeta', '-v7.3'); end if ( flgParallel ) diff --git a/docs/gen_param.m b/docs/gen_param.m new file mode 100644 index 00000000..608d1056 --- /dev/null +++ b/docs/gen_param.m @@ -0,0 +1,275 @@ +% This is a comment +% Inline comments will break the parser. + + +% String to name the structure that contains all of the metadata, projectName +subTomoMeta=full_enchilada_2_1_branch_5 + +save_mapback_classes=1 +tomoCPR_n_particles_minimum=1 + +measure_noise_variance=0 + +fastScratchDisk=ram + +nGPUs=4 +nCpuCores=16 +n_tilt_workers=4 + +refine_defocus_cisTEM=0 +rerun_refinement_cisTEM=0 + +phakePhasePlate=0 +flgQualityWeight=0 + +flgMultiRefAlignment=0 +updateClassByBestReferenceScore=0 + +max_ctf3dDepth=100e-9 +% Do not whiten (1), but apply ctf weiner filter (3) with additive term +whitenPS=[0,0,0.5] +diameter_fraction_for_local_stats=0.9 +test_local=1 +scale_mip=0 + + +nPeaks=1 +symmetry=C12 +doHelical=0 + +Pca_refineKmeans=1 + +% if > 1 use this many subtomos in the avg +% if < 1 use this fraction in the avg +%ccc_cutoff=0.6 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%% Mask parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% The particle radius in x,y,z Angstrom, smallest value to contain particle. +% For particles in a lattice, neighboring particles can be used in alignment +% by specifying a larger mask size, but this paramter must correspond to your +% target, a cetral hexamer of capsid proteins for example. + +% For TM +particleRadius=[180,180,150] +particleMass=3.2 + +Ali_mType=cylinder +Cls_mType=cylinder + + +% For special cases where repeated motifs are present which might cause one +% subtomo to drift to a neighbor. This allows a larger alignment mask to be used +% for the rotational search (Ali_m...) but limits the translational peak search. +Peak_mType=cylinder +%Peak_mRadius=[210,210,320] +Peak_mRadius=[40,40,40] +% mask radius and center - and center in Angstrom. Mask size is determined +% large enough to contain delocalized signal, proper apodization, and to +% avoid wraparound error in cross-correlation. +% mask radius and center - and center in Angstrom. Mask size is determined +% large enough to contain delocalized signal, proper apodization, and to +% avoid wraparound error in cross-correlation. +Ali_mRadius=[220,220,164] +%Ali_mRadius=[220,220,200] +Ali_mCenter=[0,0,0] +Cls_mRadius=[220,220,164] +Cls_mCenter=[ 0,0,0 ] + + +% Sampling rate +Ali_samplingRate=3 +Cls_samplingRate=3 + +move_reference_by_com=0 + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%% Tomo-constrained projection refinement parameters %%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +% I advise to avoid using this experimental feature for now. +tomoCprDefocusRefine=0 +tomoCprDefocusRange=500e-9; +tomoCprDefocusStep=20e-9; + +% By default the patch size is calculated based on the number of available fiducials and the +% mass. To limit the number of local areas, set this to something other than zero. +tomoCPR_target_n_patches_x_y=[3,4] +tomoCPR_random_subset=0 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% subTomogram alignment %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +Raw_className=0 +% Second row specifies C1 symmetry +Raw_classes_odd=[0;12.*ones(1,1)] +Raw_classes_eve=[0;12.*ones(1,1)] + +% replicate the in plane angles at each (CX) symmetry position +symmetry_constrained_search=0 +Raw_angleSearch=[0,0,180,3] +print_alignment_stats=1 +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% Template matching parameters %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +Tmp_bandpass=[0.01,1200,25] +Tmp_samplingRate=5 +Tmp_threshold=1500 +Tmp_angleSearch=[180,12,180,12] + +Tmp_targetSize=[512,512,768] + +Tmp_half_precision=0 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%% Class reference %%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + + +Cls_className=49 +Cls_classes_odd=[1:64;12.*ones(1,64)] +Cls_classes_eve=[1:64;12.*ones(1,64)] + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%% FSC Paramters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% On/Off anisotropic SSNR calc +flgCones=0 +fsc_shape_mask=0 +% B-factor applied to weighted averages and refs. Should be < 20. Can be a vector +% where the 2:end positions generate independent maps at that sharpening +% when avg paramN.m N FinalAlignment is run. + +Fsc_bfactor=10 + +% For very tightly packed subTomos set to 1 to avoid mixing halfsets +% form overlaping peripheral density. fscGoldSplitOnTomos=0 +fscGoldSplitOnTomos=0 +% Default to doing an alignment between class halfs before calculating FSC +% This should be deprecated as the halves converge to each other. +fscWithChimera=0 + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%% Classification Paramters %%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% Constrain to the asymmetric unit +% Only for Cx and cylinder masks right now. +Pca_constrain_symmetry=1 +flgPcaShapeMask=0 +% On/Off classification. This must be on when "avg paramN.m N RawAlignment" +% is run at the begining of a cycle where classification is to be run. +flgClassify=1 + +% List of different cluster sizes to try, eg [3;4] +Pca_clusters=[49,64] + +% Maximum number of eigenvalues/vectors to save +Pca_maxEigs=64 + +% Different resolution bands to run PCA on. Not all need to be used for subsequent +% clustering. (Angstrom) + +pcaScaleSpace=[21,42,84]; + + +% Different ranges of coefficients to use in the clustering. At times, the +% missing wedge can be a strong feature, such that ignoring the first few +% eigen values can be usefule. [2:40 ; 6;40 ; 10:40] +% Each row must have the same number of entries, and there must be a row +% for each scale space, even if it is all zeros. + +% NOTE: if using multi_refalignment, this must match the number of references +Pca_coeffs=[3:48;3:48;3:48;3:48]; +Pca_bandpass=[0.01,1200,20]; + + + + +% The number of subtomos to process at once before pulling tempDataMatrix off +% the gpu and into main memory. +PcaGpuPull=5000 +Pca_randSubset=0 + + + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%% Parameters for CTF all si (meters, volts)%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + + +%%%%%%%%%% Microscope parameters %%%%%%%%%% + +% Of the data saved in fixed stacks - MUST match header +PIXEL_SIZE=2.50e-10 +% Currently any super-resolution data is cropped in Fourier Space after alignment +% allowing for finer sampling when interpolating the stacks, while then +% filtering out noise due to aliasing. +SuperResolution=0 +% Spherical abberation +Cs=2.7e-3 +% Accelerating voltage +VOLTAGE=300e3 +% Percent amplitude contrast +AMPCONT=0.04 + +% search range - generally safe to test a wide range +defEstimate=3.5e-6 +defWindow=1.75e-6 +% The PS is considered from the lower resolution inflection point +% past the first zero to this cutoff resolution +defCutOff=6e-10 + +% Total dose in electron/A^2, assumed constant rate +CUM_e_DOSE=180 +% Gold fiducial diameter +beadDiameter=10e-9 + + +oneOverCosineDose=0 +startingAngle=0 +startingDirection=pos +doseSymmetricIncrement=3 +% The reported value is 4.86 but I'm scaling this down to 60% of that +doseAtMinTilt=2.9 + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%% Advanced Parameters %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + +% default is 1.5, may need to shrink for large objects, may need to increase for periodic objects. +scaleCalcSize=1.3 + +autoAli_min_sampling_rate=10 +autoAli_max_sampling_rate=4 +autoAli_patch_size_factor=4 +autoAli_patch_overlap=0.6 +autoAli_max_resolution=18 +autoAli_refine_on_beads=0 +autoAli_iterations_per_bin=2 +autoAli_patch_tracking_border=64 +autoAli_n_iters_no_rotation=2 +autoAli_max_shift_in_angstroms=300 +autoAli_max_shift_factor=1 + +ctf_tile_overlap=4 + + + diff --git a/emClarity_Tutorial.md b/emClarity_Tutorial.md new file mode 100644 index 00000000..6c1b0714 --- /dev/null +++ b/emClarity_Tutorial.md @@ -0,0 +1,580 @@ +# emClarity v1.5.3.10 Tutorial + +## Table of Contents + +1. [How to use this guide](#how-to-use-this-guide) +2. [The project directory](#the-project-directory) +3. [Get your data ready](#get-your-data-ready) +4. [Workflow](#workflow) +5. [Initial tilt-series alignment](#initial-tilt-series-alignment) +6. [Defocus estimate](#defocus-estimate) +7. [Select the sub-regions](#select-the-sub-regions) +8. [Picking](#picking) +9. [Initialize the project](#initialize-the-project) +10. [CTF 3D](#ctf-3d) +11. [Averaging](#averaging) +12. [Alignment](#alignment) +13. [TomoCPR](#tomocpr) +14. [Classification](#classification) +15. [Final map](#final-map) +16. [Algorithms](#algorithms) + +--- + +## How to use this guide + +### Run the jobs + +Our main objective in writing this tutorial is to help you get started using **emClarity** for processing sub-tomogram data as quickly as possible. To begin, we will not introduce all of the methods that **emClarity** puts at your disposal, instead focusing on core features and concepts. If at any point you are confused, or something seems to not work as you expect, you might find more information on the [wiki](https://github.com/bHimes/emClarity/wiki); feel free to also search the mailing list archive, or post new questions to the community forum, hosted on [google groups](https://groups.google.com/forum/#!forum/emclarity), should you have any questions you cannot resolve on your own. + +> **Tip**: To display every procedure available, run `emClarity help` from the command line. + +### Algorithms + +The **emClarity** source code is available on [github](https://github.com/bHimes/emClarity/tree/LTS_version_1_5_0), and we encourage you to go through the code to look at the algorithms directly. Because **emClarity** is frequently being updated, this can be a great way to see what's going on behind the scenes. + +Section [Algorithms](#algorithms) contains descriptions of the algorithms for each section presented in this tutorial. Please keep in mind that these are simplified descriptions of what **emClarity** is actually doing, as we often don't mention the details that were implemented to make the code more efficient. + +### Installation and Requirements + +Information about the [installation](https://github.com/bHimes/emClarity/wiki/Installation) and the software and hardware [requirements](https://github.com/bHimes/emClarity/wiki/Requirements) is available [online](https://github.com/bHimes/emClarity/wiki). + +### Parameter files + +**emClarity** is currently using a parameter file to manage inputs. You can find an example [here](https://github.com/bHimes/emClarity/blob/master/docs/exampleParametersAndRunScript/param0.m). + +### System parameters + +Your parameter files should have the following parameters: + +| Parameter | Description | +|-----------|-------------| +| `nGPUs` * | The number of visible GPUs. **emClarity** will try to use them in parallel as much as possible. If this number doesn't correspond to the actual number of GPUs available, **emClarity** will ask you to either adjust this number to match the number of GPUs, or modify the environment variable `CUDA_VISIBLE_DEVICE` to make some GPUs invisible to **MATLAB**. | +| `nCpuCores` * | The maximum number of processes to run simultaneously. In most **emClarity** programs, the number of processes launched in parallel on a single GPU is equal to `nCpuCores`/`nGPUs`. If your devices run out of memory, it is likely that you will have to decrease the number of processes per device, thus decreasing this parameter. | +| `fastScratchDisk` * | Path of the optional temporary cache directory, used by `ctf 3d` and `tomoCPR`. This directory is only temporary and is moved back inside the project directory at the end of the execution. We recommend setting this to the fastest storage you have available. If left empty, `ctf 3d` and `tomoCPR` will use directly the project cache directory. | + +*Required parameters are marked with * + +--- + +## The project directory + +**emClarity** should be run from the "project directory", referred to as ``. Every output will be saved in this directory, ignoring the temporary cache set by the `fastScratchDisk` parameter. As we go along, we will present in more detail each sub-directory and their content. + +### Directory Structure + +- **``**: Contains every input file and input directories **emClarity** needs and every outputs. As most of the **emClarity** programs are project based, you should run **emClarity** from this directory. + +- **`/rawData`**: **(User created)** Contains the original raw tilt-series data (`*.mrc`, `*.st`) and associated files (`*.rawtlt`) that will be used as input for `autoAlign`. This is where you should place your downloaded or collected tilt-series before starting the workflow. + +- **`/fixedStacks`**: **(Created by `autoAlign` or user)** Contains the raw (not aligned) tilt-series (`*.fixed`) and the initial tilt-series alignment files (`*.xf`, `*.tlt` and optionally `*.local` and `*.erase`). This directory is automatically created and populated by `emClarity autoAlign`, or manually created if you're starting with pre-aligned data from **ETomo**. + +- **`/fixedStacks/ctf`**: Created by `ctf estimate` and updated after tilt-series refinements by `ctf update`. Contains the radial averages (`*_psRadial1.pdf`) and stretched power spectrum (`*_PS2.mrc`) computed by `ctf estimate`, as well as the tilt-series metadata (`*_ctf.tlt`), used throughout the entire workflow and containing in particular the dose-scheme and defocus estimate of each view. + +- **`/aliStacks`**: Created by `ctf estimate` and updated after tilt-series refinement by `ctf update`. Contains the aligned, bead-erased tilt-series. These stacks are mostly used by `ctf 3d` to compute the tomograms at different binning. + +- **`/cache`**: Created and updated by **emClarity** when needed, usually during `ctf 3d`. Store any stack or reconstruction for the current binning. If a reconstruction (`*.rec`) is present at the current binning, `ctf 3d` will skip its reconstruction. + +- **`/convmap`**: When creating a project with `init`, **emClarity** will look in this directory to grab outputs from `templateSearch`. If you pick your particles with **emClarity**, the content of this directory is generated by `templateSearch`. + +- **`/recon`**: Holds the information for each reconstructed sub-region in a given tilt-series. The `*_recon.coords` files are read into the metadata created by `init` and is used whenever a tomogram is made or whenever the coordinates of a sub-region is needed. + +- **`/`**: **emClarity** does not directly use this directory, but it is used by `recScript2.sh` to define the sub-regions boundaries and create `/recon`. + +- **`/FSC`**: Created and updated during subtomogram averaging. Contains the spherical and conical FSCs for each cycle (`*fsc_GLD.txt` and `*fsc_GLD.pdf`), as well as the Figure-Of-Merit used for filtering (`*cRef_GLD.pdf`) and the CTF-corrected volume used for FSC calculations. + +- **`/alignResume`**: Contains the results of the subtomogram alignments, for each cycle. **emClarity** will look at this directory before aligning the particles from a given sub-region. If the results for this sub-region, at the current cycle, are already saved, it will skip the alignment. + +--- + +## Get your data ready + +In this tutorial, we will use the apoferritin tomography dataset deposited on [EMPIAR-11273](https://www.ebi.ac.uk/pdbe/emdb/empiar/entry/11273/). You should be able to get a sub-3Å map from this tutorial. Apoferritin is an excellent choice for learning subtomogram averaging due to its high octahedral symmetry, which makes processing faster and typically yields higher resolution results. + +### Tutorial Dataset + +| Aspect | Description | +|--------|-------------| +| **Sample** | Apoferritin (octahedral symmetry, ~12nm diameter) | +| **Tilt-series count** | 100 tilt-series (TS_12 to TS_123) | +| **Tilt-scheme** | Hagen dose-symmetric, ±48°, 3° increment, 115.5e/Ų total exposure | +| **Instruments** | Krios at 300kV, Gatan K3 camera, 0.729Å/pix calibrated pixel size | +| **Defocus range** | -1 to -3 μm | +| **Expected result** | Sub-3Å reconstruction | + +> **Note**: This dataset uses EER format movies that need to be converted to tilt-series. The high symmetry of apoferritin (octahedral) makes it ideal for tutorial purposes as it processes much faster than asymmetric particles like ribosomes. With 100 tilt-series, this provides excellent statistics for high-resolution reconstruction. + +### Setting up the project directory + +Before starting, create your project directory and the initial `rawData` subdirectory: + +```bash +# Create your project directory +mkdir -p /path/to/your/project +cd /path/to/your/project + +# Create rawData directory for your input tilt-series +mkdir rawData + +# For EMPIAR-11273, the data is available in EER format and needs to be converted +# The dataset contains 100 tilt-series (TS_12 to TS_123) +# Copy or link the converted tilt-series to rawData/ +# Each tilt-series should be named like TS_012.st, TS_013.st, etc. +``` + +The `fixedStacks` directory will be automatically created by `emClarity autoAlign` in the next step. If you're starting with pre-aligned data from **ETomo**, you would manually create and populate the `fixedStacks` directory instead. + +> **Data preparation note**: The EMPIAR-11273 dataset contains EER movies that need motion correction and tilt-series generation. Use tools like RELION's `relion_convert_to_tiff` or IMOD's `alignframes` to convert the EER data to tilt-series stacks before starting the emClarity workflow. + +--- + +## Workflow + +You will often find that it is much easier to organize every **emClarity** calls into one script. This script has two main purposes. First, it keeps track of the jobs that have been run (you can also find this information into the `logFile` directory). This is often useful to visualize the global picture and it might help you to remember how you got your final reconstruction. Second, it is a script, so you can use it directly to run **emClarity**, making the workflow much simpler. + +### Example Workflow Script + +```bash +#!/bin/bash + +# Simple function to stop on *most* errors +check_error() { + sleep 2 + if tail -n 30 ./logFile/emClarity.logfile |\ + grep -q "Error in emClarity" ; then + echo "Critical error found. Stopping the script." + exit + else + echo "No error detected. Continue." + fi +} + +# Change binning with tomoCPR +run_transition_tomoCPR() { + emClarity removeDuplicates param${i}.m ${i}; check_error + emClarity tomoCPR param${i}.m ${i}; check_error + emClarity ctf update param$((${i}+1)).m; check_error + emClarity ctf 3d param$((${i}+1)).m; check_error +} + +# Basic alignment cycle +run_avg_and_align() { + emClarity avg param${i}.m ${i} RawAlignment; check_error + emClarity alignRaw param${i}.m ${i}; check_error +} + +# autoAlign +# ctf estimate +# templateSearch + +# Create metadata and reconstruct the tomograms +emClarity init param0.m; check_error +emClarity ctf 3d param0.m; check_error + +# First reconstruction - check if that looks OK. +emClarity avg param0.m 0 RawAlignment; check_error +emClarity alignRaw param0.m 0; check_error + +# Bin 3 +for i in 1 2 3 4; do run_avg_and_align; done + +# Run tomoCPR at bin3 using cycle 4 and then switch to bin2 +run_transition_tomoCPR + +# Bin 2 +for i in 5 6 7 8 9; do run_avg_and_align; done + +# Run tomoCPR at bin2 using cycle 9 and then switch to bin1 +run_transition_tomoCPR + +# Bin 1 +for i in 10 11 12 13 14; do run_avg_and_align; done + +# Last cycle: merge the datasets +emClarity avg param15.m 15 RawAlignment; check_error +emClarity avg param15.m 15 FinalAlignment; check_error +emClarity reconstruct param15.m 15; +``` + +This example doesn't have a classification, but as explained in the classification section, classifications are encapsulated in their own cycles, so you can run them anytime you want between two cycles. + +In our experience, it is usually good practice to keep a close eye on how the half-maps and FSC evolves throughout the workflow, specially before deciding to change the sampling. Moreover, the tilt-series refinement is completely optional and you can simply change the binning by running `ctf 3d`, as opposed to `run_transition_tomoCPR`. + +> **Tip**: It is best practice to work the whole way through the workflow with the smallest data-set as possible and once you checked that everything holds, then process your full data. The same approach may be taken with this tutorial; it should be possible to obtain a low-resolution but recognizable 70S ribosome with only two or three of the tilt-series. + +--- + +## Initial tilt-series alignment + +### Objectives + +The first step of the workflow consists into finding an initial alignment for the raw tilt-series, that is the tilt, rotation and shift for each image within the series. After the alignment, the tilt-axis must be parallel to the y-axis. This alignment can be refined later on using the particles positions (tomoCPR section). + +### With emClarity + +**emClarity** can align the tilt-series for you using its `autoAlign` procedure. This procedure is based on the **IMOD** programs **tilt** and **tiltalign** and offers an automatic way of aligning tilt-series, with or without gold beads. + +#### Run + +As with every **emClarity** programs, you should run the next commands in the project directory. The `autoAlign` routine has the following signature: + +``` +emClarity autoAlign +``` + +Where: +- `` is the name of the parameter file +- `` is the tilt-series to align (e.g. `tilt1.st`) +- `` is a text file containing the raw tilt-angles (e.g. `tilt1.rawtlt`), in degrees +- `` is the image rotation (tilt-axis angle from the vertical), in degrees, as specified in **ETomo** + +For example, to run `autoAlign` on the first tilt-series of the tutorial: + +``` +emClarity autoAlign param.m TS_012.st TS_012.rawtlt 0 +``` + +For this apoferritin dataset, you may need to determine the appropriate rotation angle. You can check the first few tilt-series manually or use a small rotation angle since the data acquisition was well-controlled. + +#### Outputs + +**emClarity** creates and organizes the necessary files it needs to run the next step of the workflow. The goal here is to check whether or not the alignment is good enough to start with and the easiest way is to look at `fixedStacks/_3dfind.ali` or `fixedStacks/_binX.ali`. + +If you are familiar with **ETomo**, then you can of course also look at the log files saved in `emC_autoAlign_`. For instance, to visually check the fiducial beads: + +``` +3dmod \ +emC_autoAlign_/_X_3dfind.ali \ +emC_autoAlign_/_X_fitbyResid_X.fid +``` + +### With ETomo + +If you don't want to use `autoAlign`, we do recommend using the (fiducial) alignment procedure from the **ETomo** pipeline. One powerful option of this pipeline is to be able to solve for a series of local alignments using subsets of fiducial points, which can then be used by **emClarity**, via the IMOD **tilt** program, to reconstruct the tomograms. + +For each tilt-series, **emClarity** needs: + +- **`.fixed`**: the raw (not aligned) tilt-series. These should not be exposure-filtered nor phase flipped. + +- **`.xf`**: the file with alignment transforms to apply to the `.fixed` stacks. This file should contain one line per view, each with a linear transformation specified by six numbers. + +- **`.tlt`**: the file with the solved tilt angles. One line per view, angles in degrees. + +- **(optional) `.local`**: the file of local alignments. This file is similar to the `.xf` file, but contains one transformation per view and per patch. + +- **(optional) `.erase`**: the file with the coordinates (in pixel) of the fiducial beads to erase before ctf estimation. + +These files should be copied to `/fixedStacks`. + +> **Tip**: You don't necessarily need to copy the tilt-series to the `fixedStacks` directory; use soft links: `ln -s <...>/.mrc <...>/fixedStacks/.fixed` + +--- + +## Defocus estimate + +### Objectives + +There are two main objectives. First, create the aligned, optionally bead-erased, weighted stacks. Weighted refers to the per-view weighting applied by **emClarity** to take into account the frequency dependent drop in SNR due to radiation damage, an isotropic drop in SNR due to increased thickness with the tilt-angle causing inelastic scattering losses and optionally also for the cosine dose-scheme, also referred as Saxton scheme. These stacks will be then used to compute the tomograms at later stages. The second objective is to estimate the defocus of each view of the stack (two defoci and the astigmatism angle, per view). + +### Run + +The `ctf estimate` routine has the following signature: + +``` +emClarity ctf estimate +``` + +`` is the name of the parameter file (e.g. `param_ctf.m`), and `` is the base-name of the tilt-series in `/fixedStacks` you wish to process. + +For example, to run `ctf estimate` on the first tilt-series of the tutorial: + +``` +emClarity ctf estimate param_ctf.m TS_012 +``` + +If you have many tilt-series and you don't want to run all of them individually, you can do the following: + +```bash +#!/bin/bash +for stack in fixedStacks/*.fixed; do + prefix=${stack#fixedStacks/} + emClarity ctf estimate param_ctf.m ${prefix%.fixed} +done +``` + +For the apoferritin dataset, you generally won't need to remove specific images, but if needed, `ctf estimate` can remove images from the stack. For instance, to remove the first view: + +``` +emClarity ctf estimate param_ctf.m TS_012 1 +``` + +### Outputs + +You should make sure the tilt-series "looks aligned" and the average defocus (at the tilt axis) was correctly estimated. The best way to check: + +1. Open `aliStacks/_ali1.fixed` and go through the views. The views should be aligned to the tilt-axis, which must be parallel to the Y axis (so vertical if you use **3dmod**). If an `*.erase` file was available for this tilt-series, the beads should be removed. + +2. Open `fixedStacks/ctf/_ali1_psRadial_1.pdf` and check that the theoretical CTF estimate (green) matches the radial average of the power spectrum of the tilt-series (black). Note that the amplitude doesn't matter here, the phase on the other hand, does. + +3. If they don't match, it is likely that you will need to adjust the `defEstimate` and `defWindow` parameters. Open `fixedStacks/ctf/*_ccFIT.pdf`, which plots the cross-correlation score as a function of defocus. There is often an obvious correct peak, smoothly rising and falling. If you don't see this peak, try to change the sampled defoci with `defEstimate` ± `defWindow` and re-run `ctf estimate`. + +--- + +## Algorithms + +### Naming conventions + +There is a lot of things to cover and it is often easier to use abbreviations (CTF, FSC, CCC, etc.) and symbols to refer to something. + +Indexes are subscripts, e.g. the p-th subtomogram is referred as **s**_p. This works with multiple indexes, e.g. the p-th subtomogram rotated by the r-th rotation is referred as **s**_p,r. Labels are subscripts as well, e.g. if we want to specify that the subtomograms are in the reference frame, we would write **s**_ref. + +### Euler angles conventions + +The φ, θ, ψ Euler angles used by **emClarity** describe a z-x-z active intrinsic rotations of the particles coordinate system. That is to say, to switch the particles from the microscope frame to the reference frame, the basis vectors of the subtomograms are rotated (positive anti-clockwise) around z, the new x, and the new z axis. + +The microscope frame defines the coordinate system of the microscope, where the electron beam is the z axis. When the subtomograms are extracted from their tomogram, they are in the microscope frame. The reference frame is the coordinate system attached to the reconstruction, i.e. the subtomogram average and is usually set during the particle picking. + +### Linear transformations in Fourier space + +Linear transformations are often applied in Fourier space directly. It might be useful to write down a few useful properties of the Fourier transforms. + +- **Shift**: Shifting an image in real space is equivalent to applying a complex phase shift to its frequency spectrum +- **Magnification**: Magnifying an image by a factor a is equivalent to magnifying its frequency spectrum by 1/a +- **Rotation**: Rotating an image by an angle Θ in real space is the same as rotating its frequency spectrum by the same angle Θ + +--- + +## Select the sub-regions + +The purpose of this step is to define sub-regions within each tilt-series that will be reconstructed as tomograms. This is typically done using IMOD's reconstruction scripts to create appropriate coordinate files that define the boundaries of each sub-region. + +--- + +## Picking + +### Objectives + +It's time to pick the particles, i.e. the subtomograms. There are many ways to pick particles, but they usually all rely on the tomograms. Each particle can be described by its x, y, z coordinates and φ, θ, ψ Euler angles. **emClarity** has a template matching routine that can pick the subtomograms for you, but it requires a template. + +### Run + +#### Preparing the template + +Before running `templateSearch`, you need to prepare a template. This template should have the same pixel size as the `PIXEL_SIZE` parameter. It doesn't need to be low-pass filter, as **emClarity** will do it internally. If you want to re-scale a map, you can run: + +```bash +emClarity rescale +``` + +`` and `` are the name of your template and the output name of the re-scaled template, respectively. `` is the pixel size of your template and `` is the desired pixel size. `` can be "GPU" or "cpu". + +> **Note**: For this tutorial, you'll need an apoferritin template. Apoferritin templates are readily available from the PDB (e.g. PDB ID: 2FHA) or EMDB, and can be filtered to the appropriate resolution for initial template matching. + +#### Generating the tomograms + +The tomograms use for the template matching are CTF multiplied. To generate them, simply run: + +```bash +emClarity ctf 3d templateSearch +``` + +This will generate a tomogram for every subregion defined in the `recon/*.coords` files. + +#### Template matching + +The `templateSearch` routine has the following signature: + +```bash +emClarity templateSearch