From c5966644e2f06e275029815c6e97fe6d5835568f Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Thu, 11 Aug 2022 14:00:37 -0400 Subject: [PATCH 1/7] removed reshaping from grainmap for multiple groups --- hexrd/grainmap/nfutil.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/hexrd/grainmap/nfutil.py b/hexrd/grainmap/nfutil.py index ca3b119a0..eb7456062 100644 --- a/hexrd/grainmap/nfutil.py +++ b/hexrd/grainmap/nfutil.py @@ -1325,11 +1325,15 @@ def gen_trial_exp_data(grain_out_file,det_file,mat_file, mat_name, max_tth, comp return experiment, nf_to_ff_id_map -def process_raw_confidence(raw_confidence,vol_shape,id_remap=None,min_thresh=0.0): +def process_raw_confidence(raw_confidence,vol_shape=None,id_remap=None,min_thresh=0.0): print('Compiling Confidence Map...') - confidence_map=np.max(raw_confidence,axis=0).reshape(vol_shape) - grain_map=np.argmax(raw_confidence,axis=0).reshape(vol_shape) + if vol_shape == None: + confidence_map=np.max(raw_confidence,axis=0) + grain_map=np.argmax(raw_confidence,axis=0) + else: + confidence_map=np.max(raw_confidence,axis=0).reshape(vol_shape) + grain_map=np.argmax(raw_confidence,axis=0).reshape(vol_shape) #fix grain indexing From 7e8d859ac5bf3844278acaee8d1d66e00a3797c7 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Thu, 11 Aug 2022 14:02:07 -0400 Subject: [PATCH 2/7] Added ability to put in max RAM and split into groups --- scripts/process_nf.py | 358 ++++++++++++++++++++++++------------------ 1 file changed, 208 insertions(+), 150 deletions(-) diff --git a/scripts/process_nf.py b/scripts/process_nf.py index 7b2cd2547..d9390d3a4 100644 --- a/scripts/process_nf.py +++ b/scripts/process_nf.py @@ -15,21 +15,11 @@ import multiprocessing as mp import os -import logging +from matplotlib import pyplot as plt -#from hexrd.grainmap import nfutil from hexrd.grainmap import nfutil -try: - import matplotlib.pyplot as plt - matplot=True -except(ImportError): - logging.warning(f'no matplotlib, debug plotting disabled') - matplot=False - - - #============================================================================== # %% FILES TO LOAD -CAN BE EDITED #============================================================================== @@ -39,255 +29,323 @@ #A materials file, is a cPickle file which contains material information like lattice #parameters necessary for the reconstruction -main_dir = '/INSERT/WORKDIR/' - -det_file = main_dir + 'retiga.yml' -mat_file= main_dir + 'materials.h5' +main_dir = '/nfs/chess/user/relim/pagan-3274-a/ti7al_bruker' +# main_dir = "/Users/rachellim/Documents/PSU/CHESS_2022Feb/mono_study" +det_file = main_dir + '/retiga.yml' +mat_file = main_dir + '/materials.h5' #============================================================================== # %% OUTPUT INFO -CAN BE EDITED #============================================================================== -output_dir = main_dir +output_dir = '/nfs/chess/user/relim/NF_chunking' #============================================================================== # %% NEAR FIELD DATA FILES -CAN BE EDITED - ZERO LOAD SCAN #============================================================================== #These are the near field data files used for the reconstruction, a grains.out file -#from the far field analaysis is used as orientation guess for the grid that will -grain_out_file = '/LOC/grains.out' +#from the far field analaysis is used as orientation guess for the grid that will +grain_out_file = main_dir + '/ti7al-6_grains_best.out' +# grain_out_file = '/Users/rachellim/Documents/PSU/CHESS_2022Feb/mono_study/ti7al-6_grains.out' -#%% -stem='nf_' +#%% #Locations of near field images -data_folder='/INSERT/nf/' +data_folder = '/nfs/chess/raw/2022-1/id3a/pagan-3274-a/ti7al-5/1/nf' -img_start=6 #whatever you want the first frame to be -nframes=1440 -img_nums=np.arange(img_start,img_start+nframes,1) +img_start = 106188 # whatever you want the first frame to be +nframes = 1440 +img_nums = np.arange(img_start, img_start+nframes, 1) -output_stem='NAME_VOL_X' +output_stem = '/ti7al-6_nf_scan9' +#corresponding scan number #============================================================================== # %% USER OPTIONS -CAN BE EDITED #============================================================================== ### material for the reconstruction -mat_name='INSERT' -max_tth=None #degrees, if None is input max tth will be set by the geometry +mat_name = 'Ti7Al' +max_tth = None # degrees, if None is input max tth will be set by the geometry #reconstruction with misorientation included, for many grains, this will quickly #make the reconstruction size unmanagable -misorientation_bnd=0.0 #degrees -misorientation_spacing=0.25 #degrees +misorientation_bnd = 0.0 # degrees +misorientation_spacing = 0.2 # degrees #####image processing parameters -num_for_dark=250#num images to use for median data +num_for_dark = 250 # num images to use for median data -process_type='gaussian' -sigma=2.0 -size=3.0 -process_args=np.array([sigma,size]) +process_type = 'gaussian' +sigma = 4.5 +size = 5 +process_args = np.array([sigma, size]) # process_type='dilations_only' # num_erosions=2 #num iterations of images erosion, don't mess with unless you know what you're doing # num_dilations=3 #num iterations of images erosion, don't mess with unless you know what you're doing # process_args=np.array([num_erosions,num_dilations]) -threshold=1.5 -ome_dilation_iter=1 #num iterations of 3d image stack dilations, don't mess with unless you know what you're doing +threshold = 1.5 +# num iterations of 3d image stack dilations, don't mess with unless you know what you're doing +ome_dilation_iter = 1 #thresholds for grains in reconstructions -comp_thresh=0.0 #only use orientations from grains with completnesses ABOVE this threshold -chi2_thresh=1.0 #only use orientations from grains BELOW this chi^2 +# only use orientations from grains with completnesses ABOVE this threshold +comp_thresh = 0.0 +chi2_thresh = 1.0 # only use orientations from grains BELOW this chi^2 ######reconstruction parameters -ome_range_deg=[(0.,359.75)] #degrees +# ome_range_deg=[(0.,359.75)] #degrees +ome_range_deg = [(-0.55, 359.2)] # degrees -use_mask=True +use_mask = False #Mask info, used if use_mask=True -mask_data_file='/LOC/tomo_mask.npz' -mask_vert_offset=-0.3 #this is generally the difference in y motor positions between the tomo and nf layer (tomo_motor_z-nf_motor_z), needed for registry +mask_data_file = 'tomo_mask.npz' +# this is generally the difference in y motor positions between the tomo and nf layer (tomo_motor_z-nf_motor_z), needed for registry +mask_vert_offset = 0 #these values will be used if no mask data is provided (use_mask=False) (no tomography) -cross_sectional_dim=1.35 #cross sectional to reconstruct (should be at least 20%-30% over sample width) -voxel_spacing = 0.005 #in mm, voxel spacing for the near field reconstruction +# cross sectional to reconstruct (should be at least 20%-30% over sample width) +cross_sectional_dim = 1.35 +voxel_spacing = 0.002 # in mm, voxel spacing for the near field reconstruction ##vertical (y) reconstruction voxel bounds in mm, ALWAYS USED REGARDLESS OF MASK -v_bnds=[-0.085,0.085] #(mm) if bounds are equal, a single layer is produced +#v_bnds=[-0.0025,0.0025] #(mm) if bounds are equal, a single layer is produced +# v_bnds = [-0.085,0.085] +v_bnds = [0, 0] -beam_stop_y_cen=0.0 #mm, measured from the origin of the detector paramters -beam_stop_width=0.6 #mm, width of the beam stop verticallu -beam_stop_parms=np.array([beam_stop_y_cen,beam_stop_width]) +beam_stop_y_cen = 0.0 # mm, measured from the origin of the detector paramters +beam_stop_width = 0.6 # mm, width of the beam stop verticallu +beam_stop_parms = np.array([beam_stop_y_cen, beam_stop_width]) ##### multiprocessing controller parameters -check=None -limit=None -generate=None -ncpus=mp.cpu_count() -chunk_size=500#chunksize for multiprocessing, don't mess with unless you know what you're doing - - -#### debug plotting -output_plot_check=True +check = None +limit = None +generate = None +ncpus = mp.cpu_count() +# chunksize for multiprocessing, don't mess with unless you know what you're doing +chunk_size = 500 #============================================================================== # %% LOAD GRAIN AND EXPERIMENT DATA #============================================================================== -experiment, nf_to_ff_id_map = nfutil.gen_trial_exp_data(grain_out_file,det_file,\ - mat_file, mat_name, max_tth, \ - comp_thresh, chi2_thresh, misorientation_bnd, \ - misorientation_spacing,ome_range_deg, \ - nframes, beam_stop_parms) - - +experiment, nf_to_ff_id_map = nfutil.gen_trial_exp_data(grain_out_file, det_file, mat_file, mat_name, max_tth, comp_thresh, chi2_thresh, misorientation_bnd, + misorientation_spacing, ome_range_deg, nframes, beam_stop_parms) + + #============================================================================== # %% LOAD / GENERATE TEST DATA #============================================================================== - if use_mask: - mask_data=np.load(mask_data_file) - - - mask_full=mask_data['mask'] - Xs_mask=mask_data['Xs'] - Ys_mask=mask_data['Ys']-(mask_vert_offset) - Zs_mask=mask_data['Zs'] - voxel_spacing=mask_data['voxel_spacing'] - - - tomo_layer_centers=np.squeeze(Ys_mask[:,0,0]) #need to think about how to handle a single layer in this context - above=np.where(tomo_layer_centers>=v_bnds[0]) - below=np.where(tomo_layer_centers v_bnds[0]) + below = np.where(tomo_layer_centers < v_bnds[1]) + + in_bnds = np.intersect1d(above, below) + + mask = mask_full[in_bnds] + Xs = Xs_mask[in_bnds] + Ys = Ys_mask[in_bnds] + Zs = Zs_mask[in_bnds] + test_crds_full = np.vstack([Xs.flatten(), Ys.flatten(), Zs.flatten()]).T n_crds = len(test_crds_full) - - to_use=np.where(mask.flatten())[0] - + + to_use = np.where(mask.flatten())[0] + else: - test_crds_full, n_crds, Xs, Ys, Zs = nfutil.gen_nf_test_grid(cross_sectional_dim, v_bnds, voxel_spacing) - to_use=np.arange(len(test_crds_full)) + test_crds_full, n_crds, Xs, Ys, Zs = nfutil.gen_nf_test_grid( + cross_sectional_dim, v_bnds, voxel_spacing) + to_use = np.arange(len(test_crds_full)) + + +test_crds = test_crds_full[to_use, :] + +#============================================================================== +# %% NEAR FIELD - splitting +#============================================================================== + +max_RAM = 1 # in GB +RAM = max_RAM * 1e9 # turn into number of bytes + +RAM_to_use = 0.75 * RAM -test_crds=test_crds_full[to_use,:] +n_oris = len(nf_to_ff_id_map) +n_voxels = len(test_crds) +bits_for_arrays = 64*n_oris*n_voxels + 192 * \ + n_voxels # bits raw conf + bits voxel positions +bytes_for_array = bits_for_arrays/8. +n_groups = np.floor(bytes_for_array/RAM_to_use) # number of full groups +leftover_voxels = np.mod(n_voxels, n_groups) + +print(n_groups) +print(leftover_voxels) + +grouped_voxels = n_voxels - leftover_voxels + +voxels_per_group = grouped_voxels/n_groups # np.floor(n_voxels/(n_groups+1)) #============================================================================== # %% NEAR FIELD - MAKE MEDIAN DARK #============================================================================== -dark=nfutil.gen_nf_dark(data_folder,img_nums,num_for_dark,experiment.nrows,experiment.ncols,dark_type='median',num_digits=6,stem=stem) +# dark=nfutil.gen_nf_dark(data_folder,img_nums,num_for_dark,experiment.nrows,experiment.ncols,dark_type='median',num_digits=6) #============================================================================== # %% NEAR FIELD - LOAD IMAGE DATA AND PROCESS #============================================================================== -image_stack=nfutil.gen_nf_cleaned_image_stack(data_folder,img_nums,dark,experiment.nrows,experiment.ncols,process_type=process_type,\ - process_args=process_args,threshold=img_threshold,ome_dilation_iter=ome_dilation_iter,num_digits=6,stem=stem) +# image_stack=nfutil.gen_nf_cleaned_image_stack(data_folder,img_nums,dark,experiment.nrows,experiment.ncols,process_type=process_type,\ +# process_args=process_args,ome_dilation_iter=ome_dilation_iter,num_digits=6) +image_stack_load = np.load(main_dir + '/ti7al-6_cleaned_nf_scan9_thresh30.npz') + + +image_stack = np.array(image_stack_load['arr_0']) +# #%% +# plt.imshow(image_stack[10,:,:]) +# plt.colorbar() + +# plt.figure() +# plt.imshow(image_stack[30,:,:]) +# plt.colorbar() + - - - #============================================================================== # %% BUILD MP CONTROLLER -#============================================================================== +#============================================================================== # assume that if os has fork, it will be used by multiprocessing. # note that on python > 3.4 we could use multiprocessing get_start_method and # set_start_method for a cleaner implementation of this functionality. -multiprocessing_start_method = 'fork' if hasattr(os, 'fork') else 'spawn' - +multiprocessing_start_method = 'fork' if hasattr(os, 'fork') else 'spawn' -controller=nfutil.build_controller(ncpus=ncpus,chunk_size=chunk_size,check=check,generate=generate,limit=limit) - -#============================================================================== -# %% TEST ORIENTATIONS -#============================================================================== - -## Would be nice to pack the images, quant_and_clip will need to be edited if this is added, dcp 6.2.2021 -#packed_image_stack = nfutil.dilate_image_stack(image_stack, experiment,controller) +controller = nfutil.build_controller( + ncpus=ncpus, chunk_size=chunk_size, check=check, generate=generate, limit=limit) -print('Testing Orientations...') -raw_confidence=nfutil.test_orientations(image_stack, experiment,test_crds,controller,multiprocessing_start_method) - -del controller - +#%% Test orientations in groups -#============================================================================== -# %% PUT IT ALL BACK TOGETHER -#============================================================================== +if n_groups == 0: + raw_confidence = nfutil.test_orientations( + image_stack, experiment, test_crds, controller, multiprocessing_start_method) + + del controller + + raw_confidence_full = np.zeros( + [len(experiment.exp_maps), len(test_crds_full)]) + + for ii in np.arange(raw_confidence_full.shape[0]): + raw_confidence_full[ii, to_use] = raw_confidence[ii, :] -# note that all masking -raw_confidence_full=np.zeros([len(experiment.exp_maps),len(test_crds_full)]) -for ii in np.arange(raw_confidence_full.shape[0]): - raw_confidence_full[ii,to_use]=raw_confidence[ii,:] +else: + grain_map_list = np.zeros(n_voxels) + confidence_map_list = np.zeros(n_voxels) + + # test voxels in groups + for abcd in range(int(n_groups)): + voxels_to_test = test_crds[int( + abcd) * int(voxels_per_group):int(abcd + 1) * int(voxels_per_group), :] + print('Calculating group %d' % abcd) + raw_confidence = nfutil.test_orientations( + image_stack, experiment, voxels_to_test, controller, multiprocessing_start_method) + print('Calculated raw confidence group %d' % abcd) + grain_map_group_list, confidence_map_group_list = nfutil.process_raw_confidence( + raw_confidence, id_remap=nf_to_ff_id_map, min_thresh=0.0) + + grain_map_list[int( + abcd) * int(voxels_per_group):int(abcd + 1) * int(voxels_per_group)] = grain_map_group_list + + confidence_map_list[int( + abcd) * int(voxels_per_group):int(abcd + 1) * int(voxels_per_group)] = confidence_map_group_list + + #now for the leftover voxels + voxels_to_test = test_crds[int( + n_groups) * int(voxels_per_group):, :] + raw_confidence = nfutil.test_orientations( + image_stack, experiment, voxels_to_test, controller, multiprocessing_start_method) + grain_map_group_list, confidence_map_group_list = nfutil.process_raw_confidence( + raw_confidence, id_remap=nf_to_ff_id_map, min_thresh=0.0) + + grain_map_list[int( + n_groups) * int(voxels_per_group):] = grain_map_group_list + + confidence_map_list[int( + n_groups) * int(voxels_per_group):] = confidence_map_group_list + #reshape them + grain_map = grain_map_list.reshape(Xs.shape) + confidence_map = confidence_map_list.reshape(Xs.shape) + + #============================================================================== # %% POST PROCESS W WHEN TOMOGRAPHY HAS BEEN USED #============================================================================== #note all masking is already handled by not evaluating specific points -grain_map, confidence_map = nfutil.process_raw_confidence(raw_confidence_full,Xs.shape,id_remap=nf_to_ff_id_map,min_thresh=0.0) +grain_map, confidence_map = nfutil.process_raw_confidence( + raw_confidence_full, Xs.shape, id_remap=nf_to_ff_id_map, min_thresh=0.0) + +#%% + + +a = plt.figure() +plt.imshow(grain_map[0], cmap='jet') + +b = plt.figure() +plt.imshow(confidence_map[0], vmin=0.45, vmax=1) +plt.colorbar() #============================================================================== # %% SAVE PROCESSED GRAIN MAP DATA #============================================================================== -nfutil.save_nf_data(output_dir,output_stem,grain_map,confidence_map,Xs,Ys,Zs,experiment.exp_maps,id_remap=nf_to_ff_id_map) +nfutil.save_nf_data(output_dir, output_stem, grain_map, confidence_map, + Xs, Ys, Zs, experiment.exp_maps, id_remap=nf_to_ff_id_map) -#============================================================================== -# %% CHECKING OUTPUT -#============================================================================== +# np.savez_compressed(output_dir + output_stem + '_raw_conf.npz', raw_confidence_full) -if matplot: - if output_plot_check: - - - plt.figure(1) - plt.imshow(confidence_map[0]) - plt.title('Bottom Layer Confidence') - - plt.figure(2) - plt.imshow(confidence_map[-1]) - plt.title('Bottom Layer Confidence') - - plt.figure(3) - nfutil.plot_ori_map(grain_map, confidence_map, experiment.exp_maps, 0,id_remap=nf_to_ff_id_map) - plt.title('Top Layer Grain Map') - - plt.figure(4) - nfutil.plot_ori_map(grain_map, confidence_map, experiment.exp_maps, -1,id_remap=nf_to_ff_id_map) - plt.title('Top Layer Grain Map') - - plt.show() - - +# #============================================================================== +# # %% CHECKING OUTPUT +# #============================================================================== + +# import matplotlib.pyplot as plt + +# plt.figure(1) +# plt.imshow(confidence_map[0]) + +# plt.figure(2) +# plt.imshow(confidence_map[-1]) + +# plt.figure(3) +# plt.imshow(grain_map[0]) +# plt.figure(4) +# plt.imshow(grain_map[-1]) From 0b47852708a008970940628aedd9e231e0a27b11 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Thu, 11 Aug 2022 14:53:57 -0400 Subject: [PATCH 3/7] finished fixing NF processing script with grouping --- scripts/process_nf.py | 302 ++++++++++++++++++++++-------------------- 1 file changed, 158 insertions(+), 144 deletions(-) diff --git a/scripts/process_nf.py b/scripts/process_nf.py index d9390d3a4..5f8900025 100644 --- a/scripts/process_nf.py +++ b/scripts/process_nf.py @@ -15,11 +15,20 @@ import multiprocessing as mp import os +import logging -from matplotlib import pyplot as plt from hexrd.grainmap import nfutil +try: + import matplotlib.pyplot as plt + matplot=True +except(ImportError): + logging.warning(f'no matplotlib, debug plotting disabled') + matplot=False + + + #============================================================================== # %% FILES TO LOAD -CAN BE EDITED #============================================================================== @@ -29,158 +38,176 @@ #A materials file, is a cPickle file which contains material information like lattice #parameters necessary for the reconstruction -main_dir = '/nfs/chess/user/relim/pagan-3274-a/ti7al_bruker' -# main_dir = "/Users/rachellim/Documents/PSU/CHESS_2022Feb/mono_study" -det_file = main_dir + '/retiga.yml' -mat_file = main_dir + '/materials.h5' +main_dir = '/INSERT/WORKDIR/' + +det_file = main_dir + 'retiga.yml' +mat_file= main_dir + 'materials.h5' #============================================================================== # %% OUTPUT INFO -CAN BE EDITED #============================================================================== -output_dir = '/nfs/chess/user/relim/NF_chunking' +output_dir = main_dir #============================================================================== # %% NEAR FIELD DATA FILES -CAN BE EDITED - ZERO LOAD SCAN #============================================================================== #These are the near field data files used for the reconstruction, a grains.out file -#from the far field analaysis is used as orientation guess for the grid that will -grain_out_file = main_dir + '/ti7al-6_grains_best.out' -# grain_out_file = '/Users/rachellim/Documents/PSU/CHESS_2022Feb/mono_study/ti7al-6_grains.out' - +#from the far field analaysis is used as orientation guess for the grid that will +grain_out_file = '/LOC/grains.out' #%% +stem='nf_' + #Locations of near field images -data_folder = '/nfs/chess/raw/2022-1/id3a/pagan-3274-a/ti7al-5/1/nf' +data_folder='/INSERT/nf/' -img_start = 106188 # whatever you want the first frame to be -nframes = 1440 -img_nums = np.arange(img_start, img_start+nframes, 1) +img_start=6 #whatever you want the first frame to be +nframes=1440 +img_nums=np.arange(img_start,img_start+nframes,1) -output_stem = '/ti7al-6_nf_scan9' -#corresponding scan number +output_stem='NAME_VOL_X' #============================================================================== # %% USER OPTIONS -CAN BE EDITED #============================================================================== ### material for the reconstruction -mat_name = 'Ti7Al' -max_tth = None # degrees, if None is input max tth will be set by the geometry +mat_name='INSERT' +max_tth=None #degrees, if None is input max tth will be set by the geometry #reconstruction with misorientation included, for many grains, this will quickly #make the reconstruction size unmanagable -misorientation_bnd = 0.0 # degrees -misorientation_spacing = 0.2 # degrees +misorientation_bnd=0.0 #degrees +misorientation_spacing=0.25 #degrees #####image processing parameters -num_for_dark = 250 # num images to use for median data +num_for_dark=250#num images to use for median data -process_type = 'gaussian' -sigma = 4.5 -size = 5 -process_args = np.array([sigma, size]) +process_type='gaussian' +sigma=2.0 +size=3.0 +process_args=np.array([sigma,size]) # process_type='dilations_only' # num_erosions=2 #num iterations of images erosion, don't mess with unless you know what you're doing # num_dilations=3 #num iterations of images erosion, don't mess with unless you know what you're doing # process_args=np.array([num_erosions,num_dilations]) -threshold = 1.5 -# num iterations of 3d image stack dilations, don't mess with unless you know what you're doing -ome_dilation_iter = 1 +threshold=1.5 +ome_dilation_iter=1 #num iterations of 3d image stack dilations, don't mess with unless you know what you're doing #thresholds for grains in reconstructions -# only use orientations from grains with completnesses ABOVE this threshold -comp_thresh = 0.0 -chi2_thresh = 1.0 # only use orientations from grains BELOW this chi^2 +comp_thresh=0.0 #only use orientations from grains with completnesses ABOVE this threshold +chi2_thresh=1.0 #only use orientations from grains BELOW this chi^2 ######reconstruction parameters -# ome_range_deg=[(0.,359.75)] #degrees -ome_range_deg = [(-0.55, 359.2)] # degrees +ome_range_deg=[(0.,359.75)] #degrees -use_mask = False +use_mask=True #Mask info, used if use_mask=True -mask_data_file = 'tomo_mask.npz' -# this is generally the difference in y motor positions between the tomo and nf layer (tomo_motor_z-nf_motor_z), needed for registry -mask_vert_offset = 0 +mask_data_file='/LOC/tomo_mask.npz' +mask_vert_offset=-0.3 #this is generally the difference in y motor positions between the tomo and nf layer (tomo_motor_z-nf_motor_z), needed for registry #these values will be used if no mask data is provided (use_mask=False) (no tomography) -# cross sectional to reconstruct (should be at least 20%-30% over sample width) -cross_sectional_dim = 1.35 -voxel_spacing = 0.002 # in mm, voxel spacing for the near field reconstruction +cross_sectional_dim=1.35 #cross sectional to reconstruct (should be at least 20%-30% over sample width) +voxel_spacing = 0.005 #in mm, voxel spacing for the near field reconstruction ##vertical (y) reconstruction voxel bounds in mm, ALWAYS USED REGARDLESS OF MASK -#v_bnds=[-0.0025,0.0025] #(mm) if bounds are equal, a single layer is produced -# v_bnds = [-0.085,0.085] -v_bnds = [0, 0] +v_bnds=[-0.085,0.085] #(mm) if bounds are equal, a single layer is produced + +beam_stop_y_cen=0.0 #mm, measured from the origin of the detector paramters +beam_stop_width=0.6 #mm, width of the beam stop verticallu +beam_stop_parms=np.array([beam_stop_y_cen,beam_stop_width]) -beam_stop_y_cen = 0.0 # mm, measured from the origin of the detector paramters -beam_stop_width = 0.6 # mm, width of the beam stop verticallu -beam_stop_parms = np.array([beam_stop_y_cen, beam_stop_width]) +max_RAM = 256 #max amount of memory to available in GB ##### multiprocessing controller parameters -check = None -limit = None -generate = None -ncpus = mp.cpu_count() -# chunksize for multiprocessing, don't mess with unless you know what you're doing -chunk_size = 500 +check=None +limit=None +generate=None +ncpus=mp.cpu_count() +chunk_size=500#chunksize for multiprocessing, don't mess with unless you know what you're doing + + +#### debug plotting +output_plot_check=True #============================================================================== # %% LOAD GRAIN AND EXPERIMENT DATA #============================================================================== -experiment, nf_to_ff_id_map = nfutil.gen_trial_exp_data(grain_out_file, det_file, mat_file, mat_name, max_tth, comp_thresh, chi2_thresh, misorientation_bnd, - misorientation_spacing, ome_range_deg, nframes, beam_stop_parms) - - +experiment, nf_to_ff_id_map = nfutil.gen_trial_exp_data(grain_out_file,det_file,\ + mat_file, mat_name, max_tth, \ + comp_thresh, chi2_thresh, misorientation_bnd, \ + misorientation_spacing,ome_range_deg, \ + nframes, beam_stop_parms) + + #============================================================================== # %% LOAD / GENERATE TEST DATA #============================================================================== -if use_mask: - mask_data = np.load(mask_data_file) - - mask_full = mask_data['mask'] - Xs_mask = mask_data['Xs'] - Ys_mask = mask_data['Ys']-(mask_vert_offset) - Zs_mask = mask_data['Zs'] - voxel_spacing = mask_data['voxel_spacing'] - tomo_layer_centers = np.squeeze(Ys_mask[:, 0, 0]) - above = np.where(tomo_layer_centers > v_bnds[0]) - below = np.where(tomo_layer_centers < v_bnds[1]) +if use_mask: + mask_data=np.load(mask_data_file) + + + mask_full=mask_data['mask'] + Xs_mask=mask_data['Xs'] + Ys_mask=mask_data['Ys']-(mask_vert_offset) + Zs_mask=mask_data['Zs'] + voxel_spacing=mask_data['voxel_spacing'] + + + tomo_layer_centers=np.squeeze(Ys_mask[:,0,0]) #need to think about how to handle a single layer in this context + above=np.where(tomo_layer_centers>=v_bnds[0]) + below=np.where(tomo_layer_centers 3.4 we could use multiprocessing get_start_method and # set_start_method for a cleaner implementation of this functionality. -multiprocessing_start_method = 'fork' if hasattr(os, 'fork') else 'spawn' +multiprocessing_start_method = 'fork' if hasattr(os, 'fork') else 'spawn' + +controller=nfutil.build_controller(ncpus=ncpus,chunk_size=chunk_size,check=check,generate=generate,limit=limit) + -controller = nfutil.build_controller( - ncpus=ncpus, chunk_size=chunk_size, check=check, generate=generate, limit=limit) +#============================================================================== +# %% TEST ORIENTATIONS +#============================================================================== + +## Would be nice to pack the images, quant_and_clip will need to be edited if this is added, dcp 6.2.2021 +#packed_image_stack = nfutil.dilate_image_stack(image_stack, experiment,controller) +print('Testing Orientations...') #%% Test orientations in groups @@ -303,49 +313,53 @@ confidence_map = confidence_map_list.reshape(Xs.shape) -#============================================================================== -# %% POST PROCESS W WHEN TOMOGRAPHY HAS BEEN USED -#============================================================================== +del controller + -#note all masking is already handled by not evaluating specific points -grain_map, confidence_map = nfutil.process_raw_confidence( - raw_confidence_full, Xs.shape, id_remap=nf_to_ff_id_map, min_thresh=0.0) -#%% + +#============================================================================== +# %% POST PROCESS W WHEN TOMOGRAPHY HAS BEEN USED +#============================================================================== -a = plt.figure() -plt.imshow(grain_map[0], cmap='jet') +#note all masking is already handled by not evaluating specific points +grain_map, confidence_map = nfutil.process_raw_confidence(raw_confidence_full,Xs.shape,id_remap=nf_to_ff_id_map,min_thresh=0.0) -b = plt.figure() -plt.imshow(confidence_map[0], vmin=0.45, vmax=1) -plt.colorbar() #============================================================================== # %% SAVE PROCESSED GRAIN MAP DATA #============================================================================== -nfutil.save_nf_data(output_dir, output_stem, grain_map, confidence_map, - Xs, Ys, Zs, experiment.exp_maps, id_remap=nf_to_ff_id_map) - -# np.savez_compressed(output_dir + output_stem + '_raw_conf.npz', raw_confidence_full) +nfutil.save_nf_data(output_dir,output_stem,grain_map,confidence_map,Xs,Ys,Zs,experiment.exp_maps,id_remap=nf_to_ff_id_map) +#============================================================================== +# %% CHECKING OUTPUT +#============================================================================== -# #============================================================================== -# # %% CHECKING OUTPUT -# #============================================================================== - -# import matplotlib.pyplot as plt - -# plt.figure(1) -# plt.imshow(confidence_map[0]) - -# plt.figure(2) -# plt.imshow(confidence_map[-1]) +if matplot: + if output_plot_check: + + + plt.figure(1) + plt.imshow(confidence_map[0]) + plt.title('Bottom Layer Confidence') + + plt.figure(2) + plt.imshow(confidence_map[-1]) + plt.title('Bottom Layer Confidence') + + plt.figure(3) + nfutil.plot_ori_map(grain_map, confidence_map, experiment.exp_maps, 0,id_remap=nf_to_ff_id_map) + plt.title('Top Layer Grain Map') + + plt.figure(4) + nfutil.plot_ori_map(grain_map, confidence_map, experiment.exp_maps, -1,id_remap=nf_to_ff_id_map) + plt.title('Top Layer Grain Map') + + plt.show() + + -# plt.figure(3) -# plt.imshow(grain_map[0]) -# plt.figure(4) -# plt.imshow(grain_map[-1]) From c3fce94954003781aba5993326e18ceca49ce2a4 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Thu, 1 Sep 2022 13:33:44 -0400 Subject: [PATCH 4/7] added missing if statement to check if there are leftover voxels --- scripts/process_nf.py | 57 ++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/scripts/process_nf.py b/scripts/process_nf.py index 5f8900025..164a27b30 100644 --- a/scripts/process_nf.py +++ b/scripts/process_nf.py @@ -225,14 +225,19 @@ bytes_for_array = bits_for_arrays/8. n_groups = np.floor(bytes_for_array/RAM_to_use) # number of full groups -leftover_voxels = np.mod(n_voxels, n_groups) -print('Splitting data into %d groups with %d leftover voxels' %(int(n_groups),int(leftover_voxels)) +voxels_per_group = np.floor(n_voxels/n_groups) +leftover_voxels = n_voxels - (voxels_per_group * n_groups) + + -grouped_voxels = n_voxels - leftover_voxels -voxels_per_group = grouped_voxels/n_groups +print('Splitting data into %d groups with %d leftover voxels' %(int(n_groups),int(leftover_voxels)) + +#leftover_voxels = np.mod(n_voxels, n_groups) +#grouped_voxels = n_voxels - leftover_voxels +#voxels_per_group = grouped_voxels/n_groups #============================================================================== # %% BUILD MP CONTROLLER @@ -278,44 +283,40 @@ confidence_map_list = np.zeros(n_voxels) # test voxels in groups - for abcd in range(int(n_groups)): + for group in range(int(n_groups)): voxels_to_test = test_crds[int( - abcd) * int(voxels_per_group):int(abcd + 1) * int(voxels_per_group), :] - print('Calculating group %d' % abcd) + group) * int(voxels_per_group):int(group + 1) * int(voxels_per_group), :] + print('Calculating group %d' % group) raw_confidence = nfutil.test_orientations( image_stack, experiment, voxels_to_test, controller, multiprocessing_start_method) - print('Calculated raw confidence group %d' % abcd) + print('Calculated raw confidence group %d' % group) grain_map_group_list, confidence_map_group_list = nfutil.process_raw_confidence( raw_confidence, id_remap=nf_to_ff_id_map, min_thresh=0.0) grain_map_list[int( - abcd) * int(voxels_per_group):int(abcd + 1) * int(voxels_per_group)] = grain_map_group_list + group) * int(voxels_per_group):int(group + 1) * int(voxels_per_group)] = grain_map_group_list confidence_map_list[int( - abcd) * int(voxels_per_group):int(abcd + 1) * int(voxels_per_group)] = confidence_map_group_list + group) * int(voxels_per_group):int(group + 1) * int(voxels_per_group)] = confidence_map_group_list #now for the leftover voxels - voxels_to_test = test_crds[int( - n_groups) * int(voxels_per_group):, :] - raw_confidence = nfutil.test_orientations( - image_stack, experiment, voxels_to_test, controller, multiprocessing_start_method) - grain_map_group_list, confidence_map_group_list = nfutil.process_raw_confidence( - raw_confidence, id_remap=nf_to_ff_id_map, min_thresh=0.0) - - grain_map_list[int( - n_groups) * int(voxels_per_group):] = grain_map_group_list - - confidence_map_list[int( - n_groups) * int(voxels_per_group):] = confidence_map_group_list + if leftover_voxels != 0: + voxels_to_test = test_crds[int( + n_groups) * int(voxels_per_group):, :] + raw_confidence = nfutil.test_orientations( + image_stack, experiment, voxels_to_test, controller, multiprocessing_start_method) + grain_map_group_list, confidence_map_group_list = nfutil.process_raw_confidence( + raw_confidence, id_remap=nf_to_ff_id_map, min_thresh=0.0) + + grain_map_list[int( + n_groups) * int(voxels_per_group):] = grain_map_group_list + + confidence_map_list[int( + n_groups) * int(voxels_per_group):] = confidence_map_group_list #reshape them grain_map = grain_map_list.reshape(Xs.shape) - confidence_map = confidence_map_list.reshape(Xs.shape) - - - -del controller - + confidence_map = confidence_map_list.reshape(Xs.shape) From 0f23be810cc189175046debc8ee6f0b76ddd05e8 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Tue, 4 Oct 2022 18:25:24 -0400 Subject: [PATCH 5/7] fixed manual memory entry for NF --- scripts/process_nf.py | 96 +++++++++++++++++++++---------------------- 1 file changed, 47 insertions(+), 49 deletions(-) diff --git a/scripts/process_nf.py b/scripts/process_nf.py index 5f8900025..fc7b8f88d 100644 --- a/scripts/process_nf.py +++ b/scripts/process_nf.py @@ -16,7 +16,7 @@ import os import logging - +import psutil from hexrd.grainmap import nfutil @@ -41,20 +41,20 @@ main_dir = '/INSERT/WORKDIR/' det_file = main_dir + 'retiga.yml' -mat_file= main_dir + 'materials.h5' +mat_file= main_dir + 'materials.h5' #============================================================================== # %% OUTPUT INFO -CAN BE EDITED #============================================================================== -output_dir = main_dir +output_dir = main_dir #============================================================================== # %% NEAR FIELD DATA FILES -CAN BE EDITED - ZERO LOAD SCAN #============================================================================== #These are the near field data files used for the reconstruction, a grains.out file -#from the far field analaysis is used as orientation guess for the grid that will +#from the far field analaysis is used as orientation guess for the grid that will grain_out_file = '/LOC/grains.out' #%% @@ -62,9 +62,9 @@ stem='nf_' #Locations of near field images -data_folder='/INSERT/nf/' +data_folder='/INSERT/nf/' -img_start=6 #whatever you want the first frame to be +img_start=6 #whatever you want the first frame to be nframes=1440 img_nums=np.arange(img_start,img_start+nframes,1) @@ -80,7 +80,7 @@ #reconstruction with misorientation included, for many grains, this will quickly #make the reconstruction size unmanagable -misorientation_bnd=0.0 #degrees +misorientation_bnd=0.0 #degrees misorientation_spacing=0.25 #degrees @@ -106,7 +106,7 @@ ######reconstruction parameters -ome_range_deg=[(0.,359.75)] #degrees +ome_range_deg=[(0.,359.75)] #degrees use_mask=True #Mask info, used if use_mask=True @@ -125,9 +125,6 @@ beam_stop_width=0.6 #mm, width of the beam stop verticallu beam_stop_parms=np.array([beam_stop_y_cen,beam_stop_width]) - -max_RAM = 256 #max amount of memory to available in GB - ##### multiprocessing controller parameters check=None limit=None @@ -135,6 +132,8 @@ ncpus=mp.cpu_count() chunk_size=500#chunksize for multiprocessing, don't mess with unless you know what you're doing +RAM_set = False #if True, manually set max amount of ram +max_RAM = 256 #only used if RAM_set is true. in GB #### debug plotting output_plot_check=True @@ -149,8 +148,8 @@ comp_thresh, chi2_thresh, misorientation_bnd, \ misorientation_spacing,ome_range_deg, \ nframes, beam_stop_parms) - - + + #============================================================================== # %% LOAD / GENERATE TEST DATA #============================================================================== @@ -159,32 +158,32 @@ if use_mask: mask_data=np.load(mask_data_file) - - + + mask_full=mask_data['mask'] Xs_mask=mask_data['Xs'] Ys_mask=mask_data['Ys']-(mask_vert_offset) Zs_mask=mask_data['Zs'] voxel_spacing=mask_data['voxel_spacing'] - - + + tomo_layer_centers=np.squeeze(Ys_mask[:,0,0]) #need to think about how to handle a single layer in this context above=np.where(tomo_layer_centers>=v_bnds[0]) below=np.where(tomo_layer_centers 3.4 we could use multiprocessing get_start_method and # set_start_method for a cleaner implementation of this functionality. -multiprocessing_start_method = 'fork' if hasattr(os, 'fork') else 'spawn' - +multiprocessing_start_method = 'fork' if hasattr(os, 'fork') else 'spawn' + + +controller=nfutil.build_controller(ncpus=ncpus,chunk_size=chunk_size,check=check,generate=generate,limit=limit) -controller=nfutil.build_controller(ncpus=ncpus,chunk_size=chunk_size,check=check,generate=generate,limit=limit) - #============================================================================== # %% TEST ORIENTATIONS -#============================================================================== - +#============================================================================== + ## Would be nice to pack the images, quant_and_clip will need to be edited if this is added, dcp 6.2.2021 #packed_image_stack = nfutil.dilate_image_stack(image_stack, experiment,controller) @@ -307,19 +309,19 @@ confidence_map_list[int( n_groups) * int(voxels_per_group):] = confidence_map_group_list - + #reshape them grain_map = grain_map_list.reshape(Xs.shape) confidence_map = confidence_map_list.reshape(Xs.shape) -del controller - +del controller + + - #============================================================================== # %% POST PROCESS W WHEN TOMOGRAPHY HAS BEEN USED #============================================================================== @@ -340,26 +342,22 @@ if matplot: if output_plot_check: - - + + plt.figure(1) plt.imshow(confidence_map[0]) plt.title('Bottom Layer Confidence') - + plt.figure(2) plt.imshow(confidence_map[-1]) plt.title('Bottom Layer Confidence') - + plt.figure(3) nfutil.plot_ori_map(grain_map, confidence_map, experiment.exp_maps, 0,id_remap=nf_to_ff_id_map) plt.title('Top Layer Grain Map') - + plt.figure(4) nfutil.plot_ori_map(grain_map, confidence_map, experiment.exp_maps, -1,id_remap=nf_to_ff_id_map) plt.title('Top Layer Grain Map') - - plt.show() - - - + plt.show() From 2b6ecf32c39a24aa59a3e86fae18ad69a5de488f Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Tue, 4 Oct 2022 18:44:05 -0400 Subject: [PATCH 6/7] PEP8 --- scripts/process_nf.py | 197 ++++++++++++++++++++++-------------------- 1 file changed, 102 insertions(+), 95 deletions(-) diff --git a/scripts/process_nf.py b/scripts/process_nf.py index fc7b8f88d..0d840d953 100644 --- a/scripts/process_nf.py +++ b/scripts/process_nf.py @@ -22,11 +22,10 @@ try: import matplotlib.pyplot as plt - matplot=True + matplot = True except(ImportError): logging.warning(f'no matplotlib, debug plotting disabled') - matplot=False - + matplot = False #============================================================================== @@ -41,7 +40,7 @@ main_dir = '/INSERT/WORKDIR/' det_file = main_dir + 'retiga.yml' -mat_file= main_dir + 'materials.h5' +mat_file = main_dir + 'materials.h5' #============================================================================== # %% OUTPUT INFO -CAN BE EDITED @@ -59,95 +58,102 @@ #%% -stem='nf_' +stem = 'nf_' #Locations of near field images -data_folder='/INSERT/nf/' +data_folder = '/INSERT/nf/' -img_start=6 #whatever you want the first frame to be -nframes=1440 -img_nums=np.arange(img_start,img_start+nframes,1) +img_start = 6 # whatever you want the first frame to be +nframes = 1440 +img_nums = np.arange(img_start, img_start+nframes, 1) -output_stem='NAME_VOL_X' +output_stem = 'NAME_VOL_X' #============================================================================== # %% USER OPTIONS -CAN BE EDITED #============================================================================== ### material for the reconstruction -mat_name='INSERT' -max_tth=None #degrees, if None is input max tth will be set by the geometry +mat_name = 'INSERT' +max_tth = None # degrees, if None is input max tth will be set by the geometry #reconstruction with misorientation included, for many grains, this will quickly #make the reconstruction size unmanagable -misorientation_bnd=0.0 #degrees -misorientation_spacing=0.25 #degrees +misorientation_bnd = 0.0 # degrees +misorientation_spacing = 0.25 # degrees #####image processing parameters -num_for_dark=250#num images to use for median data +num_for_dark = 250 # num images to use for median data -process_type='gaussian' -sigma=2.0 -size=3.0 -process_args=np.array([sigma,size]) +img_threshold = 0 +process_type = 'gaussian' +sigma = 2.0 +size = 3.0 +process_args = np.array([sigma, size]) # process_type='dilations_only' # num_erosions=2 #num iterations of images erosion, don't mess with unless you know what you're doing # num_dilations=3 #num iterations of images erosion, don't mess with unless you know what you're doing # process_args=np.array([num_erosions,num_dilations]) -threshold=1.5 -ome_dilation_iter=1 #num iterations of 3d image stack dilations, don't mess with unless you know what you're doing +threshold = 1.5 +# num iterations of 3d image stack dilations, don't mess with unless you know what you're doing +ome_dilation_iter = 1 #thresholds for grains in reconstructions -comp_thresh=0.0 #only use orientations from grains with completnesses ABOVE this threshold -chi2_thresh=1.0 #only use orientations from grains BELOW this chi^2 +# only use orientations from grains with completnesses ABOVE this threshold +comp_thresh = 0.0 +chi2_thresh = 1.0 # only use orientations from grains BELOW this chi^2 ######reconstruction parameters -ome_range_deg=[(0.,359.75)] #degrees +ome_range_deg = [(0., 359.75)] # degrees -use_mask=True +use_mask = True #Mask info, used if use_mask=True -mask_data_file='/LOC/tomo_mask.npz' -mask_vert_offset=-0.3 #this is generally the difference in y motor positions between the tomo and nf layer (tomo_motor_z-nf_motor_z), needed for registry +mask_data_file = '/LOC/tomo_mask.npz' +# this is generally the difference in y motor positions between the tomo and nf layer (tomo_motor_z-nf_motor_z), needed for registry +mask_vert_offset = -0.3 #these values will be used if no mask data is provided (use_mask=False) (no tomography) -cross_sectional_dim=1.35 #cross sectional to reconstruct (should be at least 20%-30% over sample width) -voxel_spacing = 0.005 #in mm, voxel spacing for the near field reconstruction +# cross sectional to reconstruct (should be at least 20%-30% over sample width) +cross_sectional_dim = 1.35 +voxel_spacing = 0.005 # in mm, voxel spacing for the near field reconstruction ##vertical (y) reconstruction voxel bounds in mm, ALWAYS USED REGARDLESS OF MASK -v_bnds=[-0.085,0.085] #(mm) if bounds are equal, a single layer is produced +# (mm) if bounds are equal, a single layer is produced +v_bnds = [-0.085, 0.085] -beam_stop_y_cen=0.0 #mm, measured from the origin of the detector paramters -beam_stop_width=0.6 #mm, width of the beam stop verticallu -beam_stop_parms=np.array([beam_stop_y_cen,beam_stop_width]) +beam_stop_y_cen = 0.0 # mm, measured from the origin of the detector paramters +beam_stop_width = 0.6 # mm, width of the beam stop verticallu +beam_stop_parms = np.array([beam_stop_y_cen, beam_stop_width]) ##### multiprocessing controller parameters -check=None -limit=None -generate=None -ncpus=mp.cpu_count() -chunk_size=500#chunksize for multiprocessing, don't mess with unless you know what you're doing +check = None +limit = None +generate = None +ncpus = mp.cpu_count() +# chunksize for multiprocessing, don't mess with unless you know what you're doing +chunk_size = 500 -RAM_set = False #if True, manually set max amount of ram -max_RAM = 256 #only used if RAM_set is true. in GB +RAM_set = False # if True, manually set max amount of ram +max_RAM = 256 # only used if RAM_set is true. in GB #### debug plotting -output_plot_check=True +output_plot_check = True #============================================================================== # %% LOAD GRAIN AND EXPERIMENT DATA #============================================================================== -experiment, nf_to_ff_id_map = nfutil.gen_trial_exp_data(grain_out_file,det_file,\ - mat_file, mat_name, max_tth, \ - comp_thresh, chi2_thresh, misorientation_bnd, \ - misorientation_spacing,ome_range_deg, \ - nframes, beam_stop_parms) +experiment, nf_to_ff_id_map = nfutil.gen_trial_exp_data(grain_out_file, det_file, + mat_file, mat_name, max_tth, + comp_thresh, chi2_thresh, misorientation_bnd, + misorientation_spacing, ome_range_deg, + nframes, beam_stop_parms) #============================================================================== @@ -155,63 +161,61 @@ #============================================================================== - if use_mask: - mask_data=np.load(mask_data_file) - - - mask_full=mask_data['mask'] - Xs_mask=mask_data['Xs'] - Ys_mask=mask_data['Ys']-(mask_vert_offset) - Zs_mask=mask_data['Zs'] - voxel_spacing=mask_data['voxel_spacing'] - + mask_data = np.load(mask_data_file) - tomo_layer_centers=np.squeeze(Ys_mask[:,0,0]) #need to think about how to handle a single layer in this context - above=np.where(tomo_layer_centers>=v_bnds[0]) - below=np.where(tomo_layer_centers= v_bnds[0]) + below = np.where(tomo_layer_centers < v_bnds[1]) - in_bnds=np.intersect1d(above,below) + in_bnds = np.intersect1d(above, below) - mask=mask_full[in_bnds] - Xs=Xs_mask[in_bnds] - Ys=Ys_mask[in_bnds] - Zs=Zs_mask[in_bnds] + mask = mask_full[in_bnds] + Xs = Xs_mask[in_bnds] + Ys = Ys_mask[in_bnds] + Zs = Zs_mask[in_bnds] test_crds_full = np.vstack([Xs.flatten(), Ys.flatten(), Zs.flatten()]).T n_crds = len(test_crds_full) - to_use=np.where(mask.flatten())[0] + to_use = np.where(mask.flatten())[0] else: - test_crds_full, n_crds, Xs, Ys, Zs = nfutil.gen_nf_test_grid(cross_sectional_dim, v_bnds, voxel_spacing) - to_use=np.arange(len(test_crds_full)) - -test_crds=test_crds_full[to_use,:] + test_crds_full, n_crds, Xs, Ys, Zs = nfutil.gen_nf_test_grid( + cross_sectional_dim, v_bnds, voxel_spacing) + to_use = np.arange(len(test_crds_full)) +test_crds = test_crds_full[to_use, :] #============================================================================== # %% NEAR FIELD - MAKE MEDIAN DARK #============================================================================== -dark=nfutil.gen_nf_dark(data_folder,img_nums,num_for_dark,experiment.nrows,experiment.ncols,dark_type='median',num_digits=6,stem=stem) +dark = nfutil.gen_nf_dark(data_folder, img_nums, num_for_dark, experiment.nrows, + experiment.ncols, dark_type='median', num_digits=6, stem=stem) #============================================================================== # %% NEAR FIELD - LOAD IMAGE DATA AND PROCESS #============================================================================== -image_stack=nfutil.gen_nf_cleaned_image_stack(data_folder,img_nums,dark,experiment.nrows,experiment.ncols,process_type=process_type,\ - process_args=process_args,threshold=img_threshold,ome_dilation_iter=ome_dilation_iter,num_digits=6,stem=stem) +image_stack = nfutil.gen_nf_cleaned_image_stack(data_folder, img_nums, dark, experiment.nrows, experiment.ncols, process_type=process_type, + process_args=process_args, threshold=img_threshold, ome_dilation_iter=ome_dilation_iter, num_digits=6, stem=stem) #============================================================================== # %% NEAR FIELD - splitting #============================================================================== -if RAM_set = True: +if RAM_set is True: RAM = max_RAM * 1e9 else: RAM = psutil.virtual_memory().available # in GB @@ -229,7 +233,8 @@ n_groups = np.floor(bytes_for_array/RAM_to_use) # number of full groups leftover_voxels = np.mod(n_voxels, n_groups) -print('Splitting data into %d groups with %d leftover voxels' %(int(n_groups),int(leftover_voxels)) +print('Splitting data into %d groups with %d leftover voxels' % + (int(n_groups), int(leftover_voxels))) grouped_voxels = n_voxels - leftover_voxels @@ -246,7 +251,8 @@ multiprocessing_start_method = 'fork' if hasattr(os, 'fork') else 'spawn' -controller=nfutil.build_controller(ncpus=ncpus,chunk_size=chunk_size,check=check,generate=generate,limit=limit) +controller = nfutil.build_controller( + ncpus=ncpus, chunk_size=chunk_size, check=check, generate=generate, limit=limit) #============================================================================== @@ -295,46 +301,46 @@ confidence_map_list[int( abcd) * int(voxels_per_group):int(abcd + 1) * int(voxels_per_group)] = confidence_map_group_list + del raw_confidence - #now for the leftover voxels - voxels_to_test = test_crds[int( - n_groups) * int(voxels_per_group):, :] - raw_confidence = nfutil.test_orientations( - image_stack, experiment, voxels_to_test, controller, multiprocessing_start_method) - grain_map_group_list, confidence_map_group_list = nfutil.process_raw_confidence( - raw_confidence, id_remap=nf_to_ff_id_map, min_thresh=0.0) + if leftover_voxels > 0: + #now for the leftover voxels + voxels_to_test = test_crds[int( + n_groups) * int(voxels_per_group):, :] + raw_confidence = nfutil.test_orientations( + image_stack, experiment, voxels_to_test, controller, multiprocessing_start_method) + grain_map_group_list, confidence_map_group_list = nfutil.process_raw_confidence( + raw_confidence, id_remap=nf_to_ff_id_map, min_thresh=0.0) - grain_map_list[int( - n_groups) * int(voxels_per_group):] = grain_map_group_list + grain_map_list[int( + n_groups) * int(voxels_per_group):] = grain_map_group_list - confidence_map_list[int( - n_groups) * int(voxels_per_group):] = confidence_map_group_list + confidence_map_list[int( + n_groups) * int(voxels_per_group):] = confidence_map_group_list #reshape them grain_map = grain_map_list.reshape(Xs.shape) confidence_map = confidence_map_list.reshape(Xs.shape) - del controller - - - #============================================================================== # %% POST PROCESS W WHEN TOMOGRAPHY HAS BEEN USED #============================================================================== #note all masking is already handled by not evaluating specific points -grain_map, confidence_map = nfutil.process_raw_confidence(raw_confidence_full,Xs.shape,id_remap=nf_to_ff_id_map,min_thresh=0.0) +grain_map, confidence_map = nfutil.process_raw_confidence( + raw_confidence_full, Xs.shape, id_remap=nf_to_ff_id_map, min_thresh=0.0) #============================================================================== # %% SAVE PROCESSED GRAIN MAP DATA #============================================================================== -nfutil.save_nf_data(output_dir,output_stem,grain_map,confidence_map,Xs,Ys,Zs,experiment.exp_maps,id_remap=nf_to_ff_id_map) +nfutil.save_nf_data(output_dir, output_stem, grain_map, confidence_map, + Xs, Ys, Zs, experiment.exp_maps, id_remap=nf_to_ff_id_map) #============================================================================== # %% CHECKING OUTPUT @@ -343,7 +349,6 @@ if matplot: if output_plot_check: - plt.figure(1) plt.imshow(confidence_map[0]) plt.title('Bottom Layer Confidence') @@ -353,11 +358,13 @@ plt.title('Bottom Layer Confidence') plt.figure(3) - nfutil.plot_ori_map(grain_map, confidence_map, experiment.exp_maps, 0,id_remap=nf_to_ff_id_map) + nfutil.plot_ori_map(grain_map, confidence_map, + experiment.exp_maps, 0, id_remap=nf_to_ff_id_map) plt.title('Top Layer Grain Map') plt.figure(4) - nfutil.plot_ori_map(grain_map, confidence_map, experiment.exp_maps, -1,id_remap=nf_to_ff_id_map) + nfutil.plot_ori_map(grain_map, confidence_map, + experiment.exp_maps, -1, id_remap=nf_to_ff_id_map) plt.title('Top Layer Grain Map') plt.show() From 7ebd56fe326b2065eb66aec16b8270eb96ff6a24 Mon Sep 17 00:00:00 2001 From: Rachel Lim Date: Tue, 4 Oct 2022 18:59:39 -0400 Subject: [PATCH 7/7] fixed merge --- scripts/process_nf.py | 41 ++--------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) diff --git a/scripts/process_nf.py b/scripts/process_nf.py index d711f262d..f6789039c 100644 --- a/scripts/process_nf.py +++ b/scripts/process_nf.py @@ -231,30 +231,15 @@ bytes_for_array = bits_for_arrays/8. n_groups = np.floor(bytes_for_array/RAM_to_use) # number of full groups +leftover_voxels = np.mod(n_voxels, n_groups) -<<<<<<< HEAD print('Splitting data into %d groups with %d leftover voxels' % (int(n_groups), int(leftover_voxels))) -======= -voxels_per_group = np.floor(n_voxels/n_groups) -leftover_voxels = n_voxels - (voxels_per_group * n_groups) ->>>>>>> c3fce94954003781aba5993326e18ceca49ce2a4 +grouped_voxels = n_voxels - leftover_voxels - - -<<<<<<< HEAD voxels_per_group = grouped_voxels/n_groups - -======= -print('Splitting data into %d groups with %d leftover voxels' %(int(n_groups),int(leftover_voxels)) - -#leftover_voxels = np.mod(n_voxels, n_groups) -#grouped_voxels = n_voxels - leftover_voxels -#voxels_per_group = grouped_voxels/n_groups - ->>>>>>> c3fce94954003781aba5993326e18ceca49ce2a4 #============================================================================== # %% BUILD MP CONTROLLER #============================================================================== @@ -314,25 +299,17 @@ group) * int(voxels_per_group):int(group + 1) * int(voxels_per_group)] = grain_map_group_list confidence_map_list[int( -<<<<<<< HEAD abcd) * int(voxels_per_group):int(abcd + 1) * int(voxels_per_group)] = confidence_map_group_list del raw_confidence if leftover_voxels > 0: #now for the leftover voxels -======= - group) * int(voxels_per_group):int(group + 1) * int(voxels_per_group)] = confidence_map_group_list - - #now for the leftover voxels - if leftover_voxels != 0: ->>>>>>> c3fce94954003781aba5993326e18ceca49ce2a4 voxels_to_test = test_crds[int( n_groups) * int(voxels_per_group):, :] raw_confidence = nfutil.test_orientations( image_stack, experiment, voxels_to_test, controller, multiprocessing_start_method) grain_map_group_list, confidence_map_group_list = nfutil.process_raw_confidence( raw_confidence, id_remap=nf_to_ff_id_map, min_thresh=0.0) -<<<<<<< HEAD grain_map_list[int( n_groups) * int(voxels_per_group):] = grain_map_group_list @@ -346,20 +323,6 @@ del controller -======= - - grain_map_list[int( - n_groups) * int(voxels_per_group):] = grain_map_group_list - - confidence_map_list[int( - n_groups) * int(voxels_per_group):] = confidence_map_group_list - - #reshape them - grain_map = grain_map_list.reshape(Xs.shape) - confidence_map = confidence_map_list.reshape(Xs.shape) - ->>>>>>> c3fce94954003781aba5993326e18ceca49ce2a4 - #============================================================================== # %% POST PROCESS W WHEN TOMOGRAPHY HAS BEEN USED