diff --git a/src/CMM/costfunc/__pycache__/chi2.cpython-39.pyc b/src/CMM/costfunc/__pycache__/chi2.cpython-39.pyc deleted file mode 100644 index 7f7720e..0000000 Binary files a/src/CMM/costfunc/__pycache__/chi2.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/params.yml b/src/CMM/params.yml index 081833d..ffdde71 100644 --- a/src/CMM/params.yml +++ b/src/CMM/params.yml @@ -8,7 +8,7 @@ Foregrounds: ### This section define the in/out foregrounds contribution - fit_spectral_index: False # If False, the beta reconstruction is skipped + fit_spectral_index: True # If False, the beta reconstruction is skipped bin_mixing_matrix: 8 blind_method: 'minimize' # Fit the mixing matrix with different methods when all the components are fitted with blind method: 'PCG', 'alternate' or 'minimize' @@ -49,7 +49,7 @@ QUBIC: ### This section control the parameters related to QUBIC instrument: 'DB' # Instrumental design (DB or UWB) - npointings: 4000 # Number of time samples + npointings: 500 # Number of time samples nsub_in: 8 # Number of sub-acquisitions nsub_out: 8 # Number of sub-acquisitions for the reconstruction convolution_in: False # Angular resolution in the input TOD @@ -99,10 +99,10 @@ PCG: ### This section control PCG parameters - n_init_iter_pcg: 50 - n_iter_pcg: 50 # Number of PCG iterations + n_init_iter_pcg: 5 + n_iter_pcg: 5 # Number of PCG iterations tol_pcg: 1.0e-20 # Tolerance for PCG - n_iter_loop: 1 # Number of loop (1 loop is PCG + beta fitting + gain reconstruction) + n_iter_loop: 10 # Number of loop (1 loop is PCG + beta fitting + gain reconstruction) ites_to_converge: 1 # should be less than k tol_rms: 1.0e-10 #fix_pixels_outside_patch: True # Fixing pixels outside QUBIC patch diff --git a/src/CMM/preset/__pycache__/__init__.cpython-39.pyc b/src/CMM/preset/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index bdeb667..0000000 Binary files a/src/CMM/preset/__pycache__/__init__.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_.cpython-39.pyc deleted file mode 100644 index 535e424..0000000 Binary files a/src/CMM/preset/__pycache__/preset_.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_acquisition.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_acquisition.cpython-39.pyc deleted file mode 100644 index 5f49548..0000000 Binary files a/src/CMM/preset/__pycache__/preset_acquisition.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_components.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_components.cpython-39.pyc deleted file mode 100644 index 658b84d..0000000 Binary files a/src/CMM/preset/__pycache__/preset_components.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_external_data.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_external_data.cpython-39.pyc deleted file mode 100644 index f127298..0000000 Binary files a/src/CMM/preset/__pycache__/preset_external_data.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_foregrounds.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_foregrounds.cpython-39.pyc deleted file mode 100644 index 512e17e..0000000 Binary files a/src/CMM/preset/__pycache__/preset_foregrounds.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_gain.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_gain.cpython-39.pyc deleted file mode 100644 index b4bccf6..0000000 Binary files a/src/CMM/preset/__pycache__/preset_gain.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_mixingmatrix.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_mixingmatrix.cpython-39.pyc deleted file mode 100644 index ae11d0e..0000000 Binary files a/src/CMM/preset/__pycache__/preset_mixingmatrix.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_qubic.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_qubic.cpython-39.pyc deleted file mode 100644 index fcca11e..0000000 Binary files a/src/CMM/preset/__pycache__/preset_qubic.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_sky.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_sky.cpython-39.pyc deleted file mode 100644 index c3bf5f7..0000000 Binary files a/src/CMM/preset/__pycache__/preset_sky.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/__pycache__/preset_tools.cpython-39.pyc b/src/CMM/preset/__pycache__/preset_tools.cpython-39.pyc deleted file mode 100644 index 31cc1c9..0000000 Binary files a/src/CMM/preset/__pycache__/preset_tools.cpython-39.pyc and /dev/null differ diff --git a/src/CMM/preset/preset_components.py b/src/CMM/preset/preset_components.py index ff5b485..477164f 100644 --- a/src/CMM/preset/preset_components.py +++ b/src/CMM/preset/preset_components.py @@ -7,7 +7,7 @@ from pysimulators.interfaces.healpy import HealpixConvolutionGaussianOperator from pysm3 import utils -PATH = os.getcwd() + "/src/data/" +PATH = os.getcwd() + "/data/" class PresetComponents: @@ -195,12 +195,12 @@ def polarized_I(self, m, nside, polarization_fraction=0): # Read and downgrade the polarization angle map to the desired nside resolution polangle = hp.ud_grade( - hp.read_map(os.getcwd() + "/src/data/" + "psimap_dust90_512.fits"), nside + hp.read_map(os.getcwd() + "/data/" + "psimap_dust90_512.fits"), nside ) # Read and downgrade the depolarization map to the desired nside resolution depolmap = hp.ud_grade( - hp.read_map(os.getcwd() + "/src/data/" + "gmap_dust90_512.fits"), nside + hp.read_map(os.getcwd() + "/data/" + "gmap_dust90_512.fits"), nside ) # Calculate the cosine of twice the polarization angle diff --git a/src/CMM/preset/preset_tools.py b/src/CMM/preset/preset_tools.py index f6e35ff..012ec1e 100644 --- a/src/CMM/preset/preset_tools.py +++ b/src/CMM/preset/preset_tools.py @@ -34,7 +34,7 @@ def __init__(self, comm): ### Open parameters file self._print_message("========= Initialization =========") self._print_message(" => Reading parameters file") - with open("src/CMM/params.yml", "r") as stream: + with open("CMM/params.yml", "r") as stream: self.params = yaml.safe_load(stream) def _print_message(self, message): diff --git a/src/CMM/spectrum/__pycache__/get_spectra.cpython-39.pyc b/src/CMM/spectrum/__pycache__/get_spectra.cpython-39.pyc deleted file mode 100644 index 91d9ff2..0000000 Binary files a/src/CMM/spectrum/__pycache__/get_spectra.cpython-39.pyc and /dev/null differ diff --git a/src/FMM/__pycache__/__init__.cpython-39.pyc b/src/FMM/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 61575e1..0000000 Binary files a/src/FMM/__pycache__/__init__.cpython-39.pyc and /dev/null differ diff --git a/src/FMM/__pycache__/main.cpython-39.pyc b/src/FMM/__pycache__/main.cpython-39.pyc deleted file mode 100644 index d1178c0..0000000 Binary files a/src/FMM/__pycache__/main.cpython-39.pyc and /dev/null differ diff --git a/src/FMM/__pycache__/pipeline.cpython-39.pyc b/src/FMM/__pycache__/pipeline.cpython-39.pyc deleted file mode 100644 index 73320ef..0000000 Binary files a/src/FMM/__pycache__/pipeline.cpython-39.pyc and /dev/null differ diff --git a/src/FMM/model/__pycache__/externaldata.cpython-39.pyc b/src/FMM/model/__pycache__/externaldata.cpython-39.pyc deleted file mode 100644 index b6b8ba9..0000000 Binary files a/src/FMM/model/__pycache__/externaldata.cpython-39.pyc and /dev/null differ diff --git a/src/FMM/model/__pycache__/models.cpython-39.pyc b/src/FMM/model/__pycache__/models.cpython-39.pyc deleted file mode 100644 index 567a5d9..0000000 Binary files a/src/FMM/model/__pycache__/models.cpython-39.pyc and /dev/null differ diff --git a/src/FMM/model/__pycache__/planck_timeline.cpython-39.pyc b/src/FMM/model/__pycache__/planck_timeline.cpython-39.pyc deleted file mode 100644 index e41460f..0000000 Binary files a/src/FMM/model/__pycache__/planck_timeline.cpython-39.pyc and /dev/null differ diff --git a/src/FMM/model/externaldata.py b/src/FMM/model/externaldata.py index 6c5bdf2..92ae064 100644 --- a/src/FMM/model/externaldata.py +++ b/src/FMM/model/externaldata.py @@ -21,7 +21,7 @@ def __init__(self, file, params, noise_only=False): self.params = params - with open("src/FMM/model/noise.yml", "r") as stream: + with open("FMM/model/noise.yml", "r") as stream: self.noise = yaml.safe_load(stream) self.noise_only = noise_only @@ -175,7 +175,7 @@ def _get_ave_map(self, central_nu, bw, nb=100): return mysky * self.factor def _get_fwhm(self, nu): - fwhmi = self.read_pkl(f"src/data/Planck{nu:.0f}GHz.pkl")[f"fwhm{nu:.0f}"] + fwhmi = self.read_pkl(f"data/Planck{nu:.0f}GHz.pkl")[f"fwhm{nu:.0f}"] return fwhmi def _get_noise(self, nu): diff --git a/src/FMM/model/models.py b/src/FMM/model/models.py index 141c157..924c4be 100644 --- a/src/FMM/model/models.py +++ b/src/FMM/model/models.py @@ -37,7 +37,7 @@ def give_cl_cmb(self): """ - power_spectrum = hp.read_cl("src/data/Cls_Planck2018_lensed_scalar.fits")[ + power_spectrum = hp.read_cl("data/Cls_Planck2018_lensed_scalar.fits")[ :, :4000 ] if self.params["CMB"]["Alens"] != 1.0: @@ -46,7 +46,7 @@ def give_cl_cmb(self): power_spectrum += ( self.params["CMB"]["r"] * hp.read_cl( - "src/data/Cls_Planck2018_unlensed_scalar_and_tensor_r1.fits" + "data/Cls_Planck2018_unlensed_scalar_and_tensor_r1.fits" )[:, :4000] ) return power_spectrum diff --git a/src/FMM/model/planck_timeline.py b/src/FMM/model/planck_timeline.py index 072ab3a..65c40d9 100644 --- a/src/FMM/model/planck_timeline.py +++ b/src/FMM/model/planck_timeline.py @@ -5,7 +5,7 @@ import qubic sys.path.append(os.getcwd()) -CMB_FILE = "src/data/" +CMB_FILE = "data/" import healpy as hp import matplotlib.pyplot as plt diff --git a/src/FMM/params.yml b/src/FMM/params.yml index 7cf1472..0f19c72 100644 --- a/src/FMM/params.yml +++ b/src/FMM/params.yml @@ -17,17 +17,17 @@ QUBIC: ### This section control the parameters related to QUBIC instrument: 'DB' # Instrumental design (DB or UWB) - npointings: 1000 # Number of time samples - nsub_in: 8 # Number of sub-acquisitions - nsub_out: 8 - nrec: 2 # Number of reconstructed frequency maps - convolution_in: True # Angular resolution in the input TOD + npointings: 6000 # Number of time samples + nsub_in: 12 # Number of sub-acquisitions + nsub_out: 12 + nrec: 6 # Number of reconstructed frequency maps + convolution_in: False # Angular resolution in the input TOD convolution_out: False # Angular resolution in the reconstruction bandpass_correction: True # Correction to keep the edges of the integration band constant NOISE: - ndet: 0 # Level of detector noise - npho150: 0 # Level of photon noise at 150 GHz - npho220: 0 # Level of photon noise at 220 GHz + ndet: 1 # Level of detector noise + npho150: 1 # Level of photon noise at 150 GHz + npho220: 1 # Level of photon noise at 220 GHz detector_nep: 4.7e-17 # NEP of detector noise SYNTHBEAM: synthbeam_kmax: 1 # Order of synthesized beam diffraction (synthbeam_kmax = 1 means 9 peaks) @@ -46,7 +46,7 @@ PLANCK: ### This section define which external data are we using - external_data: False + external_data: True weight_planck: 0 # Weight of Planck data within the QUBIC patch level_noise_planck: 1 # Noise level for Planck data bandwidth_planck: 0.2 # Multiplicative factor to define the bandwidth of Planck's data @@ -54,7 +54,7 @@ PLANCK: Pipeline: mapmaking: True # Run MapMaking Pipeline - spectrum: True # Compute Power Spectra of the frequency maps + spectrum: False # Compute Power Spectra of the frequency maps PCG: diff --git a/src/FMM/pipeline.py b/src/FMM/pipeline.py index e723c52..9ccd704 100644 --- a/src/FMM/pipeline.py +++ b/src/FMM/pipeline.py @@ -69,7 +69,7 @@ def __init__(self, comm, file, params): self.fsub = int(self.params["QUBIC"]["nsub_out"] / self.params["QUBIC"]["nrec"]) self.file = file - self.plot_folder = "src/FMM/" + self.params["path_out"] + "png/" + self.plot_folder = "FMM/" + self.params["path_out"] + "png/" self.externaldata = PipelineExternalData(file, self.params) self.externaldata.run(fwhm=self.params["QUBIC"]["convolution_in"], noise=True) @@ -82,10 +82,10 @@ def __init__(self, comm, file, params): ) if comm.Get_rank() == 0: - if not os.path.isdir("src/FMM/" + self.params["path_out"] + "maps/"): - os.makedirs("src/FMM/" + self.params["path_out"] + "maps/") - if not os.path.isdir("src/FMM/" + self.params["path_out"] + "png/"): - os.makedirs("src/FMM/" + self.params["path_out"] + "png/") + if not os.path.isdir("FMM/" + self.params["path_out"] + "maps/"): + os.makedirs("FMM/" + self.params["path_out"] + "maps/") + if not os.path.isdir("FMM/" + self.params["path_out"] + "png/"): + os.makedirs("FMM/" + self.params["path_out"] + "png/") self.job_id = os.environ.get("SLURM_JOB_ID") self.center = qubic.equ2gal( @@ -98,13 +98,10 @@ def __init__(self, comm, file, params): self.rank = self.comm.Get_rank() ### Sky - self.dict_in, self.dict_mono_in = self.get_dict(key="in") - self.dict_out, self.dict_mono_out = self.get_dict(key="out") + self.dict_in = self.get_dict(key="in") + self.dict_out = self.get_dict(key="out") self.skyconfig = self.get_sky_config() - ### Initial maps - self.m_nu_in = self.get_input_map() - ### Joint acquisition for TOD making self.joint_tod = JointAcquisitionFrequencyMapMaking( self.dict_in, @@ -156,6 +153,9 @@ def __init__(self, comm, file, params): corrected_bandpass=self.params["QUBIC"]["bandpass_correction"], ) + ### Initial maps + self.m_nu_in = self.get_input_map() + ### Define reconstructed and TOD operator self.get_H() @@ -492,7 +492,7 @@ def get_input_map(self): m_nu_in = np.zeros( (self.params["QUBIC"]["nrec"], 12 * self.params["SKY"]["nside"] ** 2, 3) ) - + for i in range(self.params["QUBIC"]["nrec"]): m_nu_in[i] = np.mean( self.external_timeline.m_nu[i * self.fsub : (i + 1) * self.fsub], axis=0 @@ -662,7 +662,7 @@ def _barrier(self): if self.comm is None: pass else: - self.comm._Barrier() + self.comm.Barrier() def _print_message(self, message): """ @@ -767,9 +767,9 @@ def pcg(self, d, x0, seenpix): comm=self.comm, x0=x0, M=M, - tol=self.params["PCG"]["tolpcg"], + tol=self.params["PCG"]["tol_pcg"], disp=True, - maxiter=self.params["PCG"]["n_iterpcg"], + maxiter=self.params["PCG"]["n_iter_pcg"], gif_folder=gif_folder, job_id=self.job_id, seenpix=self.seenpix, @@ -912,16 +912,16 @@ class PipelineEnd2End: def __init__(self, comm): - with open("src/FMM/params.yml", "r") as stream: + with open("FMM/params.yml", "r") as stream: self.params = yaml.safe_load(stream) self.comm = comm self.job_id = os.environ.get("SLURM_JOB_ID") - self.folder = "src/FMM/" + self.params["path_out"] + "maps/" + self.folder = "FMM/" + self.params["path_out"] + "maps/" self.file = self.folder + self.params["datafilename"] + f"_{self.job_id}.pkl" self.file_spectrum = ( - "src/FMM/" + "FMM/" + self.params["path_out"] + "spectrum/" + "spectrum_" @@ -947,7 +947,7 @@ def main(self, specific_file=None): if self.params["Pipeline"]["spectrum"]: if self.comm.Get_rank() == 0: create_folder_if_not_exists( - self.comm, "src/FMM/" + self.params["path_out"] + "spectrum/" + self.comm, "FMM/" + self.params["path_out"] + "spectrum/" ) if self.mapmaking is not None: diff --git a/src/__init__.py b/src/__init__.py index e69de29..da79187 100644 --- a/src/__init__.py +++ b/src/__init__.py @@ -0,0 +1,3 @@ +from .FMM import * +from .CMM import * +from .lib import * \ No newline at end of file diff --git a/src/lib/Qcg.py b/src/lib/Qcg.py index efea2ac..75812f0 100644 --- a/src/lib/Qcg.py +++ b/src/lib/Qcg.py @@ -200,7 +200,7 @@ def iteration(self): self.gif + f"iter_{self.niterations+self.iter_init}.png", self.center, reso=self.reso, - figsize=(12, 11), + figsize=(12, 2.7*mymap.shape[0]), min=min, max=max, fwhm=self.fwhm, diff --git a/src/run_cmm.py b/src/run_cmm.py deleted file mode 100644 index 4c1415e..0000000 --- a/src/run_cmm.py +++ /dev/null @@ -1,16 +0,0 @@ -import sys - -from pyoperators import * - -from CMM.pipeline import Pipeline - -seed_noise = int(sys.argv[1]) - -### MPI common arguments -comm = MPI.COMM_WORLD - -if __name__ == "__main__": - - pipeline = Pipeline(comm, 1, seed_noise) - - pipeline.main() diff --git a/src/run_cmm.sh b/src/run_cmm.sh deleted file mode 100644 index 74c4016..0000000 --- a/src/run_cmm.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -#SBATCH --job-name=FMM - -# we ask for n MPI tasks with N cores each on c nodes - -#SBATCH --partition=htc -#SBATCH --nodes=1 # c -#SBATCH --ntasks-per-node=1 # n -#SBATCH --cpus-per-task=4 # N -#SBATCH --mem=60G -#SBATCH --time=3-00:00:00 -#SBATCH --output=mulitple_jobs_%j.log -#SBATCH --array=1-500 - -export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} - -module load mpich - -mpirun -np $SLURM_NTASKS python run_cmm.py $1 diff --git a/src/run_fit_fmm.py b/src/run_fit_fmm.py deleted file mode 100644 index 8fc1e57..0000000 --- a/src/run_fit_fmm.py +++ /dev/null @@ -1,45 +0,0 @@ -import os - -import matplotlib.pyplot as plt -import numpy as np -from multiprocess import Pool -from pyoperators import MPI -from schwimmbad import MPIPool - -from lib.Qfit import FitEllSpace -from lib.Qfoldertools import MergeAllFiles -from lib.Qspectra_component import SkySpectra - -comm = MPI.COMM_WORLD - -### Concatenate all realizations -files = MergeAllFiles( - "/Users/mregnier/Desktop/git/Pipeline/src/FMM/CMBDUST_nrec2_new_code/spectrum/" -) - -nus_index = np.array([True, True, False, False, False, False, False, False, True]) -NBINS = 16 - -ell = files._reads_one_file(0, "ell")[:NBINS] -nus = files._reads_one_file(0, "nus")[nus_index] - -BBsignal = np.mean(files._reads_all_files("Dls"), axis=0)[:, nus_index, :NBINS][ - nus_index, :, :NBINS -] -BBnoise = files._reads_all_files("Nl")[:, :, nus_index, :NBINS][:, nus_index, :, :NBINS] -BBsignal -= np.mean(BBnoise, axis=0) - -sky = SkySpectra(ell, nus) -fit = FitEllSpace(ell, BBsignal, BBnoise, model=sky.model) - -samples, samples_flat = fit.run(300, 10, discard=200, comm=comm) - -plt.figure() -plt.plot(samples[..., 0], "-k", alpha=0.1) -plt.axhline(0) -plt.show() - -print() -print(f"Average : {np.mean(samples_flat, axis=0)}") -print(f"Error : {np.std(samples_flat, axis=0)}") -print() diff --git a/src/run_fmm.py b/src/run_fmm.py deleted file mode 100644 index ae12596..0000000 --- a/src/run_fmm.py +++ /dev/null @@ -1,21 +0,0 @@ -import sys - -from pyoperators import * - -from FMM.pipeline import PipelineEnd2End - -try: - file = str(sys.argv[1]) -except IndexError: - file = None - -if __name__ == "__main__": - - ### Common MPI arguments - comm = MPI.COMM_WORLD - - ### Initialization - pipeline = PipelineEnd2End(comm) - - ### Execution - pipeline.main(specific_file=file) diff --git a/src/run_fmm.sh b/src/run_fmm.sh deleted file mode 100644 index fd59745..0000000 --- a/src/run_fmm.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash - -#SBATCH --job-name=FMM - -# we ask for n MPI tasks with N cores each on c nodes - -#SBATCH --partition=htc -#SBATCH --nodes=1 # c -#SBATCH --ntasks-per-node=1 # n -#SBATCH --cpus-per-task=4 # N -#SBATCH --mem=60G -#SBATCH --time=3-00:00:00 -#SBATCH --output=mulitple_jobs_%j.log -#SBATCH --array=1-500 - -export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} - -module load mpich - -mpirun -np $SLURM_NTASKS python src/run_fmm.py \ No newline at end of file diff --git a/src/run_spectra_cmm.py b/src/run_spectra_cmm.py deleted file mode 100644 index 7bcad1c..0000000 --- a/src/run_spectra_cmm.py +++ /dev/null @@ -1 +0,0 @@ -from CMM.spectrum.get_spectra import Spectra