diff --git a/UNet/Sim_logs/UNet_32_V9_V10_J.log b/UNet/Sim_logs/UNet_32_V9_V10_J.log new file mode 100644 index 0000000000000000000000000000000000000000..c5e6f0cd9f906edd3713c3477a04bfdfb9d153bc --- /dev/null +++ b/UNet/Sim_logs/UNet_32_V9_V10_J.log @@ -0,0 +1,70 @@ +(OK) Loading cuda 10.2.89 +(OK) Loading python 3.7.11 +(!!) The SciPy Stack is available: http://www.scipy.org/stackspec.html + Built with GCC compilers. +Collecting torch==1.10.1 + Using cached torch-1.10.1-cp37-cp37m-manylinux1_x86_64.whl (881.9 MB) +Collecting typing-extensions + Using cached typing_extensions-4.1.1-py3-none-any.whl (26 kB) +Installing collected packages: typing-extensions, torch + WARNING: The scripts convert-caffe2-to-onnx, convert-onnx-to-caffe2 and torchrun are installed in '/home/yk138599/.local/bin' which is not on PATH. + Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location. +Successfully installed torch-1.10.1 typing-extensions-4.1.1 +WARNING: You are using pip version 21.2.4; however, version 22.0.3 is available. +You should consider upgrading via the '/usr/local_rwth/sw/python/3.7.11/x86_64/bin/python3.7 -m pip install --upgrade pip' command. +number auf epochs: 500 +batchsize: 32 +learning rate: 1e-05 +kernel size is: 5 + seed is: 373686838 +Traceback (most recent call last): + File "./UNet_V9_1.py", line 247, in <module> + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/npyio.py", line 441, in load + pickle_kwargs=pickle_kwargs) + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/format.py", line 743, in read_array + raise ValueError("Object arrays cannot be loaded when " +ValueError: Object arrays cannot be loaded when allow_pickle=False +python3 ./UNet_V9_1.py 0.86s user 1.37s system 63% cpu 3.518 total +number auf epochs: 500 +batchsize: 32 +learning rate: 1e-05 +kernel size is: 7 + seed is: 373686838 +Traceback (most recent call last): + File "./UNet_V9_2.py", line 247, in <module> + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/npyio.py", line 441, in load + pickle_kwargs=pickle_kwargs) + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/format.py", line 743, in read_array + raise ValueError("Object arrays cannot be loaded when " +ValueError: Object arrays cannot be loaded when allow_pickle=False +python3 ./UNet_V9_2.py 0.87s user 1.33s system 79% cpu 2.786 total +number auf epochs: 500 +batchsize: 32 +learning rate: 1e-05 +kernel size is: 3 + seed is: 373686838 +Traceback (most recent call last): + File "./UNet_V9_3.py", line 247, in <module> + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/npyio.py", line 441, in load + pickle_kwargs=pickle_kwargs) + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/format.py", line 743, in read_array + raise ValueError("Object arrays cannot be loaded when " +ValueError: Object arrays cannot be loaded when allow_pickle=False +python3 ./UNet_V9_3.py 0.85s user 1.39s system 82% cpu 2.716 total +number auf epochs: 500 +batchsize: 32 +learning rate: 1e-05 +kernel size is: 5 + seed is: 2193910023 +Traceback (most recent call last): + File "./UNet_V10.py", line 243, in <module> + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/npyio.py", line 441, in load + pickle_kwargs=pickle_kwargs) + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/format.py", line 743, in read_array + raise ValueError("Object arrays cannot be loaded when " +ValueError: Object arrays cannot be loaded when allow_pickle=False +python3 ./UNet_V10.py 0.86s user 1.38s system 79% cpu 2.806 total diff --git a/UNet/Sim_logs/UNet_V9_32_25584245.log b/UNet/Sim_logs/UNet_V9_32_25584245.log new file mode 100644 index 0000000000000000000000000000000000000000..b7bd00424b0b120a1ac35fe1a96f78f2c96ebd62 --- /dev/null +++ b/UNet/Sim_logs/UNet_V9_32_25584245.log @@ -0,0 +1,62 @@ +(OK) Loading cuda 10.2.89 +(OK) Loading python 3.7.11 +(!!) The SciPy Stack is available: http://www.scipy.org/stackspec.html + Built with GCC compilers. +Collecting torch==1.10.1 + Using cached torch-1.10.1-cp37-cp37m-manylinux1_x86_64.whl (881.9 MB) +Collecting typing-extensions + Using cached typing_extensions-4.1.1-py3-none-any.whl (26 kB) +Installing collected packages: typing-extensions, torch + WARNING: The scripts convert-caffe2-to-onnx, convert-onnx-to-caffe2 and torchrun are installed in '/home/yk138599/.local/bin' which is not on PATH. + Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location. +Successfully installed torch-1.10.1 typing-extensions-4.1.1 +WARNING: You are using pip version 21.2.4; however, version 22.0.3 is available. +You should consider upgrading via the '/usr/local_rwth/sw/python/3.7.11/x86_64/bin/python3.7 -m pip install --upgrade pip' command. +number auf epochs: 500 +batchsize: 32 +learning rate: 1e-05 +kernel size is: 5 + seed is: 373686838 +Traceback (most recent call last): + File "./UNet_V9_1.py", line 247, in <module> + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/npyio.py", line 417, in load + fid = stack.enter_context(open(os_fspath(file), "rb")) +FileNotFoundError: [Errno 2] No such file or directory: 'home/yk138599/Hiwi/UNet/Trainingsdata/Norm_min_max_32_angles.npy' +python3 ./UNet_V9_1.py 0.84s user 1.49s system 45% cpu 5.157 total +number auf epochs: 500 +batchsize: 32 +learning rate: 1e-05 +kernel size is: 7 + seed is: 373686838 +Traceback (most recent call last): + File "./UNet_V9_2.py", line 247, in <module> + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/npyio.py", line 417, in load + fid = stack.enter_context(open(os_fspath(file), "rb")) +FileNotFoundError: [Errno 2] No such file or directory: 'home/yk138599/Hiwi/UNet/Trainingsdata/Norm_min_max_32_angles.npy' +python3 ./UNet_V9_2.py 0.86s user 1.41s system 76% cpu 2.953 total +number auf epochs: 500 +batchsize: 32 +learning rate: 1e-05 +kernel size is: 3 + seed is: 373686838 +Traceback (most recent call last): + File "./UNet_V9_3.py", line 247, in <module> + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/npyio.py", line 417, in load + fid = stack.enter_context(open(os_fspath(file), "rb")) +FileNotFoundError: [Errno 2] No such file or directory: 'home/yk138599/Hiwi/UNet/Trainingsdata/Norm_min_max_32_angles.npy' +python3 ./UNet_V9_3.py 0.89s user 1.33s system 75% cpu 2.963 total +number auf epochs: 500 +batchsize: 32 +learning rate: 1e-05 +kernel size is: 5 + seed is: 2193910023 +Traceback (most recent call last): + File "./UNet_V10.py", line 243, in <module> + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') + File "/rwthfs/rz/SW/UTIL.common/Python/3.7.11/x86_64/lib/python3.7/site-packages/numpy/lib/npyio.py", line 417, in load + fid = stack.enter_context(open(os_fspath(file), "rb")) +FileNotFoundError: [Errno 2] No such file or directory: 'home/yk138599/Hiwi/UNet/Trainingsdata/Norm_min_max_32_angles.npy' +python3 ./UNet_V10.py 0.89s user 1.41s system 69% cpu 3.295 total diff --git a/UNet/Train_model.sh b/UNet/Train_model.sh index 04db9528c80a33d7bef53bf147074d422d32934e..c9969a09599aa13ae9ab0c32bfa74966a80beb7b 100644 --- a/UNet/Train_model.sh +++ b/UNet/Train_model.sh @@ -6,16 +6,19 @@ #SBATCH --partition=c18g #SBATCH -J training_model -#SBATCH -o Sim_logs/UNet_V9_32_%J.log +#SBATCH -o Sim_logs/UNet_32_V9_V10_J.log #SBATCH --gres=gpu:1 -#SBATCH --time=50:00:00 +#SBATCH --time=90:00:00 ### Request memory you need for your job in MB #SBATCH --mem-per-cpu=10000 #SBATCH --mem-per-gpu=16000 module load cuda module load python/3.7.11 pip3 install --user -Iv -q torch==1.10.1 -time python3 ./UNet_V9.py +time python3 ./UNet_V9_1.py +time python3 ./UNet_V9_2.py +time python3 ./UNet_V9_3.py +time python3 ./UNet_V10.py #print GPU Information #$CUDA_ROOT/extras/demo_suite/deviceQuery -noprompt diff --git a/UNet/UNet_V10.py b/UNet/UNet_V10.py index 5efc2f75d0d3e1feaf208d306a42e6d369268a97..9f16b123be0a8f93a75178dc61d0d23273a6dbfa 100644 --- a/UNet/UNet_V10.py +++ b/UNet/UNet_V10.py @@ -224,7 +224,7 @@ if __name__ == '__main__': path_to_rep = '/home/yk138599/Hiwi/damask3' use_seeds = True seed = 2193910023 - num_epochs = 1300 + num_epochs = 500 b_size = 32 opt_func = torch.optim.Adam lr = 0.00001 @@ -240,8 +240,8 @@ if __name__ == '__main__': random.seed(seed) np.random.seed(seed) device = get_default_device() - normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') - train_dl, valid_dl = Create_Dataloader(f'{path_to_rep}/UNet/Trainingsdata/Training_Dataset_normalized_32_V2.pt', batch_size= b_size ) + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy', allow_pickle = True) + train_dl, valid_dl = Create_Dataloader(f'{path_to_rep}/UNet/Trainingsdata/TD_norm_32_phase_only.pt', batch_size= b_size ) train_dl = DeviceDataLoader(train_dl, device) valid_dl = DeviceDataLoader(valid_dl, device) diff --git a/UNet/UNet_V9_1.py b/UNet/UNet_V9_1.py index 67edfbcdb7440ef6cbe86c974746ccfb49fe932a..9d0a6421475873e2e5eee12ced18dd07b79bc22e 100644 --- a/UNet/UNet_V9_1.py +++ b/UNet/UNet_V9_1.py @@ -228,7 +228,7 @@ if __name__ == '__main__': path_to_rep = '/home/yk138599/Hiwi/damask3' use_seeds = True seed = 373686838 - num_epochs = 1300 + num_epochs = 500 b_size = 32 opt_func = torch.optim.Adam lr = 0.00001 @@ -244,8 +244,8 @@ if __name__ == '__main__': random.seed(seed) np.random.seed(seed) device = get_default_device() - normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') - train_dl, valid_dl = Create_Dataloader(f'{path_to_rep}/UNet/Trainingsdata/Training_Dataset_normalized_32_V2.pt', batch_size= b_size ) + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_phase_only.npy', allow_pickle = True) + train_dl, valid_dl = Create_Dataloader(f'{path_to_rep}/UNet/Trainingsdata/TD_norm_32_phase_only.pt', batch_size= b_size ) train_dl = DeviceDataLoader(train_dl, device) valid_dl = DeviceDataLoader(valid_dl, device) diff --git a/UNet/UNet_V9_2.py b/UNet/UNet_V9_2.py index 7337baa0bf57519d089d9edffc07139c2a2e0a1e..886846e464e23278dbde1bd5df84be223152bacd 100644 --- a/UNet/UNet_V9_2.py +++ b/UNet/UNet_V9_2.py @@ -228,7 +228,7 @@ if __name__ == '__main__': path_to_rep = '/home/yk138599/Hiwi/damask3' use_seeds = True seed = 373686838 - num_epochs = 1300 + num_epochs = 500 b_size = 32 opt_func = torch.optim.Adam lr = 0.00001 @@ -244,8 +244,8 @@ if __name__ == '__main__': random.seed(seed) np.random.seed(seed) device = get_default_device() - normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') - train_dl, valid_dl = Create_Dataloader(f'{path_to_rep}/UNet/Trainingsdata/Training_Dataset_normalized_32_V2.pt', batch_size= b_size ) + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy', allow_pickle = True) + train_dl, valid_dl = Create_Dataloader(f'{path_to_rep}/UNet/Trainingsdata/TD_norm_32_phase_only.pt', batch_size= b_size ) train_dl = DeviceDataLoader(train_dl, device) valid_dl = DeviceDataLoader(valid_dl, device) diff --git a/UNet/UNet_V9_3.py b/UNet/UNet_V9_3.py index 997dcfec17c601400f5116467ba2563f18cfed06..8453bc792f10eb753a017f1f514e110bc0a4d7e3 100644 --- a/UNet/UNet_V9_3.py +++ b/UNet/UNet_V9_3.py @@ -228,7 +228,7 @@ if __name__ == '__main__': path_to_rep = '/home/yk138599/Hiwi/damask3' use_seeds = True seed = 373686838 - num_epochs = 1300 + num_epochs = 500 b_size = 32 opt_func = torch.optim.Adam lr = 0.00001 @@ -244,8 +244,8 @@ if __name__ == '__main__': random.seed(seed) np.random.seed(seed) device = get_default_device() - normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy') - train_dl, valid_dl = Create_Dataloader(f'{path_to_rep}/UNet/Trainingsdata/Training_Dataset_normalized_32_V2.pt', batch_size= b_size ) + normalization = np.load(f'{path_to_rep}/UNet/Trainingsdata/Norm_min_max_32_angles.npy', allow_pickle = True) + train_dl, valid_dl = Create_Dataloader(f'{path_to_rep}/UNet/Trainingsdata/TD_norm_32_phase_only.pt', batch_size= b_size ) train_dl = DeviceDataLoader(train_dl, device) valid_dl = DeviceDataLoader(valid_dl, device)