Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Unable to Train Small Version on Cityscapes #2

Open
Michel-Liao opened this issue Jul 10, 2024 · 0 comments
Open

Unable to Train Small Version on Cityscapes #2

Michel-Liao opened this issue Jul 10, 2024 · 0 comments

Comments

@Michel-Liao
Copy link

Execution Command

Running python -m torch.distributed.launch --nproc_per_node=4 tools/train.py --config-file configs/cityscapes/pvt_small_FPT.yaml on four GTX 1080s.

Error

Found an error within /segmentron/data/dataloader/cityscapes.py. In _get_city_pairs(), the code doesn't account for if split='test'. Bug partially fixed if you change if split in ('train', 'val'): in _get_city_pairs() to if split in ('train', 'val', 'test'):.

However, when running the execution command again, I get an AssertionError: AssertionError== cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]

Error Log

python -m torch.distributed.launch --nproc_per_node=4 tools/train.py --config-file configs/cityscapes/pvt_small_FPT.yaml
*****************************************
Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. 
*****************************************
apex is not installed
apex is not installed
apex is not installedapex is not installed

INFO:Segmentron:Using 4 GPUs
2024-07-10 15:09:21,711 Segmentron INFO: Using 4 GPUs
INFO:Segmentron:Namespace(config_file='configs/cityscapes/pvt_small_FPT.yaml', device='cuda', distributed=True, input_img='tools/demo_vis.png', local_rank=0, log_iter=10, no_cuda=False, num_gpus=4, opts=[], resume=None, skip_val=False, test=False, val_epoch=1, vis=False)
2024-07-10 15:09:21,711 Segmentron INFO: Namespace(config_file='configs/cityscapes/pvt_small_FPT.yaml', device='cuda', distributed=True, input_img='tools/demo_vis.png', local_rank=0, log_iter=10, no_cuda=False, num_gpus=4, opts=[], resume=None, skip_val=False, test=False, val_epoch=1, vis=False)
INFO:Segmentron:{
        "SEED": 1024,
        "TIME_STAMP": "2024-07-10-15-08",
        "ROOT_PATH": "/n/fs/pvl-dpvorot/t4t_internal",
        "PHASE": "train",
        "DATASET": {
                "NAME": "cityscape",
                "MEAN": [
                        0.485,
                        0.456,
                        0.406
                ],
                "STD": [
                        0.229,
                        0.224,
                        0.225
                ],
                "IGNORE_INDEX": -1,
                "WORKERS": 8,
                "MODE": "testval"
        },
        "AUG": {
                "MIRROR": true,
                "BLUR_PROB": 0.0,
                "BLUR_RADIUS": 0.0,
                "COLOR_JITTER": null,
                "CROP": true
        },
        "TRAIN": {
                "EPOCHS": 200,
                "ITERS": 40000,
                "BATCH_SIZE": 4,
                "CROP_SIZE": [
                        512,
                        512
                ],
                "BASE_SIZE": 512,
                "MODEL_SAVE_DIR": "workdirs/cityscapes/pvt_small_FPT",
                "LOG_SAVE_DIR": "workdirs/",
                "PRETRAINED_MODEL_PATH": "",
                "BACKBONE_PRETRAINED": true,
                "BACKBONE_PRETRAINED_PATH": "pretrained/pvt_small.pth",
                "RESUME_MODEL_PATH": "",
                "SYNC_BATCH_NORM": true,
                "SNAPSHOT_EPOCH": 1,
                "APEX": true
        },
        "SOLVER": {
                "LR": 5e-05,
                "OPTIMIZER": "adamw",
                "EPSILON": 1e-08,
                "MOMENTUM": 0.9,
                "WEIGHT_DECAY": 0.0001,
                "DECODER_LR_FACTOR": 10.0,
                "LR_SCHEDULER": "poly",
                "POLY": {
                        "POWER": 0.9
                },
                "STEP": {
                        "GAMMA": 0.1,
                        "DECAY_EPOCH": [
                                10,
                                20
                        ]
                },
                "WARMUP": {
                        "EPOCHS": 0.0,
                        "FACTOR": 0.3333333333333333,
                        "METHOD": "linear"
                },
                "OHEM": false,
                "AUX": false,
                "AUX_WEIGHT": 0.4,
                "LOSS_NAME": ""
        },
        "TEST": {
                "TEST_MODEL_PATH": "workdirs/cityscapes/pvt_small_FPT/best_model.pth",
                "BATCH_SIZE": 4,
                "CROP_SIZE": [
                        512,
                        512
                ],
                "SCALES": [
                        1.0
                ],
                "FLIP": false
        },
        "VISUAL": {
                "OUTPUT_DIR": "../runs/visual/"
        },
        "MODEL": {
                "MODEL_NAME": "PVT_FPT",
                "BACKBONE": "pvt_small",
                "BACKBONE_SCALE": 1.0,
                "MULTI_LOSS_WEIGHT": [
                        1.0
                ],
                "DEFAULT_GROUP_NUMBER": 32,
                "DEFAULT_EPSILON": 1e-05,
                "BN_TYPE": "BN",
                "BN_EPS_FOR_ENCODER": null,
                "BN_EPS_FOR_DECODER": null,
                "OUTPUT_STRIDE": 16,
                "BN_MOMENTUM": null,
                "EMB_CHANNELS": 128,
                "DANET": {
                        "MULTI_DILATION": null,
                        "MULTI_GRID": false
                },
                "DEEPLABV3_PLUS": {
                        "USE_ASPP": true,
                        "ENABLE_DECODER": true,
                        "ASPP_WITH_SEP_CONV": true,
                        "DECODER_USE_SEP_CONV": true
                },
                "OCNet": {
                        "OC_ARCH": "base"
                },
                "ENCNET": {
                        "SE_LOSS": true,
                        "SE_WEIGHT": 0.2,
                        "LATERAL": true
                },
                "CCNET": {
                        "RECURRENCE": 2
                },
                "CGNET": {
                        "STAGE2_BLOCK_NUM": 3,
                        "STAGE3_BLOCK_NUM": 21
                },
                "POINTREND": {
                        "BASEMODEL": "DeepLabV3_Plus"
                },
                "HRNET": {
                        "PRETRAINED_LAYERS": [
                                "*"
                        ],
                        "STEM_INPLANES": 64,
                        "FINAL_CONV_KERNEL": 1,
                        "WITH_HEAD": true,
                        "STAGE1": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 1,
                                "NUM_BLOCKS": [
                                        1
                                ],
                                "NUM_CHANNELS": [
                                        32
                                ],
                                "BLOCK": "BOTTLENECK",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE2": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 2,
                                "NUM_BLOCKS": [
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE3": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 3,
                                "NUM_BLOCKS": [
                                        4,
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64,
                                        128
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE4": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 4,
                                "NUM_BLOCKS": [
                                        4,
                                        4,
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64,
                                        128,
                                        256
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        }
                },
                "TRANSLAB": {
                        "BOUNDARY_WEIGHT": 5
                },
                "TRANS4TRANS": {
                        "embed_dim": 256,
                        "depth": 4,
                        "num_heads": 8,
                        "mlp_ratio": 3.0,
                        "hid_dim": 64
                }
        }
}
2024-07-10 15:09:21,712 Segmentron INFO: {
        "SEED": 1024,
        "TIME_STAMP": "2024-07-10-15-08",
        "ROOT_PATH": "/n/fs/pvl-dpvorot/t4t_internal",
        "PHASE": "train",
        "DATASET": {
                "NAME": "cityscape",
                "MEAN": [
                        0.485,
                        0.456,
                        0.406
                ],
                "STD": [
                        0.229,
                        0.224,
                        0.225
                ],
                "IGNORE_INDEX": -1,
                "WORKERS": 8,
                "MODE": "testval"
        },
        "AUG": {
                "MIRROR": true,
                "BLUR_PROB": 0.0,
                "BLUR_RADIUS": 0.0,
                "COLOR_JITTER": null,
                "CROP": true
        },
        "TRAIN": {
                "EPOCHS": 200,
                "ITERS": 40000,
                "BATCH_SIZE": 4,
                "CROP_SIZE": [
                        512,
                        512
                ],
                "BASE_SIZE": 512,
                "MODEL_SAVE_DIR": "workdirs/cityscapes/pvt_small_FPT",
                "LOG_SAVE_DIR": "workdirs/",
                "PRETRAINED_MODEL_PATH": "",
                "BACKBONE_PRETRAINED": true,
                "BACKBONE_PRETRAINED_PATH": "pretrained/pvt_small.pth",
                "RESUME_MODEL_PATH": "",
                "SYNC_BATCH_NORM": true,
                "SNAPSHOT_EPOCH": 1,
                "APEX": true
        },
        "SOLVER": {
                "LR": 5e-05,
                "OPTIMIZER": "adamw",
                "EPSILON": 1e-08,
                "MOMENTUM": 0.9,
                "WEIGHT_DECAY": 0.0001,
                "DECODER_LR_FACTOR": 10.0,
                "LR_SCHEDULER": "poly",
                "POLY": {
                        "POWER": 0.9
                },
                "STEP": {
                        "GAMMA": 0.1,
                        "DECAY_EPOCH": [
                                10,
                                20
                        ]
                },
                "WARMUP": {
                        "EPOCHS": 0.0,
                        "FACTOR": 0.3333333333333333,
                        "METHOD": "linear"
                },
                "OHEM": false,
                "AUX": false,
                "AUX_WEIGHT": 0.4,
                "LOSS_NAME": ""
        },
        "TEST": {
                "TEST_MODEL_PATH": "workdirs/cityscapes/pvt_small_FPT/best_model.pth",
                "BATCH_SIZE": 4,
                "CROP_SIZE": [
                        512,
                        512
                ],
                "SCALES": [
                        1.0
                ],
                "FLIP": false
        },
        "VISUAL": {
                "OUTPUT_DIR": "../runs/visual/"
        },
        "MODEL": {
                "MODEL_NAME": "PVT_FPT",
                "BACKBONE": "pvt_small",
                "BACKBONE_SCALE": 1.0,
                "MULTI_LOSS_WEIGHT": [
                        1.0
                ],
                "DEFAULT_GROUP_NUMBER": 32,
                "DEFAULT_EPSILON": 1e-05,
                "BN_TYPE": "BN",
                "BN_EPS_FOR_ENCODER": null,
                "BN_EPS_FOR_DECODER": null,
                "OUTPUT_STRIDE": 16,
                "BN_MOMENTUM": null,
                "EMB_CHANNELS": 128,
                "DANET": {
                        "MULTI_DILATION": null,
                        "MULTI_GRID": false
                },
                "DEEPLABV3_PLUS": {
                        "USE_ASPP": true,
                        "ENABLE_DECODER": true,
                        "ASPP_WITH_SEP_CONV": true,
                        "DECODER_USE_SEP_CONV": true
                },
                "OCNet": {
                        "OC_ARCH": "base"
                },
                "ENCNET": {
                        "SE_LOSS": true,
                        "SE_WEIGHT": 0.2,
                        "LATERAL": true
                },
                "CCNET": {
                        "RECURRENCE": 2
                },
                "CGNET": {
                        "STAGE2_BLOCK_NUM": 3,
                        "STAGE3_BLOCK_NUM": 21
                },
                "POINTREND": {
                        "BASEMODEL": "DeepLabV3_Plus"
                },
                "HRNET": {
                        "PRETRAINED_LAYERS": [
                                "*"
                        ],
                        "STEM_INPLANES": 64,
                        "FINAL_CONV_KERNEL": 1,
                        "WITH_HEAD": true,
                        "STAGE1": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 1,
                                "NUM_BLOCKS": [
                                        1
                                ],
                                "NUM_CHANNELS": [
                                        32
                                ],
                                "BLOCK": "BOTTLENECK",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE2": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 2,
                                "NUM_BLOCKS": [
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE3": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 3,
                                "NUM_BLOCKS": [
                                        4,
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64,
                                        128
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        },
                        "STAGE4": {
                                "NUM_MODULES": 1,
                                "NUM_BRANCHES": 4,
                                "NUM_BLOCKS": [
                                        4,
                                        4,
                                        4,
                                        4
                                ],
                                "NUM_CHANNELS": [
                                        32,
                                        64,
                                        128,
                                        256
                                ],
                                "BLOCK": "BASIC",
                                "FUSE_METHOD": "SUM"
                        }
                },
                "TRANSLAB": {
                        "BOUNDARY_WEIGHT": 5
                },
                "TRANS4TRANS": {
                        "embed_dim": 256,
                        "depth": 4,
                        "num_heads": 8,
                        "mlp_ratio": 3.0,
                        "hid_dim": 64
                }
        }
}
INFO:Segmentron:Found 2975 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/train
2024-07-10 15:09:23,330 Segmentron INFO: Found 2975 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/train
INFO:Segmentron:Found 500 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/val
2024-07-10 15:09:23,606 Segmentron INFO: Found 500 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/val
INFO:Segmentron:Found 1525 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/test
2024-07-10 15:09:24,414 Segmentron INFO: Found 1525 images in the folder /n/fs/pvl-dpvorot/t4t_internal/datasets/cityscapes/leftImg8bit/test
INFO:Segmentron:Load backbone pretrained model from pretrained/pvt_small.pth
2024-07-10 15:09:27,335 Segmentron INFO: Load backbone pretrained model from pretrained/pvt_small.pth
INFO:Segmentron:_IncompatibleKeys(missing_keys=[], unexpected_keys=['cls_token', 'norm.weight', 'norm.bias', 'head.weight', 'head.bias'])
2024-07-10 15:09:35,993 Segmentron INFO: _IncompatibleKeys(missing_keys=[], unexpected_keys=['cls_token', 'norm.weight', 'norm.bias', 'head.weight', 'head.bias'])
Traceback (most recent call last):
  File "tools/train.py", line 301, in <module>
    trainer = Trainer(args)
  File "tools/train.py", line 90, in __init__
    self.model = get_segmentation_model().to(self.device)
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/model_zoo.py", line 22, in get_segmentation_model
Traceback (most recent call last):
  File "tools/train.py", line 301, in <module>
    trainer = Trainer(args)
  File "tools/train.py", line 90, in __init__
    self.model = get_segmentation_model().to(self.device)
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/model_zoo.py", line 22, in get_segmentation_model
Traceback (most recent call last):
  File "tools/train.py", line 301, in <module>
    trainer = Trainer(args)
  File "tools/train.py", line 90, in __init__
    self.model = get_segmentation_model().to(self.device)
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/model_zoo.py", line 22, in get_segmentation_model
Traceback (most recent call last):
  File "tools/train.py", line 301, in <module>
    trainer = Trainer(args)
  File "tools/train.py", line 90, in __init__
    self.model = get_segmentation_model().to(self.device)
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/model_zoo.py", line 22, in get_segmentation_model
        model = MODEL_REGISTRY.get(model_name)()model = MODEL_REGISTRY.get(model_name)()

          File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/pvt_fpt.py", line 43, in __init__
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/pvt_fpt.py", line 43, in __init__
model = MODEL_REGISTRY.get(model_name)()model = MODEL_REGISTRY.get(model_name)()

  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/pvt_fpt.py", line 43, in __init__
  File "/n/fs/pvl-dpvorot/t4t_internal/segmentron/models/pvt_fpt.py", line 43, in __init__
    == cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]
    AssertionError== cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]    

== cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]    
== cfg.TRAIN.BASE_SIZE == cfg.TEST.CROP_SIZE[0] == cfg.TEST.CROP_SIZE[1]
AssertionError
AssertionError
AssertionError
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees
No one assigned
Labels
None yet
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
1 participant
@Michel-Liao