Python - argparse module part2)

In [1]:

from IPython.core.display import display, HTML

display(HTML("<style>.container { width:100% !important; }</style>"))

View Source

Python argparse Part2)¶

usuage of argparse python module

- Attempt1. Making parser arguments precedes over config.json
- Attempt2. Handling multiple parameters for parser argument using json format 
- Goal. Handling model parameters in depth and changing default setting using config.json 

Attempt 1. Making parser argument precedes over config.json¶

As discussed in the argparse Part1), we want to set defaults using config.json and updating with its parameters from command line input which is more common than vice versa.

In [1]:

import argparse
import json
import os
from pprint import pprint

In [17]:

parser = argparse.ArgumentParser()

In [18]:

MYDICT = {'key': 'value'}

# Data and model checkpoints directories
parser.add_argument('--seed', type=int, default=42, help='random seed (default: 42)')
parser.add_argument('--epochs', type=int, default=1, help='number of epochs to train (default: 1)')
parser.add_argument('--dataset', type=str, default='MaskBaseDataset', help='dataset augmentation type (default: MaskBaseDataset)')
parser.add_argument('--augmentation', type=str, default='BaseAugmentation', help='data augmentation type (default: BaseAugmentation)')
parser.add_argument("--resize", nargs="+", type=int,default=[128, 96], help='resize size for image when training')
parser.add_argument('--batch_size', type=int, default=64, help='input batch size for training (default: 64)')
parser.add_argument('--valid_batch_size', type=int, default=1000, help='input batch size for validing (default: 1000)')
parser.add_argument('--model', type=str, default='BaseModel', help='model type (default: BaseModel)')
parser.add_argument('--optimizer', type=str, default='SGD', help='optimizer type (default: SGD)')
parser.add_argument('--lr', type=float, default=1e-3, help='learning rate (default: 1e-3)')
parser.add_argument('--lr_scheduler', type=str, default='StepLR', help='learning scheduler (default: StepLR)')
parser.add_argument('--val_ratio', type=float, default=0.2, help='ratio for validaton (default: 0.2)')
parser.add_argument('--criterion', type=str, default='cross_entropy', help='criterion type (default: cross_entropy)')
parser.add_argument('--lr_decay_step', type=int, default=20, help='learning rate scheduler deacy step (default: 20)')
parser.add_argument('--log_interval', type=int, default=20, help='how many batches to wait before logging training status')
parser.add_argument('--name', default='exp', help='model save at {SM_MODEL_DIR}/{name}')
parser.add_argument('--config', default='./model_config_custom.json', help='config.json file')
parser.add_argument('--early_stopping', type=int, default=5, help='early stopping on validation f-score')
parser.add_argument('--lr_sch_params', type=json.loads, default = MYDICT)
parser.add_argument('--data_dir', type=str, default='/opt/ml/input/data/train/images')
parser.add_argument('--model_dir', type=str, default='./model')

Out[18]:

_StoreAction(option_strings=['--model_dir'], dest='model_dir', nargs=None, const=None, default='./model', type=<class 'str'>, choices=None, help=None, metavar=None)

In [2]:

def read_json(file):
    with open(file) as json_file:
        data = json.load(json_file)
    return data

In [19]:

# checking parser arguments
args = parser.parse_args([])
pprint(vars(args))

{'augmentation': 'BaseAugmentation',
 'batch_size': 64,
 'config': './model_config_custom.json',
 'criterion': 'cross_entropy',
 'data_dir': '/opt/ml/input/data/train/images',
 'dataset': 'MaskBaseDataset',
 'early_stopping': 5,
 'epochs': 1,
 'log_interval': 20,
 'lr': 0.001,
 'lr_decay_step': 20,
 'lr_sch_params': {'key': 'value'},
 'lr_scheduler': 'StepLR',
 'model': 'BaseModel',
 'model_dir': './model',
 'name': 'exp',
 'optimizer': 'SGD',
 'resize': [128, 96],
 'seed': 42,
 'val_ratio': 0.2,
 'valid_batch_size': 1000}

as described in the parser.add_argument above, all arguments are now set to script definitions

model_config_custom.json¶

{
    "train" : {
        "seed": 42,
        "epochs": 50,
        "dataset": "MaskSplitByProfileDataset",
        "augmentation": "BasicAugmentation2",
        "resize": [224,224],
        "batch_size": 64,
        "valid_batch_size": 128,
        "model": "EfficientNet",
        "optimizer": "Adam",
        "lr": 1e-4,
        "lr_scheduler": "MultiStepLR",
        "lr_sch_params" : {
            "milestones" : [2,4,6,8],
            "gamma" : 0.5
         },
        "val_ratio": 0.2,
        "criterion": "cross_entropy",
        "lr_decay_step": 20,
        "log_interval": 20,
        "data_dir": "/opt/ml/input/data/train/images",
        "model_dir": "./model",
        "early_stopping" : 5 
    },
    "valid" : {
        "seed": 42,
        "batch_size": 500,
        "resize": [224,224],
        "model": "EfficientNet",
        "dataset": "MaskSplitByProfileDataset",
        "augmentation": "CustomAugmentation",
        "data_dir": "/opt/ml/input/data/train/images",
        "model_path": "",
        "output_path": ""
    },
     "inference" : {
        "batch_size": 500,
        "dataset": "BasicTestDataset2",
        "resize": [224,224],
        "model": "EfficientNet",
        "data_dir": "/opt/ml/input/data/eval",
        "model_path": "./model/exp51/",
        "output_path": "./model/exp51"
    }
}

In [29]:

config = read_json(args.config)
parser.set_defaults(**config['train'])
pprint(parser._defaults)

{'augmentation': 'BasicAugmentation2',
 'batch_size': 64,
 'criterion': 'cross_entropy',
 'data_dir': '/opt/ml/input/data/train/images',
 'dataset': 'MaskSplitByProfileDataset',
 'early_stopping': 5,
 'epochs': 50,
 'log_interval': 20,
 'lr': 0.0001,
 'lr_decay_step': 20,
 'lr_sch_params': {'gamma': 0.5, 'milestones': [2, 4, 6, 8]},
 'lr_scheduler': 'MultiStepLR',
 'model': 'EfficientNet',
 'model_dir': './model',
 'optimizer': 'Adam',
 'resize': [224, 224],
 'seed': 42,
 'val_ratio': 0.2,
 'valid_batch_size': 128}
<class 'argparse.ArgumentParser'>

parser class is now set to defaults according to model_config_custom.json

In [32]:

args = parser.parse_args(['--model', 'ConvNextLIn22Custom'])
pprint(vars(args))

{'augmentation': 'BasicAugmentation2',
 'batch_size': 64,
 'config': './model_config_custom.json',
 'criterion': 'cross_entropy',
 'data_dir': '/opt/ml/input/data/train/images',
 'dataset': 'MaskSplitByProfileDataset',
 'early_stopping': 5,
 'epochs': 50,
 'log_interval': 20,
 'lr': 0.0001,
 'lr_decay_step': 20,
 'lr_sch_params': {'gamma': 0.5, 'milestones': [2, 4, 6, 8]},
 'lr_scheduler': 'MultiStepLR',
 'model': 'ConvNextLIn22Custom',
 'model_dir': './model',
 'name': 'exp',
 'optimizer': 'Adam',
 'resize': [224, 224],
 'seed': 42,
 'val_ratio': 0.2,
 'valid_batch_size': 128}

parser changed the model argument (EfficientNet -> ConvNextLIn22Custom) from in-line command

Recap on attempt1¶

args = parser.parse_args(['--model', 'ConvNextLIn22Custom'])
def read_json(file):
    with open(file) as json_file:
        data = json.load(json_file)
    return data

config = read_json(args.config)

Here parser do checkout out any command line arguments; however, it only uses either default config.json path or specified config.json path from command line.

parser.set_defaults(**config['train'])

Second, setting parser defaults with the given config.json.

args = parser.parse_args(['--model', 'ConvNextLIn22Custom'])

Third, overwriting parser with command line arguments.

$ python train.py --model ConvNextLIn22Custom
# setting parameters with default `./model_config_custom.json` but only changing model parameter

$ python train.py --config ./model_config_other.json
# setting parameters with ./model_config_other.json

$ python train.py --config ./model_config_other.json --model ConvNextLIn22Custom
# setting parameters with './model_config_other.json' and changing model parameter

Conclusion on attempt1¶

config.json must contain all the parameters required for arguments
Personally prefer changing config.json over using command-line arguments
For each time of training, saving config.json along with model output would be a good practice to start tracking down parameter searching (Of course there are tons of better ways of doing this such as wandb, mlflow and etc)

Attempt 2. Handling multiple parameters for parser argument using json format¶

> Here we are going to create argumnet for `learning scheduler` class and its own `parameters`
> Unsurprisingly, this can be done using `json`, `dictionary`, and `**` unpacking. 

In Pytorch, there are many lr_scheduler such as CosineAnnealingLR, MultiStepLR, CyclicLR and so on.
They do have their own parameters.

CosineAnnealingLR : "T_max", "eta_min"
MultiStepLR : "milestones", "gamma"
CyclicLR : "cycle_momentum", "max_lr", "base_lr", "step_size_up" : 50, "step_size_down", "mode" : "triangular"

Let's denote those parameters as lr_sch_params. So we want to pass lr_sch_params from the command-line or config.json.

Luckily, for config file, we can apply same format since config.json is converted into dictionary class.
That is, for each lr_sscheduler adding lr_sch_params with its arguments

"lr_scheduler": "MultiStepLR",
"lr_sch_params" : {
    "milestones" : [2,4,6,8],
    "gamma" : 0.5
 },


"lr_scheduler": "CyclicLR",
"lr_sch_params" : {
    "cycle_momentum" : false,
    "max_lr" : 0.1,
    "base_lr" : 0.001,
    "step_size_up" : 50,
    "step_size_down" : 100,
    "mode" : "triangular"
 },    


"lr_scheduler": "CosineAnnealingLR",
"lr_sch_params" : {
    "T_max" : 10,
    "eta_min" : 0
 },

Problem arises from command-line (at least for me). As always, answer can be found on stackover flow
Using json.loads on type

MYDICT = {'key': 'value'} # for default mean nothing
parser.add_argument('--lr_sch_params', type=json.loads, default = MYDICT)

In [3]:

parser = argparse.ArgumentParser()

In [4]:

MYDICT = {'key': 'value'}

# Data and model checkpoints directories
parser.add_argument('--seed', type=int, default=42, help='random seed (default: 42)')
parser.add_argument('--epochs', type=int, default=1, help='number of epochs to train (default: 1)')
parser.add_argument('--dataset', type=str, default='MaskBaseDataset', help='dataset augmentation type (default: MaskBaseDataset)')
parser.add_argument('--augmentation', type=str, default='BaseAugmentation', help='data augmentation type (default: BaseAugmentation)')
parser.add_argument("--resize", nargs="+", type=int,default=[128, 96], help='resize size for image when training')
parser.add_argument('--batch_size', type=int, default=64, help='input batch size for training (default: 64)')
parser.add_argument('--valid_batch_size', type=int, default=1000, help='input batch size for validing (default: 1000)')
parser.add_argument('--model', type=str, default='BaseModel', help='model type (default: BaseModel)')
parser.add_argument('--optimizer', type=str, default='SGD', help='optimizer type (default: SGD)')
parser.add_argument('--lr', type=float, default=1e-3, help='learning rate (default: 1e-3)')
parser.add_argument('--lr_scheduler', type=str, default='StepLR', help='learning scheduler (default: StepLR)')
parser.add_argument('--val_ratio', type=float, default=0.2, help='ratio for validaton (default: 0.2)')
parser.add_argument('--criterion', type=str, default='cross_entropy', help='criterion type (default: cross_entropy)')
parser.add_argument('--lr_decay_step', type=int, default=20, help='learning rate scheduler deacy step (default: 20)')
parser.add_argument('--log_interval', type=int, default=20, help='how many batches to wait before logging training status')
parser.add_argument('--name', default='exp', help='model save at {SM_MODEL_DIR}/{name}')
parser.add_argument('--config', default='./model_config_custom.json', help='config.json file')
parser.add_argument('--early_stopping', type=int, default=5, help='early stopping on validation f-score')
parser.add_argument('--lr_sch_params', type=json.loads, default = MYDICT)
parser.add_argument('--data_dir', type=str, default='/opt/ml/input/data/train/images')
parser.add_argument('--model_dir', type=str, default='./model')

Out[4]:

_StoreAction(option_strings=['--model_dir'], dest='model_dir', nargs=None, const=None, default='./model', type=<class 'str'>, choices=None, help=None, metavar=None)

In [5]:

args = parser.parse_args(['--lr_scheduler', 'CosineAnnealingLR', '--lr_sch_params', '{ "T_max" : 10, "eta_min" : 0}'])
config = read_json(args.config)
parser.set_defaults(**config['train'])
args = parser.parse_args(['--lr_scheduler', 'CosineAnnealingLR', '--lr_sch_params', '{ "T_max" : 10, "eta_min" : 0}'])
pprint(vars(args))

{'augmentation': 'BasicAugmentation2',
 'batch_size': 64,
 'config': './model_config_custom.json',
 'criterion': 'cross_entropy',
 'data_dir': '/opt/ml/input/data/train/images',
 'dataset': 'MaskSplitByProfileDataset',
 'early_stopping': 5,
 'epochs': 50,
 'log_interval': 20,
 'lr': 0.0001,
 'lr_decay_step': 20,
 'lr_sch_params': {'T_max': 10, 'eta_min': 0},
 'lr_scheduler': 'CosineAnnealingLR',
 'model': 'EfficientNet',
 'model_dir': './model',
 'name': 'exp',
 'optimizer': 'Adam',
 'resize': [224, 224],
 'seed': 42,
 'val_ratio': 0.2,
 'valid_batch_size': 128}

Lastly these paraemters can be passed to lr_scheduler class using unpacking.

sch_module = getattr(import_module('torch.optim.lr_scheduler'), args.lr_scheduler)
scheduler = sch_module(
    optimizer,
    **args.lr_sch_params
)

Recap on attempt2¶

above example is equivalent to command-line below

$ python train.py --lr_scheduler CosineAnnealingLR --lr_sch_params '{"T_max" : 10, "eta_min" : 0}'
# setting parameters with default `./model_config_custom.json` but changing lr_scheduler & lr_sch_params

Conclusion on attempt2¶

json.load is a nice hack!

Caution In windows, pass -m {\\"key1\\":\\"value1\\"} in the terminal.(adding backslash \ in front of ", without spaces in the whole {} string) by Johnson Lai

With this, most of configurations can be done easily just changing config.json.