# Module `opts`

Expand source code

    #!/usr/bin/env python3

    import argparse
    import time
    from datetime import datetime
    import os
    import utils.gpu_utils as gpu_utils
    import random

    # Global simulation counter
    gSimulationCounter = 0

    def parse_args(args):
        parser = initialise_arg_parser(args, 'FLPyTorch, running arguments.')

        # SERVER OPTIMIZATION PARAMS
        parser.add_argument(
            "--rounds",
            type=int,
            default=1,
            help="Number of rounds of federated learning",
        )
        parser.add_argument(
            '--client-sampling-type',
            type=str,
            choices=['uniform', 'poisson', 'poisson-no-empty'],
            default='uniform',
            help='Sampling strategy to sample clients'
        )
        parser.add_argument(
            "--num-clients-per-round",
            type=int,
            default=0,
            help="Number of available clients used for one communication round",
        )
        parser.add_argument(
            "--client-sampling-poisson",
            type=float,
            default=0.0,
            help="Sampling probability for Poisson sampling."
                 "Coincident with probability for clients to be selected in communication round",
        )
        parser.add_argument(
            '--global-lr',
            type=float,
            default=1,
            help='Global initial local learning rate (default: 1)'
        )
        parser.add_argument(
            '--global-lr-type',
            type=str,
            choices=['cosine', 'cifar_1', 'cifar_2', 'static'],
            default='static',
            help='Global learning rate strategy (default: static)'
        )
        parser.add_argument(
            "--global-optimiser",
            type=str,
            choices=['sgd', 'adam', 'rmsprop'],
            default='sgd',
            help='Global optimiser to use (default: SGD)'
        )
        parser.add_argument(
            '--global-momentum',
            type=float,
            default=0.,
            help='Global momentum (default: 0.)'
        )
        parser.add_argument(
            '--global-weight-decay',
            type=float,
            default=0.,
            help='Global weight decay (default: 0.)'
        )

        # LOCAL OPTIMISATION PARAMETERS
        parser.add_argument(
            "--run-local-steps",
            action="store_true",
            default=False,
            help="Run local epochs or local iterations, "
                 "if 'True' then each worker runs '--number-of-local-iters' steps in batches"
                 "          else each worker runs '--number-of-local-iters' local epochs."
        )
        parser.add_argument(
            "-li", "--number-of-local-iters",
            type=int,
            default=None,
            help="Static number of local steps to run training for defined by the server training configuration"
        )
        parser.add_argument(
            "-b", "--batch-size",
            type=int,
            default=32,
            help="Static batch size for data loading"
        )
        parser.add_argument(
            '--local-lr',
            type=float,
            default=0.1,
            help='initial local learning rate (default: 0.1)'
        )
        parser.add_argument(
            '--local-lr-type',
            type=str,
            choices=['cosine', 'cifar_1', 'cifar_2', 'static'],
            default='static',
            help='Local learning rate strategy (default: static)'
        )
        parser.add_argument(
            "--local-optimiser",
            type=str,
            choices=['sgd', 'adam', 'rmsprop'],
            default='sgd',
            help='Local optimiser to use (default: SGD)'
        )
        parser.add_argument(
            '--local-momentum',
            type=float,
            default=0.0,
            help='Momentum (default: 0.0)'
        )
        parser.add_argument(
            '--local-weight-decay',
            type=float,
            default=0.0,
            help='Local weight decay (default: 1e-4)'
        )

        # MODEL and DATA PARAMETERS
        parser.add_argument(
            "--dataset",
            type=str,
            required=True,
            choices=[
                "cifar10", "cifar100", "emnist", "full_shakespeare",
                "cifar10_fl", "cifar10_fl_by_class",
                "cifar100_fl", "shakespeare", "femnist",
                # Artificial problem
                "generated_for_quadratic_minimization",
                # For Logistic Regression
                "w9a", "w8a", "w7a", "w6a", "w5a", "w4a", "w3a", "w2a", "w1a",
                "a9a", "a8a", "a7a", "a6a", "a5a", "a4a", "a3a", "a2a", "a1a",
                "mushrooms", "phishing"
            ],
            help="Define which dataset to load"
        )
        parser.add_argument(
            "--dataset-generation-spec",
            type=str,
            default='',
            help="Specification when data is artificially generated"
        )
        parser.add_argument(
            "--metric",
            type=str,
            default='top_1_acc',
            choices=["top_1_acc", "top_5_acc", "neg_perplexity", "loss", "none"],
            help="Define which metric to optimize. None ignores validation step."
        )
        parser.add_argument(
            "--model",
            type=str,
            help="Define which model to load"
        )

        parser.add_argument(
            "--use-pretrained",
            action="store_true",
            default=False,
            help="Define should be modeled be pretrained or not"
        )

        parser.add_argument(
            "--train-last-layer",
            action="store_true",
            default=False,
            help="Train only last linear layer"
        )

        parser.add_argument(
            "--turn-off-batch-normalization-and-dropout",
            action="store_true",
            default=False,
            help="During train time do not use Batch Normalization and Dropout if it is inside compute graph f(x;ai,bi)."
        )

        # SETUP ARGUMENTS
        parser.add_argument(
            "--checkpoint-dir",
            type=str,
            default='../check_points',
            help="Directory to persist run meta data_preprocess, e.g. best/last models."
        )
        parser.add_argument(
            "--do-not-save-eval-checkpoints",
            action="store_true",
            default=False,
            help="Turn off save evaluate checkpoints."
        )
        parser.add_argument(
            "--resume-from",
            type=str,
            help="Resume from checkpoint."
        )
        parser.add_argument(
            "--load-best",
            default=False,
            action='store_true',
            help="Load best from checkpoint"
        )
        parser.add_argument(
            "--data-path",
            type=str,
            default="../data/",
            help="Base root directory for the dataset."
        )

        parser.add_argument(
            "--compute-type",
            type=str,
            default='fp32',
            choices=["bfp16", "fp16", "fp32", "fp64"],
            help="Define the type for trainable parameters and for buffers used e.g. in Batch Normalization."
        )

        parser.add_argument(
            "--gpu",
            type=str,
            default='0',
            help="Define on which GPU to run the model (comma-separated for multiple). If -1, use CPU."
        )

        parser.add_argument(
            "--log-gpu-usage",
            default=False,
            action='store_true',
            help="Log GPU usage"
        )

        parser.add_argument(
            "-n", "--num-workers-train",
            type=int,
            default=0,
            help="Num workers for train dataset loading"
        )
        parser.add_argument(
            "-nt", "--num-workers-test",
            type=int,
            default=0,
            help="Num workers for test dataset loading"
        )
        parser.add_argument(
            "--deterministic",
            action="store_true",
            default=False,
            help="Run deterministically for reproducibility."
        )

        parser.add_argument(
            "--manual-init-seed",
            type=int,
            default=123,
            help="Random seed to use for model initialization and data generation"
        )

        parser.add_argument(
            "--manual-runtime-seed",
            type=int,
            default=123,
            help="Random seed to use during runtime mostly for stochastic optimization algorithms"
        )

        parser.add_argument(
            "--group-name",
            type=str,
            default="",
            help="Name of the group which allow group experiments for statistics"
        )
        parser.add_argument(
            "--comment",
            type=str,
            default="",
            help="Extra arbitrarily comment during transferring experimental results"
        )
        parser.add_argument(
            "--hostname",
            type=str,
            default="",
            help="Name of the machine. If empty string is passed the name is determinate automatically."
        )
        parser.add_argument(
            "--eval-every",
            type=int,
            default=5,
            help="How often to do validation."
        )
        parser.add_argument(
            "--eval-async-threads",
            type=int,
            default=0,
            help="Thread pool size for perform evaluation checkpoint asynchronously"
        )
        parser.add_argument(
            "--save-async-threads",
            type=int,
            default=0,
            help="Thread pool size for perform serialization into filesystem asynchronously"
        )
        parser.add_argument(
            "--threadpool-for-local-opt",
            type=int,
            default=0,
            help="Perform local training within a pool of threads."
                 "Each one execute sequence of optimization for assigned clients in a serialized way."
        )
        parser.add_argument(
            "--run-id",
            type=str,
            default=str(time.time()),
            help="Identifier for the current job"
        )
        parser.add_argument(
            "--algorithm",
            type=str,
            default="dcgd",
            help="Algorithm (gradskip, marina, diana, dcgd, fedavg, scaffold, fedprox, ef21, cofig, frecon, ef21-pp, pp-marina)"
        )
        parser.add_argument(
            "--algorithm-options",
            type=str,
            default="",
            help="Extra option for pass into algorithm implementation to carry experiments (sgd:name-of-sgd)"
        )
        parser.add_argument(
            "--client-compressor",
            type=str,
            default="ident",
            help="Client compressor. ident, randk:k|<percentage-of-D>%, bernulli:p, natural, qsgd:levels, "
                 "nat.dithering:levels:norm, std.dithering:levels:norm, topk:k|topk:<percentage-of-D>%),"
                 "rank_k:k|rank_k:<percentage-of-D>%, terngrad"
        )
        parser.add_argument(
            "--extra-track",
            type=str,
            default="",
            help="Extra expensive comma separated tracking characterstics that should be collected"
                 "(full_gradient_norm_train,full_objective_value_train,full_gradient_norm_val,full_objective_value_val)"
        )
        parser.add_argument(
            "--loss",
            type=str,
            default="crossentropy",
            help="Name of the use loss function (crossentropy, mse, logistic)"
        )
        parser.add_argument(
            "--global-regulizer",
            type=str,
            choices=["none", "noncvx_robust_linear_regression", "cvx_l2norm_square_div_2"],
            default="none",
            help="Name of the extra global regulizer (default none)"
        )
        parser.add_argument(
            "--global-regulizer-alpha",
            type=float,
            default=0.0,
            help="Global regulizer scalar multiple (default value 0.0)"
        )
        parser.add_argument(
            "--out",
            type=str,
            default="",
            help="Name of the file for serialize results into filesystem"
        )

        parser.add_argument(
            "--wandb-key",
            type=str,
            default='',
            help="Personal Wandb key to wandb.ai online plotting tool"
        )

        parser.add_argument(
            "--wandb-project-name",
            type=str,
            default='fl_pytorch_simulation',
            help="Project name for wandb.ai online plotting tool in which plots will be generated"
        )

        parser.add_argument(
            "--external-devices",
            type=str,
            default='',
            help="List of external devices ip:port:device that can be used during simulation process"
        )

        parser.add_argument(
            "--worker-listen-mode",
            type=int,
            default=-1,
            help="This flag activate worker in local listener mode. For local worker mode only this flag is needed"
        )

        parser.add_argument(
            "--loglevel",
            type=str,
            choices=["debug", "info", "warn", "error", "critical"],
            default="INFO"
        )
        parser.add_argument(
            "--logfilter",
            type=str,
            default='.*',
            help="Regular expression to filter logging strings in which we're interesting in during debugging"
        )
        parser.add_argument(
            "--store-client-state-in-cpu",
            action="store_true",
            default=False,
            help="Store client state in CPU DRAM memory. Useful when number of clients is relatively big"
        )
        parser.add_argument(
            "--per-round-clean-torch-cache",
            action="store_true",
            default=False,
            help="Force clean GPU PyTorch cache at the end of each communication round used internally by PyTorch."
                 "Good to fix fragmentation issues."
        )
        parser.add_argument(
            "--allow-use-nv-tensorcores",
            action="store_true",
            default=False,
            help="Allow to use NVIDIA Tensor Cores available from NVIDIA Ampere architecture"
        )
        parser.add_argument(
            "--initialize-shifts-policy",
            type=str,
            choices=["zero", "full_gradient_at_start"],
            default="zero",
            help="Policy for initial shifts for opt. algorithms that contains notion of shifts"
        )
        parser.add_argument(
            "--sort-dataset-by-class-before-split",
            action="store_true",
            default=False,
            help="Sort train and test dataset by class label"
        )

        now = datetime.now()
        now = now.strftime("%Y%m%d%H%M%S")
        os.makedirs("../logs/", exist_ok=True)
        parser.add_argument(
            "--logfile",
            type=str,
            default=f"../logs/log_{now}.txt"
        )

        # Evaluation mode, do not run training
        parser.add_argument(
            "--evaluate",
            action='store_true',
            default=False,
            help="Evaluation or Training mode"
        )

        # Parse cmdline arguments
        args_dict = {}
        i = 0
        while True:
            if i == len(args):
                # Terminating the loop
                break
            if args[i].find('--') == 0 and (i + 1 == len(args) or args[i + 1].find('--') == 0):
                # Flag
                key = args[i][2:]
                args_dict[key] = True
                i += 1
            else:
                # Key-value pair
                key = args[i][2:]
                value = args[i+1]
                args_dict[key] = value
                i += 2

        # Perform substitution
        global gSimulationCounter
        gSimulationCounter += 1

        args_dict["simcounter"] = gSimulationCounter     # Simulation counter
        args_dict["now"] = int(time.time())              # Time
        args_dict["rnd-salt"] = random.randint(0, 1024*1024)  # Any seeds are setting after parameters initialization

        # Make substitution into arguments
        for kNested in range(5):
            for i in range(len(args)):
                args[i] = str(args[i]).format(**args_dict).lower().strip()

        args = parser.parse_args(args)

        # Make args.gpu as a list of target devices
        transform_gpu_args(args)
        return args

    def initialise_arg_parser(args, description):
        parser = argparse.ArgumentParser(args, description=description)
        return parser

    def transform_gpu_args(args):
        if args.gpu == "-1":
            args.gpu = ["cpu"]
        else:
            gpu_str_arg = args.gpu.split(',')
            if len(gpu_str_arg) > 1:
                # args.gpu = sorted([int(card) for card in gpu_str_arg])
                # Create a list of cuda(gpu) devices, followed at the end by cpu devices
                args.gpu = sorted([gpu_utils.get_target_device_str(int(card)) for card in gpu_str_arg if int(card) >= 0])
                args.gpu += sorted([gpu_utils.get_target_device_str(int(card)) for card in gpu_str_arg if int(card) < 0])
            else:
                args.gpu = [f"cuda:{args.gpu}"]

## Functions

`def initialise_arg_parser(args, description)`
  Expand source code

      def initialise_arg_parser(args, description):
          parser = argparse.ArgumentParser(args, description=description)
          return parser

`def parse_args(args)`
  Expand source code

      def parse_args(args):
          parser = initialise_arg_parser(args, 'FLPyTorch, running arguments.')

          # SERVER OPTIMIZATION PARAMS
          parser.add_argument(
              "--rounds",
              type=int,
              default=1,
              help="Number of rounds of federated learning",
          )
          parser.add_argument(
              '--client-sampling-type',
              type=str,
              choices=['uniform', 'poisson', 'poisson-no-empty'],
              default='uniform',
              help='Sampling strategy to sample clients'
          )
          parser.add_argument(
              "--num-clients-per-round",
              type=int,
              default=0,
              help="Number of available clients used for one communication round",
          )
          parser.add_argument(
              "--client-sampling-poisson",
              type=float,
              default=0.0,
              help="Sampling probability for Poisson sampling."
                   "Coincident with probability for clients to be selected in communication round",
          )
          parser.add_argument(
              '--global-lr',
              type=float,
              default=1,
              help='Global initial local learning rate (default: 1)'
          )
          parser.add_argument(
              '--global-lr-type',
              type=str,
              choices=['cosine', 'cifar_1', 'cifar_2', 'static'],
              default='static',
              help='Global learning rate strategy (default: static)'
          )
          parser.add_argument(
              "--global-optimiser",
              type=str,
              choices=['sgd', 'adam', 'rmsprop'],
              default='sgd',
              help='Global optimiser to use (default: SGD)'
          )
          parser.add_argument(
              '--global-momentum',
              type=float,
              default=0.,
              help='Global momentum (default: 0.)'
          )
          parser.add_argument(
              '--global-weight-decay',
              type=float,
              default=0.,
              help='Global weight decay (default: 0.)'
          )

          # LOCAL OPTIMISATION PARAMETERS
          parser.add_argument(
              "--run-local-steps",
              action="store_true",
              default=False,
              help="Run local epochs or local iterations, "
                   "if 'True' then each worker runs '--number-of-local-iters' steps in batches"
                   "          else each worker runs '--number-of-local-iters' local epochs."
          )
          parser.add_argument(
              "-li", "--number-of-local-iters",
              type=int,
              default=None,
              help="Static number of local steps to run training for defined by the server training configuration"
          )
          parser.add_argument(
              "-b", "--batch-size",
              type=int,
              default=32,
              help="Static batch size for data loading"
          )
          parser.add_argument(
              '--local-lr',
              type=float,
              default=0.1,
              help='initial local learning rate (default: 0.1)'
          )
          parser.add_argument(
              '--local-lr-type',
              type=str,
              choices=['cosine', 'cifar_1', 'cifar_2', 'static'],
              default='static',
              help='Local learning rate strategy (default: static)'
          )
          parser.add_argument(
              "--local-optimiser",
              type=str,
              choices=['sgd', 'adam', 'rmsprop'],
              default='sgd',
              help='Local optimiser to use (default: SGD)'
          )
          parser.add_argument(
              '--local-momentum',
              type=float,
              default=0.0,
              help='Momentum (default: 0.0)'
          )
          parser.add_argument(
              '--local-weight-decay',
              type=float,
              default=0.0,
              help='Local weight decay (default: 1e-4)'
          )

          # MODEL and DATA PARAMETERS
          parser.add_argument(
              "--dataset",
              type=str,
              required=True,
              choices=[
                  "cifar10", "cifar100", "emnist", "full_shakespeare",
                  "cifar10_fl", "cifar10_fl_by_class",
                  "cifar100_fl", "shakespeare", "femnist",
                  # Artificial problem
                  "generated_for_quadratic_minimization",
                  # For Logistic Regression
                  "w9a", "w8a", "w7a", "w6a", "w5a", "w4a", "w3a", "w2a", "w1a",
                  "a9a", "a8a", "a7a", "a6a", "a5a", "a4a", "a3a", "a2a", "a1a",
                  "mushrooms", "phishing"
              ],
              help="Define which dataset to load"
          )
          parser.add_argument(
              "--dataset-generation-spec",
              type=str,
              default='',
              help="Specification when data is artificially generated"
          )
          parser.add_argument(
              "--metric",
              type=str,
              default='top_1_acc',
              choices=["top_1_acc", "top_5_acc", "neg_perplexity", "loss", "none"],
              help="Define which metric to optimize. None ignores validation step."
          )
          parser.add_argument(
              "--model",
              type=str,
              help="Define which model to load"
          )

          parser.add_argument(
              "--use-pretrained",
              action="store_true",
              default=False,
              help="Define should be modeled be pretrained or not"
          )

          parser.add_argument(
              "--train-last-layer",
              action="store_true",
              default=False,
              help="Train only last linear layer"
          )

          parser.add_argument(
              "--turn-off-batch-normalization-and-dropout",
              action="store_true",
              default=False,
              help="During train time do not use Batch Normalization and Dropout if it is inside compute graph f(x;ai,bi)."
          )

          # SETUP ARGUMENTS
          parser.add_argument(
              "--checkpoint-dir",
              type=str,
              default='../check_points',
              help="Directory to persist run meta data_preprocess, e.g. best/last models."
          )
          parser.add_argument(
              "--do-not-save-eval-checkpoints",
              action="store_true",
              default=False,
              help="Turn off save evaluate checkpoints."
          )
          parser.add_argument(
              "--resume-from",
              type=str,
              help="Resume from checkpoint."
          )
          parser.add_argument(
              "--load-best",
              default=False,
              action='store_true',
              help="Load best from checkpoint"
          )
          parser.add_argument(
              "--data-path",
              type=str,
              default="../data/",
              help="Base root directory for the dataset."
          )

          parser.add_argument(
              "--compute-type",
              type=str,
              default='fp32',
              choices=["bfp16", "fp16", "fp32", "fp64"],
              help="Define the type for trainable parameters and for buffers used e.g. in Batch Normalization."
          )

          parser.add_argument(
              "--gpu",
              type=str,
              default='0',
              help="Define on which GPU to run the model (comma-separated for multiple). If -1, use CPU."
          )

          parser.add_argument(
              "--log-gpu-usage",
              default=False,
              action='store_true',
              help="Log GPU usage"
          )

          parser.add_argument(
              "-n", "--num-workers-train",
              type=int,
              default=0,
              help="Num workers for train dataset loading"
          )
          parser.add_argument(
              "-nt", "--num-workers-test",
              type=int,
              default=0,
              help="Num workers for test dataset loading"
          )
          parser.add_argument(
              "--deterministic",
              action="store_true",
              default=False,
              help="Run deterministically for reproducibility."
          )

          parser.add_argument(
              "--manual-init-seed",
              type=int,
              default=123,
              help="Random seed to use for model initialization and data generation"
          )

          parser.add_argument(
              "--manual-runtime-seed",
              type=int,
              default=123,
              help="Random seed to use during runtime mostly for stochastic optimization algorithms"
          )

          parser.add_argument(
              "--group-name",
              type=str,
              default="",
              help="Name of the group which allow group experiments for statistics"
          )
          parser.add_argument(
              "--comment",
              type=str,
              default="",
              help="Extra arbitrarily comment during transferring experimental results"
          )
          parser.add_argument(
              "--hostname",
              type=str,
              default="",
              help="Name of the machine. If empty string is passed the name is determinate automatically."
          )
          parser.add_argument(
              "--eval-every",
              type=int,
              default=5,
              help="How often to do validation."
          )
          parser.add_argument(
              "--eval-async-threads",
              type=int,
              default=0,
              help="Thread pool size for perform evaluation checkpoint asynchronously"
          )
          parser.add_argument(
              "--save-async-threads",
              type=int,
              default=0,
              help="Thread pool size for perform serialization into filesystem asynchronously"
          )
          parser.add_argument(
              "--threadpool-for-local-opt",
              type=int,
              default=0,
              help="Perform local training within a pool of threads."
                   "Each one execute sequence of optimization for assigned clients in a serialized way."
          )
          parser.add_argument(
              "--run-id",
              type=str,
              default=str(time.time()),
              help="Identifier for the current job"
          )
          parser.add_argument(
              "--algorithm",
              type=str,
              default="dcgd",
              help="Algorithm (gradskip, marina, diana, dcgd, fedavg, scaffold, fedprox, ef21, cofig, frecon, ef21-pp, pp-marina)"
          )
          parser.add_argument(
              "--algorithm-options",
              type=str,
              default="",
              help="Extra option for pass into algorithm implementation to carry experiments (sgd:name-of-sgd)"
          )
          parser.add_argument(
              "--client-compressor",
              type=str,
              default="ident",
              help="Client compressor. ident, randk:k|<percentage-of-D>%, bernulli:p, natural, qsgd:levels, "
                   "nat.dithering:levels:norm, std.dithering:levels:norm, topk:k|topk:<percentage-of-D>%),"
                   "rank_k:k|rank_k:<percentage-of-D>%, terngrad"
          )
          parser.add_argument(
              "--extra-track",
              type=str,
              default="",
              help="Extra expensive comma separated tracking characterstics that should be collected"
                   "(full_gradient_norm_train,full_objective_value_train,full_gradient_norm_val,full_objective_value_val)"
          )
          parser.add_argument(
              "--loss",
              type=str,
              default="crossentropy",
              help="Name of the use loss function (crossentropy, mse, logistic)"
          )
          parser.add_argument(
              "--global-regulizer",
              type=str,
              choices=["none", "noncvx_robust_linear_regression", "cvx_l2norm_square_div_2"],
              default="none",
              help="Name of the extra global regulizer (default none)"
          )
          parser.add_argument(
              "--global-regulizer-alpha",
              type=float,
              default=0.0,
              help="Global regulizer scalar multiple (default value 0.0)"
          )
          parser.add_argument(
              "--out",
              type=str,
              default="",
              help="Name of the file for serialize results into filesystem"
          )

          parser.add_argument(
              "--wandb-key",
              type=str,
              default='',
              help="Personal Wandb key to wandb.ai online plotting tool"
          )

          parser.add_argument(
              "--wandb-project-name",
              type=str,
              default='fl_pytorch_simulation',
              help="Project name for wandb.ai online plotting tool in which plots will be generated"
          )

          parser.add_argument(
              "--external-devices",
              type=str,
              default='',
              help="List of external devices ip:port:device that can be used during simulation process"
          )

          parser.add_argument(
              "--worker-listen-mode",
              type=int,
              default=-1,
              help="This flag activate worker in local listener mode. For local worker mode only this flag is needed"
          )

          parser.add_argument(
              "--loglevel",
              type=str,
              choices=["debug", "info", "warn", "error", "critical"],
              default="INFO"
          )
          parser.add_argument(
              "--logfilter",
              type=str,
              default='.*',
              help="Regular expression to filter logging strings in which we're interesting in during debugging"
          )
          parser.add_argument(
              "--store-client-state-in-cpu",
              action="store_true",
              default=False,
              help="Store client state in CPU DRAM memory. Useful when number of clients is relatively big"
          )
          parser.add_argument(
              "--per-round-clean-torch-cache",
              action="store_true",
              default=False,
              help="Force clean GPU PyTorch cache at the end of each communication round used internally by PyTorch."
                   "Good to fix fragmentation issues."
          )
          parser.add_argument(
              "--allow-use-nv-tensorcores",
              action="store_true",
              default=False,
              help="Allow to use NVIDIA Tensor Cores available from NVIDIA Ampere architecture"
          )
          parser.add_argument(
              "--initialize-shifts-policy",
              type=str,
              choices=["zero", "full_gradient_at_start"],
              default="zero",
              help="Policy for initial shifts for opt. algorithms that contains notion of shifts"
          )
          parser.add_argument(
              "--sort-dataset-by-class-before-split",
              action="store_true",
              default=False,
              help="Sort train and test dataset by class label"
          )

          now = datetime.now()
          now = now.strftime("%Y%m%d%H%M%S")
          os.makedirs("../logs/", exist_ok=True)
          parser.add_argument(
              "--logfile",
              type=str,
              default=f"../logs/log_{now}.txt"
          )

          # Evaluation mode, do not run training
          parser.add_argument(
              "--evaluate",
              action='store_true',
              default=False,
              help="Evaluation or Training mode"
          )

          # Parse cmdline arguments
          args_dict = {}
          i = 0
          while True:
              if i == len(args):
                  # Terminating the loop
                  break
              if args[i].find('--') == 0 and (i + 1 == len(args) or args[i + 1].find('--') == 0):
                  # Flag
                  key = args[i][2:]
                  args_dict[key] = True
                  i += 1
              else:
                  # Key-value pair
                  key = args[i][2:]
                  value = args[i+1]
                  args_dict[key] = value
                  i += 2

          # Perform substitution
          global gSimulationCounter
          gSimulationCounter += 1

          args_dict["simcounter"] = gSimulationCounter     # Simulation counter
          args_dict["now"] = int(time.time())              # Time
          args_dict["rnd-salt"] = random.randint(0, 1024*1024)  # Any seeds are setting after parameters initialization

          # Make substitution into arguments
          for kNested in range(5):
              for i in range(len(args)):
                  args[i] = str(args[i]).format(**args_dict).lower().strip()

          args = parser.parse_args(args)

          # Make args.gpu as a list of target devices
          transform_gpu_args(args)
          return args

`def transform_gpu_args(args)`
  Expand source code

      def transform_gpu_args(args):
          if args.gpu == "-1":
              args.gpu = ["cpu"]
          else:
              gpu_str_arg = args.gpu.split(',')
              if len(gpu_str_arg) > 1:
                  # args.gpu = sorted([int(card) for card in gpu_str_arg])
                  # Create a list of cuda(gpu) devices, followed at the end by cpu devices
                  args.gpu = sorted([gpu_utils.get_target_device_str(int(card)) for card in gpu_str_arg if int(card) >= 0])
                  args.gpu += sorted([gpu_utils.get_target_device_str(int(card)) for card in gpu_str_arg if int(card) < 0])
              else:
                  args.gpu = [f"cuda:{args.gpu}"]