First Commit

MyOptimizer/lookahead.py

@@ -35,7 +35,9 @@ class Lookahead(Optimizer):
                 param_state['slow_buffer'] = torch.empty_like(fast_p.data)
                 param_state['slow_buffer'].copy_(fast_p.data)
             slow = param_state['slow_buffer']
-            slow.add_(group['lookahead_alpha'], fast_p.data - slow)
+            # slow.add_(group['lookahead_alpha'], fast_p.data - slow)
+            slow.add_(fast_p.data-slow, alpha=group['lookahead_alpha'])
+
             fast_p.data.copy_(slow)
 
     def sync_lookahead(self):

MyOptimizer/optim_factory.py

@@ -75,7 +75,8 @@ def create_optimizer(args, model, filter_bias_and_bn=True):
     elif opt_lower == 'nadam':
         optimizer = Nadam(parameters, **opt_args)
     elif opt_lower == 'radam':
-        optimizer = RAdam(parameters, **opt_args)
+        # optimizer = RAdam(parameters, **opt_args)
+        optimizer = optim.RAdam(parameters, **opt_args)
     elif opt_lower == 'adamp':        
         optimizer = AdamP(parameters, wd_ratio=0.01, nesterov=True, **opt_args)
     elif opt_lower == 'sgdp':