From 16d33077fbc61ee6a94f62e629ef37ed9776b9ac Mon Sep 17 00:00:00 2001
From: jason-dou <jiachend@andrew.cmu.edu>
Date: Tue, 26 Apr 2022 10:41:51 -0400
Subject: [PATCH 1/4] Keep threshold always positive

---
 train.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/train.py b/train.py
index 3d97e71..f9d2a7e 100644
--- a/train.py
+++ b/train.py
@@ -160,7 +160,7 @@ def train_epoch(
     z = opt_params['z']
     gamma = opt_params['gamma']  # Target quantile
     lr_c = opt_params['lr_c']
-    sigma_b = 1.1  # Test value for sigma used in adaptive clipping
+    sigma_b = 1.5  # Test value for sigma used in adaptive clipping
     sigma = z * S
 
     # Define loss and optimizer

From c7514c032eac29720763882682be828d3792a38e Mon Sep 17 00:00:00 2001
From: jason-dou <jiachend@andrew.cmu.edu>
Date: Tue, 26 Apr 2022 10:41:51 -0400
Subject: [PATCH 2/4] Keep threshold always positive

---
 train.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/train.py b/train.py
index f9d2a7e..e463c14 100644
--- a/train.py
+++ b/train.py
@@ -222,6 +222,7 @@ def train_epoch(
         total += y.size(0)
         correct += (predicted == y).sum().item()
         S_e = S_e.item() if adaptive_clipping != 'Fixed' else S_e
+        S_e = max(S_e, 0.0)
 
     return running_loss / total, correct / total, S_e
 

From 659edff78ae2a6c268046f4a79686241aeef1bb2 Mon Sep 17 00:00:00 2001
From: jason-dou <jiachend@andrew.cmu.edu>
Date: Tue, 26 Apr 2022 17:27:06 -0400
Subject: [PATCH 3/4] Add params recommendation to exp3_params

---
 experiments.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/experiments.py b/experiments.py
index cd5d5d2..9473e70 100644
--- a/experiments.py
+++ b/experiments.py
@@ -82,10 +82,10 @@ def run_central_baseline_adaptive_cuttoff(params, clipping_methods):
         'clipping': 'Linear',
         'num_microbatches': 32,
         'batch_size': 32,
-        'S': 1,
-        'z': 0.2,
+        'S': 5,
+        'z': 0.1,
         'gamma': 0.7,
-        'lr_c': 0.1,
+        'lr_c': 0.01,
         'momentum': 0.5,
         'decay': 0,
         'n_epochs': 6,

From 14c2212ef2640506714dedf14feff64fa386e5f6 Mon Sep 17 00:00:00 2001
From: jason-dou <jiachend@andrew.cmu.edu>
Date: Thu, 28 Apr 2022 15:35:49 -0400
Subject: [PATCH 4/4] First test auto adaptive

---
 train.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/train.py b/train.py
index e463c14..b70e41b 100644
--- a/train.py
+++ b/train.py
@@ -106,6 +106,7 @@ def train(
     momentum = opt_params['momentum']
     decay = opt_params['decay']
     S_e = opt_params['S']
+    gamma = opt_params['gamma']
 
     criterion = nn.CrossEntropyLoss(reduction='none')
     optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=decay)
@@ -116,12 +117,13 @@ def train(
         'train_acc': [],
         'test_loss': [],
         'test_acc': [],
-        'S': []
+        'S': [],
+        'gamma': [],
     }
 
     print(f"Training {n_epochs} epoch(s) w/ {len(trainloader)} batches each.", flush=True)
     for epoch in range(n_epochs):
-        train_loss, train_acc, S_e = train_epoch(model, trainloader, device, optimizer, criterion, S_e, opt_params)
+        train_loss, train_acc, S_e, gamma = train_epoch(model, trainloader, device, optimizer, criterion, S_e, gamma, opt_params)
         test_loss, test_acc = test(model, testloader, device)
 
         # Write training metrics
@@ -138,6 +140,7 @@ def train(
         log['test_loss'].append(test_loss)
         log['test_acc'].append(test_acc)
         log['S'].append(S_e)
+        log['gamma'].append(gamma)
 
     # export scalar data to JSON for external processing
     helpers.write_logs(exp_name, log, opt_params)
@@ -151,6 +154,7 @@ def train_epoch(
         optimizer: torch.optim,
         criterion,
         S_e,
+        gamma,
         opt_params,
 ) -> List[Tuple[float, float]]:
     # DP-SGD parameters
@@ -158,7 +162,6 @@ def train_epoch(
     num_microbatches = opt_params['num_microbatches']
     S = S_e
     z = opt_params['z']
-    gamma = opt_params['gamma']  # Target quantile
     lr_c = opt_params['lr_c']
     sigma_b = 1.5  # Test value for sigma used in adaptive clipping
     sigma = z * S
@@ -198,6 +201,7 @@ def train_epoch(
                 saved_var[tensor_name].add_(new_grad)
             model.zero_grad()
 
+        gamma += (b/num_microbatches-gamma)/2
         if adaptive_clipping == 'Linear':
             b += torch.randn(1) * sigma_b
             b_t = b / num_microbatches
@@ -224,7 +228,7 @@ def train_epoch(
         S_e = S_e.item() if adaptive_clipping != 'Fixed' else S_e
         S_e = max(S_e, 0.0)
 
-    return running_loss / total, correct / total, S_e
+    return running_loss / total, correct / total, S_e, gamma
 
 
 def test(