From ae781c33c87a488d36d82800a2dc219988bd7d16 Mon Sep 17 00:00:00 2001 From: Bingqing Chen <33130314+bingqingchen@users.noreply.github.com> Date: Wed, 15 Sep 2021 11:08:17 -0400 Subject: [PATCH 1/2] Update README.md --- README.md | 62 ------------------------------------------------------- 1 file changed, 62 deletions(-) diff --git a/README.md b/README.md index 71baeb6..a17564b 100644 --- a/README.md +++ b/README.md @@ -85,65 +85,3 @@ location = {Virtual Event, Italy}, series = {e-Energy '21} } ``` -======= -# Name - -This is the official repository that implements the following paper: - -... - -[[slides]](docs/slides.pdf)[[paper]](https://dl.acm.org/doi/10.1145/3447555.3464874)[[video]](https://www.youtube.com/watch?v=rH64WyPHCVE) - -# Overview - -**Framework.** - -# Code Usage -### Clone repository -``` -git clone https://github.com/INFERLab/PROF.git -cd PROF -``` - -### Set up the environment -Set up the virtual environment with your preferred environment/package manager. - -The instruction here is based on **conda**. ([Install conda](https://docs.anaconda.com/anaconda/install/)) -``` -conda create --name cohort-env python=3.7 -c conda-forge -f requirements.txt -condo activate cohort-env -``` - -### File Structure -``` -. -├── agents -│ ├── base.py # Implement a controller that creates the CVXPY problem given building parameters -│ └── nn_policy.py # Inherit the controller from base.py; Forward pass: NN + Differentiable projection -├── algo -│ └── ppo.py # A PPO trainer -├── env -│ └── inverter.py # Implements the IEEE 37-bus case -├── utils -│ ├── network.py # Implements vanilla MLP and LSTM -│ └── ppo_utils.py # Helper function for PPO trainer, e.g. Replay_Memory, Advantage_func -├── network # Matlab code for linearization; Data for grid; -└── mypypower # Include some small changes from PyPower source code - -``` - -### Running -You can replicate our experiments for *Energy efficiency in a single building* with `main_IW.py`. - - -### Feedback - -Feel free to send any questions/feedback to: [Bingqing Chen](mailto:bingqinc@andrew.cmu.edu) - -### Citation - -If you use PROF, please cite us as follows: - -``` -Bingqing Chen, Priya L. Donti, Kyri Baker, J. Zico Kolter, and Mario Bergés. 2021. Enforcing Policy Feasibility Constraints through Differentiable Projection for Energy Optimization. In Proceedings of the Twelfth ACM International Conference on Future Energy Systems (e-Energy '21). Association for Computing Machinery, New York, NY, USA, 199–210. DOI:https://doi.org/10.1145/3447555.3464874 -``` From 35f50571a77e8ad79608126ebdeb55491c180e4a Mon Sep 17 00:00:00 2001 From: chenbq1234 Date: Wed, 15 Sep 2021 11:14:34 -0400 Subject: [PATCH 2/2] Replace inverter_policy with version in private repo --- agents/inverter_policy.py | 90 ++++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 35 deletions(-) diff --git a/agents/inverter_policy.py b/agents/inverter_policy.py index c91731e..89e6f45 100644 --- a/agents/inverter_policy.py +++ b/agents/inverter_policy.py @@ -15,19 +15,15 @@ DEVICE = "cuda" if torch.cuda.is_available() else "cpu" +### Can move to utils.network if appropriate class Net(nn.Module): def __init__(self, n_bus, n_inverters, shared_hidden_layer_sizes, indiv_hidden_layer_sizes, n_input = 3): super(Net, self).__init__() - - ''' - TODO: If want to support batch size = 1, need to take out batch norm - ''' - #### Multi-headed architecture # "Shared" model - # Set up non-linear network of Linear -> BatchNorm -> ReLU + # Set up non-linear network of Linear -> ReLU layer_sizes = [n_input * n_bus] + shared_hidden_layer_sizes[:-1] - layers = reduce(operator.add, + layers = reduce(operator.add, [[nn.Linear(a,b), nn.ReLU(), ] # nn.BatchNorm1d(b), nn.Dropout(p=0.2)] for a,b in zip(layer_sizes[0:-1], layer_sizes[1:])]) layers += [nn.Linear(layer_sizes[-1], shared_hidden_layer_sizes[-1])] @@ -35,7 +31,7 @@ def __init__(self, n_bus, n_inverters, shared_hidden_layer_sizes, indiv_hidden_l # Individual inverter model layer_sizes = [shared_hidden_layer_sizes[-1]] + indiv_hidden_layer_sizes - layers = reduce(operator.add, + layers = reduce(operator.add, [[nn.Linear(a,b), nn.ReLU(), ] # nn.BatchNorm1d(b), nn.Dropout(p=0.2)] for a,b in zip(layer_sizes[0:-1], layer_sizes[1:])]) layers += [nn.Linear(layer_sizes[-1], 2)] # output p and q @@ -49,8 +45,8 @@ def __init__(self, n_bus, n_inverters, shared_hidden_layer_sizes, indiv_hidden_l # # Set up non-linear network of Linear -> BatchNorm -> ReLU -> Dropout layers # self.n_inverters = n_inverters # layer_sizes = [4 * n_inverters] + shared_hidden_layer_sizes - # layers = reduce(operator.add, - # [[nn.Linear(a,b), nn.BatchNorm1d(b), nn.ReLU(), nn.Dropout(p=0.2)] + # layers = reduce(operator.add, + # [[nn.Linear(a,b), nn.BatchNorm1d(b), nn.ReLU(), nn.Dropout(p=0.2)] # for a,b in zip(layer_sizes[0:-1], layer_sizes[1:])]) # layers += [nn.Linear(layer_sizes[-1], 2 * n_inverters)] # self.nn = nn.Sequential(*layers) @@ -85,11 +81,11 @@ def __init__(self, network, memory, lr, lam = 10, scaler = 1000, **env_params): n_bus = env_params['n_bus'] self.gen_idx = env_params['gen_idx'] - self.other_idx = [i for i in range(self.n_bus) if i not in self.gen_idx] + self.other_idx = [i for i in range(n_bus) if i not in self.gen_idx] H = env_params['H'] - R = H[:, :self.n_bus] - B = H[:, self.n_bus:] + R = H[:, :n_bus] + B = H[:, n_bus:] R_new = np.vstack([np.hstack([R[self.gen_idx][:, self.gen_idx], R[self.gen_idx][:, self.other_idx]]), np.hstack([R[self.other_idx][:, self.gen_idx], @@ -134,7 +130,7 @@ def __init__(self, network, memory, lr, lam = 10, scaler = 1000, **env_params): P_av = cp.Parameter(len(self.gen_idx)) # Voltage: Apply to All Buses - z = cp.hstack([self.P, self.P_nc, self.Q, self.Q_nc]) # z: (70, ) + z = cp.hstack([P, P_nc, Q, Q_nc]) # z: (70, ) constraints = [self.V_lower - self.V0 <= H_new@z, H_new@z <= self.V_upper - self.V0] @@ -157,7 +153,7 @@ def __init__(self, network, memory, lr, lam = 10, scaler = 1000, **env_params): def forward(self, state, Sbus, P_av, inference_flag = True): ''' Input: - state: [dV(k-1), P_nc, Q_nc] + state: [dV(k-1), P_nc, Q_nc] where, Z_nc = Z - Z0 May get (n, dim) or (dim); @@ -165,39 +161,50 @@ def forward(self, state, Sbus, P_av, inference_flag = True): P, Q (with repsect to the reference point) ''' ## Get information for non-controllable loads - P_nc = Sbus.real[self.other_idx] / self.scaler - Q_nc = Sbus.imag[self.other_idx] / self.scaler - self.P_nc.value = P_nc - self.Q_nc.value = Q_nc - self.P_av.value = P_av - - if P_tilde.ndimension() == 1: - P_tilde, Q_tilde = self.nn(state.to(DEVICE)) # n x n_inverter + P_all = Sbus.real /self.scaler + Q_all = Sbus.imag /self.scaler + if len(Sbus.shape)==1: + P_nc = Sbus.real[self.other_idx] / self.scaler + Q_nc = Sbus.imag[self.other_idx] / self.scaler + elif len(Sbus.shape)==2: + P_nc = Sbus.real[:, self.other_idx] / self.scaler + Q_nc = Sbus.imag[:, self.other_idx] / self.scaler else: - P_tilde, Q_tilde = self.nn(state.to(DEVICE)) # n x n_inverter + print("Well, not expected to happen") + P_tilde, Q_tilde = self.nn(state.to(DEVICE)) # n x n_inverter + ## During inference if the action is already feasible, not need to project if inference_flag: - if self.is_feasible(P_tilde.detach().clone()/self.scaler, Q_tilde.detach().clone()/self.scaler, - P_nc, Q_nc, P_av): - return P_tilde/self.scaler, Q_tilde/self.scaler + P_tilde = P_tilde.squeeze() + Q_tilde = Q_tilde.squeeze() + if self.is_feasible(P_tilde.detach().clone()/self.scaler, + Q_tilde.detach().clone()/self.scaler, + P_nc, Q_nc, P_av): + P_all[self.gen_idx] = P_tilde.detach().cpu().numpy() / self.scaler + Q_all[self.gen_idx] = Q_tilde.detach().cpu().numpy() / self.scaler + return P_all, Q_all else: - try: + try: P, Q = self.proj_layer(P_tilde/self.scaler, Q_tilde/self.scaler, torch.tensor(P_nc).float().to(DEVICE), torch.tensor(Q_nc).float().to(DEVICE), torch.tensor(P_av).float().to(DEVICE)) self.proj_count += 1 + P_all[self.gen_idx] = P.detach().cpu().numpy() + Q_all[self.gen_idx] = Q.detach().cpu().numpy() except: # The solver dies for some reason - P = torch.zeros_like(P_tilde) - Q = torch.zeros_like(Q_tilde) - return P, Q + P_all[self.gen_idx] = 0 + Q_all[self.gen_idx] = 0 + return P_all, Q_all else: + #pdb.set_trace() P, Q = self.proj_layer(P_tilde/self.scaler, Q_tilde/self.scaler, torch.tensor(P_nc).float().to(DEVICE), torch.tensor(Q_nc).float().to(DEVICE), torch.tensor(P_av).float().to(DEVICE)) - proj_loss = self.mse(P.detach(), P_tilde/self.scaler) + self.mse(Q.detach(), Q_tilde/self.scaler) + proj_loss = self.mse(P.detach(), P_tilde/self.scaler) \ + + self.mse(Q.detach(), Q_tilde/self.scaler) return P, Q, proj_loss def update(self, batch_size = 64, n_batch = 16): @@ -205,7 +212,7 @@ def update(self, batch_size = 64, n_batch = 16): state, Sbus, P_av = self.memory.sample_batch(batch_size = batch_size) P, Q, proj_loss = self.forward(state, Sbus, P_av, inference_flag = False) #pdb.set_trace() - curtail = self.ReLU(torch.tensor(P_av).to(DEVICE) - P[:, self.gen_idx]) + curtail = self.ReLU(torch.tensor(P_av).to(DEVICE) - P) loss = curtail.mean() + self.lam * proj_loss print(f'curtail = {curtail.mean().item()}, proj_loss = {proj_loss.item()}') @@ -227,8 +234,8 @@ def is_feasible(self, P, Q, P_nc, Q_nc, P_av): if torch.any(v < self.V_lower_torch -self.V0_torch - eps) | torch.any(v > self.V_upper_torch-self.V0_torch+eps): return False - P = P[self.gen_idx] + self.P0_torch[self.gen_idx] - Q = Q[self.gen_idx] + self.Q0_torch[self.gen_idx] + P = P + self.P0_torch[self.gen_idx] + Q = Q + self.Q0_torch[self.gen_idx] PQ = torch.stack([P, Q]) # (2, 21) if torch.any(torch.norm(PQ, dim = 0) > self.S_rating_torch + eps): return False @@ -238,3 +245,16 @@ def is_feasible(self, P, Q, P_nc, Q_nc, P_av): else: return True + + + + +# if __name__ == "__main__": +# # Toy test code +# n = 3 +# nbatch = 2 +# model = Net(n, [200,200,40], [30,5]) +# controller = NeuralController(n, model, np.random.randn(n, 2*n), 0.95, 1.05) +# action = controller(torch.randn(nbatch, 4*n), torch.randn(nbatch, n), torch.randn(nbatch, n)) + +