Layer-specific feature preference in a trained AlexNet model revealed by stylized image analysis.

1. import os
2. import time
3. import requests
4. import tarfile
5. import numpy as np
6. import argparse
8. import torch
9. from torch import nn
10. import torch.nn.functional as F
11. from torch.optim import Adam
14. class GraphConv(nn.Module):
15. """
16. Graph Convolutional Layer described in "Semi-Supervised Classification with Graph Convolutional Networks".
18. Given an input feature representation for each node in a graph, the Graph Convolutional Layer aims to aggregate
19. information from the node's neighborhood to update its own representation. This is achieved by applying a graph
20. convolutional operation that combines the features of a node with the features of its neighboring nodes.
22. Mathematically, the Graph Convolutional Layer can be described as follows:
24. H' = f(D^(-1/2) * A * D^(-1/2) * H * W)
26. where:
27. H: Input feature matrix with shape (N, F_in), where N is the number of nodes and F_in is the number of
28. input features per node.
29. A: Adjacency matrix of the graph with shape (N, N), representing the relationships between nodes.
30. W: Learnable weight matrix with shape (F_in, F_out), where F_out is the number of output features per node.
31. D: The degree matrix.
32. """
33. def __init__(self, input_dim, output_dim, use_bias=False):
34. super(GraphConv, self).__init__()
36. # Initialize the weight matrix W (in this case called `kernel`)
37. self.kernel = nn.Parameter(torch.Tensor(input_dim, output_dim))
38. nn.init.xavier_normal_(self.kernel) # Initialize the weights using Xavier initialization
40. # Initialize the bias (if use_bias is True)
41. self.bias = None
42. if use_bias:
43. self.bias = nn.Parameter(torch.Tensor(output_dim))
44. nn.init.zeros_(self.bias) # Initialize the bias to zeros
46. def forward(self, input_tensor, adj_mat):
47. """
48. Performs a graph convolution operation.
50. Args:
51. input_tensor (torch.Tensor): Input tensor representing node features.
52. adj_mat (torch.Tensor): Normalized adjacency matrix representing graph structure.
54. Returns:
55. torch.Tensor: Output tensor after the graph convolution operation.
56. """
58. support = torch.mm(input_tensor, self.kernel) # Matrix multiplication between input and weight matrix
59. output = torch.spmm(adj_mat, support) # Sparse matrix multiplication between adjacency matrix and support
60. # Add the bias (if bias is not None)
61. if self.bias is not None:
62. output = output + self.bias
64. return output
67. class GCN(nn.Module):
68. """
69. Graph Convolutional Network (GCN) as described in the paper `"Semi-Supervised Classification with Graph
70. Convolutional Networks" <https://arxiv.org/pdf/1609.02907.pdf>`.
72. The Graph Convolutional Network is a deep learning architecture designed for semi-supervised node
73. classification tasks on graph-structured data. It leverages the graph structure to learn node representations
74. by propagating information through the graph using graph convolutional layers.
76. The original implementation consists of two stacked graph convolutional layers. The ReLU activation function is
77. applied to the hidden representations, and the Softmax activation function is applied to the output representations.
78. """
79. def __init__(self, input_dim, hidden_dim, output_dim, use_bias=True, dropout_p=0.1):
80. super(GCN, self).__init__()
82. # Define the Graph Convolution layers
83. self.gc1 = GraphConv(input_dim, hidden_dim, use_bias=use_bias)
84. self.gc2 = GraphConv(hidden_dim, output_dim, use_bias=use_bias)
86. # Define the dropout layer
87. self.dropout = nn.Dropout(dropout_p)
89. def forward(self, input_tensor, adj_mat):
90. """
91. Performs forward pass of the Graph Convolutional Network (GCN).
93. Args:
94. input_tensor (torch.Tensor): Input node feature matrix with shape (N, input_dim), where N is the number of nodes
95. and input_dim is the number of input features per node.
96. adj_mat (torch.Tensor): Normalized adjacency matrix of the graph with shape (N, N), representing the relationships between
97. nodes.
99. Returns:
100. torch.Tensor: Output tensor with shape (N, output_dim), representing the predicted class probabilities for each node.
101. """
103. # Perform the first graph convolutional layer
104. x = self.gc1(input_tensor, adj_mat)
105. x = F.relu(x) # Apply ReLU activation function
106. x = self.dropout(x) # Apply dropout regularization
108. # Perform the second graph convolutional layer
109. x = self.gc2(x, adj_mat)
111. # Apply log-softmax activation function for classification
112. return F.log_softmax(x, dim=1)
115. def load_cora(path='./cora', device='cpu'):
116. """
117. The graph convolutional operation rquires the normalized adjacency matrix: D^(-1/2) * A * D^(-1/2). This step
118. scales the adjacency matrix such that the features of neighboring nodes are weighted appropriately during
119. aggregation. The steps involved in the renormalization trick are as follows:
120. - Compute the degree matrix.
121. - Compute the inverse square root of the degree matrix.
122. - Multiply the inverse square root of the degree matrix with the adjacency matrix.
123. """
125. # Set the paths to the data files
126. content_path = os.path.join(path, 'cora.content')
127. cites_path = os.path.join(path, 'cora.cites')
129. # Load data from files
130. content_tensor = np.genfromtxt(content_path, dtype=np.dtype(str))
131. cites_tensor = np.genfromtxt(cites_path, dtype=np.int32)
133. # Process features
134. features = torch.FloatTensor(content_tensor[:, 1:-1].astype(np.int32)) # Extract feature values
135. scale_vector = torch.sum(features, dim=1) # Compute sum of features for each node
136. scale_vector = 1 / scale_vector # Compute reciprocal of the sums
137. scale_vector[scale_vector == float('inf')] = 0 # Handle division by zero cases
138. scale_vector = torch.diag(scale_vector).to_sparse() # Convert the scale vector to a sparse diagonal matrix
139. features = scale_vector @ features # Scale the features using the scale vector
141. # Process labels
142. classes, labels = np.unique(content_tensor[:, -1], return_inverse=True) # Extract unique classes and map labels to indices
143. labels = torch.LongTensor(labels) # Convert labels to a tensor
145. # Process adjacency matrix
146. idx = content_tensor[:, 0].astype(np.int32) # Extract node indices
147. idx_map = {id: pos for pos, id in enumerate(idx)} # Create a dictionary to map indices to positions
149. # Map node indices to positions in the adjacency matrix
150. edges = np.array(
151. list(map(lambda edge: [idx_map[edge[0]], idx_map[edge[1]]],
152. cites_tensor)), dtype=np.int32)
154. V = len(idx) # Number of nodes
155. E = edges.shape[0] # Number of edges
156. adj_mat = torch.sparse_coo_tensor(edges.T, torch.ones(E), (V, V), dtype=torch.int64) # Create the initial adjacency matrix as a sparse tensor
157. adj_mat = torch.eye(V) + adj_mat # Add self-loops to the adjacency matrix
159. degree_mat = torch.sum(adj_mat, dim=1) # Compute the sum of each row in the adjacency matrix (degree matrix)
160. degree_mat = torch.sqrt(1 / degree_mat) # Compute the reciprocal square root of the degrees
161. degree_mat[degree_mat == float('inf')] = 0 # Handle division by zero cases
162. degree_mat = torch.diag(degree_mat).to_sparse() # Convert the degree matrix to a sparse diagonal matrix
164. adj_mat = degree_mat @ adj_mat @ degree_mat # Apply the renormalization trick
166. return features.to_sparse().to(device), labels.to(device), adj_mat.to_sparse().to(device)
168. def train_iter(epoch, model, optimizer, criterion, input, target, mask_train, mask_val, print_every=10):
169. start_t = time.time()
170. model.train()
171. optimizer.zero_grad()
173. # Forward pass
174. output = model(*input)
175. loss = criterion(output[mask_train], target[mask_train]) # Compute the loss using the training mask
177. loss.backward()
178. optimizer.step()
180. # Evaluate the model performance on training and validation sets
181. loss_train, acc_train = test(model, criterion, input, target, mask_train)
182. loss_val, acc_val = test(model, criterion, input, target, mask_val)
184. if epoch % print_every == 0:
185. # Print the training progress at specified intervals
186. print(f'Epoch: {epoch:04d} ({(time.time() - start_t):.4f}s) loss_train: {loss_train:.4f} acc_train: {acc_train:.4f} loss_val: {loss_val:.4f} acc_val: {acc_val:.4f}')
189. def test(model, criterion, input, target, mask):
190. model.eval()
191. with torch.no_grad():
192. output = model(*input)
193. output, target = output[mask], target[mask]
195. loss = criterion(output, target)
196. acc = (output.argmax(dim=1) == target).float().sum() / len(target)
197. return loss.item(), acc.item()
200. if __name__ == '__main__':
201. parser = argparse.ArgumentParser(description='PyTorch Graph Convolutional Network')
202. parser.add_argument('--epochs', type=int, default=200,
203. help='number of epochs to train (default: 200)')
204. parser.add_argument('--lr', type=float, default=0.01,
205. help='learning rate (default: 0.01)')
206. parser.add_argument('--l2', type=float, default=5e-4,
207. help='weight decay (default: 5e-4)')
208. parser.add_argument('--dropout-p', type=float, default=0.5,
209. help='dropout probability (default: 0.5)')
210. parser.add_argument('--hidden-dim', type=int, default=16,
211. help='dimension of the hidden representation (default: 16)')
212. parser.add_argument('--val-every', type=int, default=20,
213. help='epochs to wait for print training and validation evaluation (default: 20)')
214. parser.add_argument('--include-bias', action='store_true',
215. help='use bias term in convolutions (default: False)')
216. parser.add_argument('--no-accel', action='store_true',
217. help='disables accelerator')
218. parser.add_argument('--dry-run', action='store_true',
219. help='quickly check a single pass')
220. parser.add_argument('--seed', type=int, default=42, metavar='S',
221. help='random seed (default: 42)')
222. args = parser.parse_args()
224. use_accel = not args.no_accel and torch.accelerator.is_available()
226. torch.manual_seed(args.seed)
228. if use_accel:
229. device = torch.accelerator.current_accelerator()
230. else:
231. device = torch.device('cpu')
233. print(f'Using {device} device')
235. cora_url = 'https://linqs-data.soe.ucsc.edu/public/lbc/cora.tgz'
236. print('Downloading dataset...')
237. with requests.get(cora_url, stream=True) as tgz_file:
238. with tarfile.open(fileobj=tgz_file.raw, mode='r:gz') as tgz_object:
239. tgz_object.extractall()
241. print('Loading dataset...')
242. features, labels, adj_mat = load_cora(device=device)
243. idx = torch.randperm(len(labels)).to(device)
244. idx_test, idx_val, idx_train = idx[:1000], idx[1000:1500], idx[1500:]
246. gcn = GCN(features.shape[1], args.hidden_dim, labels.max().item() + 1, args.include_bias, args.dropout_p).to(device)
247. optimizer = Adam(gcn.parameters(), lr=args.lr, weight_decay=args.l2)
248. criterion = nn.NLLLoss()
250. for epoch in range(args.epochs):
251. train_iter(epoch + 1, gcn, optimizer, criterion, (features, adj_mat), labels, idx_train, idx_val, args.val_every)
252. if args.dry_run:
253. break
255. loss_test, acc_test = test(gcn, criterion, (features, adj_mat), labels, idx_test)
256. print(f'Test set results: loss {loss_test:.4f} accuracy {acc_test:.4f}')

main.py at commit acc295d, under BSD-3-Clause · at the source