Quantization

# Load model and tokenizer model_id = 'gpt2' model = AutoModelForCausalLM.from_pretrained(model_id).to(device) tokenizer = AutoTokenizer.from_pretrained(model_id) # Print model size # get_memory_footprint : 모델이 차지하고 있는 메모리 공간을 나타내준다. > print(f"Model size: {model.get_memory_footprint():,} bytes") Model size: 510,342,192 bytes

def absmax_quantize(X): # Calculate scale scale = 127 / torch.max(torch.abs(X)) # Quantize X_quant = (scale * X).round() # Dequantize X_dequant = X_quant / scale return X_quant.to(torch.int8), X_dequant

def zeropoint_quantize(X): # Calculate value range (denominator) x_range = torch.max(X) - torch.min(X) x_range = 1 if x_range == 0 else x_range # Calculate scale scale = 255 / x_range # Shift by zero-point zeropoint = (-scale * torch.min(X) - 128).round() # Scale and round the inputs X_quant = torch.clip((X * scale + zeropoint).round(), -128, 127) # Dequantize X_dequant = (X_quant - zeropoint) / scale return X_quant.to(torch.int8), X_dequant

import numpy as np from copy import deepcopy # Store original weights weights = [param.data.clone() for param in model.parameters()] # Create model to quantize model_abs = deepcopy(model) # Quantize all model weights weights_abs = [] for param in model_abs.parameters(): _, dequantized = absmax_quantize(param.data) param.data = dequantized weights_abs.append(dequantized) # Create model to quantize model_zp = deepcopy(model) # Quantize all model weights weights_zp = [] for param in model_zp.parameters(): _, dequantized = zeropoint_quantize(param.data) param.data = dequantized weights_zp.append(dequantized)

import matplotlib.pyplot as plt import matplotlib.ticker as ticker # Flatten weight tensors weights = np.concatenate([t.cpu().numpy().flatten() for t in weights]) weights_abs = np.concatenate([t.cpu().numpy().flatten() for t in weights_abs]) weights_zp = np.concatenate([t.cpu().numpy().flatten() for t in weights_zp]) # Set background style plt.style.use('ggplot') # Create figure and axes fig, axs = plt.subplots(2, figsize=(10,10), dpi=300, sharex=True) # Plot the histograms for original and zero-point weights axs[0].hist(weights, bins=150, alpha=0.5, label='Original weights', color='blue', range=(-2, 2)) axs[0].hist(weights_abs, bins=150, alpha=0.5, label='Absmax weights', color='red', range=(-2, 2)) # Plot the histograms for original and absmax weights axs[1].hist(weights, bins=150, alpha=0.5, label='Original weights', color='blue', range=(-2, 2)) axs[1].hist(weights_zp, bins=150, alpha=0.5, label='Zero-point weights', color='green', range=(-2, 2)) # Add grid for ax in axs: ax.grid(True, linestyle='--', alpha=0.6) # Add legend axs[0].legend() axs[1].legend() # Add title and labels axs[0].set_title('Comparison of Original and Absmax Quantized Weights', fontsize=16) axs[1].set_title('Comparison of Original and Zeropoint Quantized Weights', fontsize=16) for ax in axs: ax.set_xlabel('Weights', fontsize=14) ax.set_ylabel('Count', fontsize=14) ax.yaxis.set_major_formatter(ticker.EngFormatter()) # Make y-ticks more human readable # Improve font plt.rc('font', size=12) plt.tight_layout() plt.show()

def calculate_perplexity(model, text): # Encode the text encodings = tokenizer(text, return_tensors='pt').to(device) # Define input_ids and target_ids input_ids = encodings.input_ids target_ids = input_ids.clone() with torch.no_grad(): outputs = model(input_ids, labels=target_ids) # Loss calculation neg_log_likelihood = outputs.loss # Perplexity calculation ppl = torch.exp(neg_log_likelihood) return ppl ppl = calculate_perplexity(model, original_text) ppl_abs = calculate_perplexity(model_abs, absmax_text) ppl_zp = calculate_perplexity(model_zp, absmax_text) print(f"Original perplexity: {ppl.item():.2f}") print(f"Absmax perplexity: {ppl_abs.item():.2f}") print(f"Zeropoint perplexity: {ppl_zp.item():.2f}")

Original perplexity: 7.59 Absmax perplexity: 21.52 Zeropoint perplexity: 18.08