VisualMultidimensionalMarkovChainsGenerator / visual_markov_patterns_generator.py
Skupydoom's picture
Add all the needed files
af6e161 verified
Raw History Blame Contribute Delete
14.7 kB
import numpy as np
import random
import torch
import torch.nn as nn
from collections import defaultdict
import cv2
import pickle
import os
from PIL import Image
class Config:
"""Generation system configuration"""
IMG_SIZE = 128
PATCH_SIZE = 16
GRID_SIZE = 8
PATTERN_DIM = 32
PATTERN_TYPES = ["sobel", "haar", "gradient", "texture", "color", "frequency"]
@property
def active_pattern_types(self):
return self.PATTERN_TYPES
@property
def n_active_patterns(self):
return len(self.active_pattern_types)
class PatternExtractor:
"""Extract various pattern types from images"""
def __init__(self):
self.pattern_dim = Config.PATTERN_DIM
def sobel_filter(self, patch):
"""Sobel filters for edge detection"""
gray = torch.mean(patch, dim=0).numpy()
sobel_x = cv2.Sobel(gray, cv2.CV_64F, 1, 0, ksize=3)
sobel_y = cv2.Sobel(gray, cv2.CV_64F, 0, 1, ksize=3)
sobel_x = self._normalize_and_reduce(sobel_x)
sobel_y = self._normalize_and_reduce(sobel_y)
return np.concatenate([sobel_x, sobel_y])
def haar_features(self, patch):
"""Haar-like features"""
gray = torch.mean(patch, dim=0).numpy()
h, w = gray.shape
left_half = gray[:, : w // 2]
right_half = gray[:, w // 2 :]
haar_vertical = np.mean(left_half) - np.mean(right_half)
top_half = gray[: h // 2, :]
bottom_half = gray[h // 2 :, :]
haar_horizontal = np.mean(top_half) - np.mean(bottom_half)
haar_features = np.array([haar_vertical, haar_horizontal])
return self._expand_to_dim(haar_features)
def gradient_features(self, patch):
"""Gradient features"""
gradients = []
for c in range(patch.shape[0]):
channel = patch[c].numpy()
grad_x = np.gradient(channel, axis=1)
grad_y = np.gradient(channel, axis=0)
grad_stats = np.array(
[
np.mean(np.abs(grad_x)),
np.mean(np.abs(grad_y)),
np.std(grad_x),
np.std(grad_y),
]
)
gradients.extend(grad_stats)
return self._expand_to_dim(np.array(gradients))
def texture_features(self, patch):
"""Texture features"""
gray = torch.mean(patch, dim=0).numpy()
h, w = gray.shape
lbp_features = []
for i in range(1, h - 1):
for j in range(1, w - 1):
center = gray[i, j]
pattern = 0
for di in [-1, 0, 1]:
for dj in [-1, 0, 1]:
if di == 0 and dj == 0:
continue
if gray[i + di, j + dj] > center:
pattern += 1
lbp_features.append(pattern)
lbp_stats = np.array(
[
np.mean(lbp_features),
np.std(lbp_features),
np.max(lbp_features),
np.min(lbp_features),
]
)
return self._expand_to_dim(lbp_stats)
def color_features(self, patch):
"""Color features"""
color_stats = []
for c in range(patch.shape[0]):
channel = patch[c]
stats = torch.tensor(
[
torch.mean(channel),
torch.std(channel),
torch.max(channel),
torch.min(channel),
]
)
color_stats.extend(stats.numpy())
return self._expand_to_dim(np.array(color_stats))
def frequency_features(self, patch):
"""Frequency features via FFT"""
gray = torch.mean(patch, dim=0).numpy()
fft = np.fft.fft2(gray)
fft_magnitude = np.abs(fft)
freq_stats = np.array(
[
np.mean(fft_magnitude),
np.std(fft_magnitude),
np.sum(fft_magnitude[: len(fft_magnitude) // 4, :]),
np.sum(fft_magnitude[3 * len(fft_magnitude) // 4 :, :]),
]
)
return self._expand_to_dim(freq_stats)
def _normalize_and_reduce(self, array):
"""Normalize and reduce to smaller dimension"""
array = (array - np.min(array)) / (np.max(array) - np.min(array) + 1e-8)
return array.flatten()[: self.pattern_dim // 2]
def _expand_to_dim(self, features):
"""Expand features to required dimension"""
if len(features) >= self.pattern_dim:
return features[: self.pattern_dim]
else:
expanded = np.zeros(self.pattern_dim)
expanded[: len(features)] = features
return expanded
def extract_all_patterns(self, patch):
"""Extract all pattern types"""
patterns = {}
patterns["sobel"] = self.sobel_filter(patch)
patterns["haar"] = self.haar_features(patch)
patterns["gradient"] = self.gradient_features(patch)
patterns["texture"] = self.texture_features(patch)
patterns["color"] = self.color_features(patch)
patterns["frequency"] = self.frequency_features(patch)
return patterns
class PatternReconstructor(nn.Module):
"""Neural network for reconstructing RGB patches from patterns"""
def __init__(self, config):
super(PatternReconstructor, self).__init__()
input_dim = config.n_active_patterns * config.PATTERN_DIM
hidden_dim = 512
self.encoder = nn.Sequential(
nn.Linear(input_dim, hidden_dim),
nn.ReLU(),
nn.Dropout(0.2),
nn.Linear(hidden_dim, hidden_dim // 2),
nn.ReLU(),
nn.Dropout(0.2),
nn.Linear(hidden_dim // 2, hidden_dim // 4),
nn.ReLU(),
)
self.decoder_conv = nn.Sequential(
nn.ConvTranspose2d(128, 128, 4, 1, 0),
nn.ReLU(),
nn.ConvTranspose2d(128, 64, 4, 2, 1),
nn.ReLU(),
nn.ConvTranspose2d(64, 32, 4, 2, 1),
nn.ReLU(),
nn.ConvTranspose2d(32, 3, 3, 1, 1),
nn.Tanh(),
)
def forward(self, patterns):
encoded = self.encoder(patterns)
encoded = encoded.view(-1, 128, 1, 1)
reconstructed = self.decoder_conv(encoded)
return reconstructed
class MultiPatternMarkovChain:
"""Multi-dimensional Markov chain for patterns"""
def __init__(self, config):
self.config = config
self.chains = {}
self.pattern_store = {}
for pattern_type in config.active_pattern_types:
self.chains[pattern_type] = defaultdict(lambda: defaultdict(list))
def generate_patterns(self, grid_size, temperature=1.0):
"""Generate patterns for all active types"""
result = {}
for pattern_type in self.config.active_pattern_types:
if pattern_type in self.chains:
result[pattern_type] = self._generate_single_pattern_grid(
pattern_type, grid_size, temperature
)
return result
def _generate_single_pattern_grid(self, pattern_type, grid_size, temperature):
"""Generate grid for single pattern type"""
chain = self.chains[pattern_type]
grid = [[None for _ in range(grid_size)] for _ in range(grid_size)]
if not chain:
return grid
start_hash = random.choice(list(chain.keys()))
grid[0][0] = start_hash
for i in range(grid_size):
for j in range(grid_size):
if i == 0 and j == 0:
continue
candidates = []
if i > 0 and grid[i - 1][j] is not None:
candidates.extend(chain[grid[i - 1][j]].get("down", []))
if j > 0 and grid[i][j - 1] is not None:
candidates.extend(chain[grid[i][j - 1]].get("right", []))
if candidates:
grid[i][j] = self._softmax_selection(candidates, temperature)
else:
grid[i][j] = random.choice(list(chain.keys()))
return grid
def _softmax_selection(self, candidates, temperature):
"""Selection with temperature consideration"""
if not candidates:
return None
if len(candidates) == 1:
return candidates[0][0]
probs = np.array([c[1] for c in candidates])
if temperature == 0:
max_idx = np.argmax(probs)
return candidates[max_idx][0]
else:
probs = probs / temperature
exp_probs = np.exp(probs - np.max(probs))
softmax_probs = exp_probs / np.sum(exp_probs)
idx = np.random.choice(len(candidates), p=softmax_probs)
return candidates[idx][0]
class VisualMarkovGenerator:
"""Main image generator class"""
def __init__(
self,
model_path="pattern_reconstructor_final.pth",
markov_path="markov_chains.pkl",
):
self.config = Config()
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
self.model = PatternReconstructor(self.config).to(self.device)
if os.path.exists(model_path):
self.model.load_state_dict(torch.load(model_path, map_location=self.device))
print(f"Model loaded from {model_path}")
else:
print(f"Model file {model_path} not found. Using random weights.")
if os.path.exists(markov_path):
with open(markov_path, "rb") as f:
save_data = pickle.load(f)
self.markov_chain = MultiPatternMarkovChain(self.config)
self.markov_chain.pattern_store = save_data["pattern_store"]
for pattern_type in self.config.active_pattern_types:
if pattern_type in save_data["chains"]:
chain_data = save_data["chains"][pattern_type]
for state, directions in chain_data.items():
for direction, candidates in directions.items():
self.markov_chain.chains[pattern_type][state][
direction
] = candidates
print(f"Markov chains loaded from {markov_path}")
else:
print(f"Markov chains file {markov_path} not found. Creating empty chain.")
self.markov_chain = MultiPatternMarkovChain(self.config)
self.model.eval()
def generate_image(self, temperature=1.0, seed=None):
"""Generate image with specified temperature"""
if seed is not None:
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
pattern_grids = self.markov_chain.generate_patterns(
self.config.GRID_SIZE, temperature=temperature
)
if not pattern_grids:
print("Warning: empty patterns for generation")
return self._create_random_image()
result_image = self._generate_image_from_patterns(pattern_grids)
img_np = torch.clamp(result_image, 0, 1).permute(1, 2, 0).numpy()
img_np = (img_np * 255).astype(np.uint8)
return Image.fromarray(img_np)
def _generate_image_from_patterns(self, pattern_grids):
"""Generate image from patterns"""
first_pattern_type = list(pattern_grids.keys())[0]
grid_size = len(pattern_grids[first_pattern_type])
result_image = torch.zeros(
(3, grid_size * self.config.PATCH_SIZE, grid_size * self.config.PATCH_SIZE)
)
with torch.no_grad():
for i in range(grid_size):
for j in range(grid_size):
pattern_vector = []
for pattern_type in self.config.active_pattern_types:
if pattern_type in pattern_grids:
pattern_hash = pattern_grids[pattern_type][i][j]
if pattern_hash in self.markov_chain.pattern_store:
pattern = self.markov_chain.pattern_store[pattern_hash]
pattern_vector.extend(pattern)
else:
pattern_vector.extend(np.zeros(self.config.PATTERN_DIM))
else:
pattern_vector.extend(np.zeros(self.config.PATTERN_DIM))
if len(pattern_vector) > 0:
pattern_tensor = (
torch.FloatTensor(pattern_vector)
.unsqueeze(0)
.to(self.device)
)
reconstructed_patch = (
self.model(pattern_tensor).cpu().squeeze(0)
)
start_i = i * self.config.PATCH_SIZE
end_i = start_i + self.config.PATCH_SIZE
start_j = j * self.config.PATCH_SIZE
end_j = start_j + self.config.PATCH_SIZE
result_image[:, start_i:end_i, start_j:end_j] = (
reconstructed_patch
)
return result_image
def _create_random_image(self):
"""Create random image in case of error"""
random_img = torch.rand(3, self.config.IMG_SIZE, self.config.IMG_SIZE)
img_np = (random_img.permute(1, 2, 0).numpy() * 255).astype(np.uint8)
return Image.fromarray(img_np)
def create_generator(
model_path="pattern_reconstructor_final.pth", markov_path="markov_chains.pkl"
):
"""Create generator instance"""
return VisualMarkovGenerator(model_path, markov_path)
if __name__ == "__main__":
generator = create_generator()
stats = generator.get_stats()
print("Generator statistics:")
print(f"Total patterns: {stats['total_patterns']}")
print(f"Model loaded: {stats['model_loaded']}")
img = generator.generate_image(temperature=1.0, seed=42)
print(f"Generated image size: {img.size}")