diff --git a/BiRefNet/README.md b/BiRefNet/README.md new file mode 100644 index 0000000000000000000000000000000000000000..23918cc97c34542ce2ab59977364ad4f81833f01 --- /dev/null +++ b/BiRefNet/README.md @@ -0,0 +1,8 @@ + +--- +license: mit +--- + +This is used to store the checkpoints of BiRefNet, please refer the following repo link +1. Official implement https://github.com/zhengpeng7/birefnet +2. ComfyUI BiRefNet node https://github.com/viperyl/ComfyUI-BiRefNet \ No newline at end of file diff --git a/BiRefNet/RMBG-2.0/BiRefNet_config.py b/BiRefNet/RMBG-2.0/BiRefNet_config.py new file mode 100644 index 0000000000000000000000000000000000000000..37c8ac58bec2f52dac34204978a7b61b69e3da76 --- /dev/null +++ b/BiRefNet/RMBG-2.0/BiRefNet_config.py @@ -0,0 +1,11 @@ +from transformers import PretrainedConfig + +class BiRefNetConfig(PretrainedConfig): + model_type = "SegformerForSemanticSegmentation" + def __init__( + self, + bb_pretrained=False, + **kwargs + ): + self.bb_pretrained = bb_pretrained + super().__init__(**kwargs) diff --git a/BiRefNet/RMBG-2.0/birefnet.py b/BiRefNet/RMBG-2.0/birefnet.py new file mode 100644 index 0000000000000000000000000000000000000000..d68d7d39d6b4537d3dac2be8dfd7d63b72d4b83d --- /dev/null +++ b/BiRefNet/RMBG-2.0/birefnet.py @@ -0,0 +1,2244 @@ +### config.py + +import os +import math +from transformers import PretrainedConfig + +class Config(PretrainedConfig): + def __init__(self) -> None: + # PATH settings + self.sys_home_dir = os.path.expanduser('~') # Make up your file system as: SYS_HOME_DIR/codes/dis/BiRefNet, SYS_HOME_DIR/datasets/dis/xx, SYS_HOME_DIR/weights/xx + + # TASK settings + self.task = ['DIS5K', 'COD', 'HRSOD', 'DIS5K+HRSOD+HRS10K', 'P3M-10k'][0] + self.training_set = { + 'DIS5K': ['DIS-TR', 'DIS-TR+DIS-TE1+DIS-TE2+DIS-TE3+DIS-TE4'][0], + 'COD': 'TR-COD10K+TR-CAMO', + 'HRSOD': ['TR-DUTS', 'TR-HRSOD', 'TR-UHRSD', 'TR-DUTS+TR-HRSOD', 'TR-DUTS+TR-UHRSD', 'TR-HRSOD+TR-UHRSD', 'TR-DUTS+TR-HRSOD+TR-UHRSD'][5], + 'DIS5K+HRSOD+HRS10K': 'DIS-TE1+DIS-TE2+DIS-TE3+DIS-TE4+DIS-TR+TE-HRS10K+TE-HRSOD+TE-UHRSD+TR-HRS10K+TR-HRSOD+TR-UHRSD', # leave DIS-VD for evaluation. + 'P3M-10k': 'TR-P3M-10k', + }[self.task] + self.prompt4loc = ['dense', 'sparse'][0] + + # Faster-Training settings + self.load_all = True + self.compile = True # 1. Trigger CPU memory leak in some extend, which is an inherent problem of PyTorch. + # Machines with > 70GB CPU memory can run the whole training on DIS5K with default setting. + # 2. Higher PyTorch version may fix it: https://github.com/pytorch/pytorch/issues/119607. + # 3. But compile in Pytorch > 2.0.1 seems to bring no acceleration for training. + self.precisionHigh = True + + # MODEL settings + self.ms_supervision = True + self.out_ref = self.ms_supervision and True + self.dec_ipt = True + self.dec_ipt_split = True + self.cxt_num = [0, 3][1] # multi-scale skip connections from encoder + self.mul_scl_ipt = ['', 'add', 'cat'][2] + self.dec_att = ['', 'ASPP', 'ASPPDeformable'][2] + self.squeeze_block = ['', 'BasicDecBlk_x1', 'ResBlk_x4', 'ASPP_x3', 'ASPPDeformable_x3'][1] + self.dec_blk = ['BasicDecBlk', 'ResBlk', 'HierarAttDecBlk'][0] + + # TRAINING settings + self.batch_size = 4 + self.IoU_finetune_last_epochs = [ + 0, + { + 'DIS5K': -50, + 'COD': -20, + 'HRSOD': -20, + 'DIS5K+HRSOD+HRS10K': -20, + 'P3M-10k': -20, + }[self.task] + ][1] # choose 0 to skip + self.lr = (1e-4 if 'DIS5K' in self.task else 1e-5) * math.sqrt(self.batch_size / 4) # DIS needs high lr to converge faster. Adapt the lr linearly + self.size = 1024 + self.num_workers = max(4, self.batch_size) # will be decrease to min(it, batch_size) at the initialization of the data_loader + + # Backbone settings + self.bb = [ + 'vgg16', 'vgg16bn', 'resnet50', # 0, 1, 2 + 'swin_v1_t', 'swin_v1_s', # 3, 4 + 'swin_v1_b', 'swin_v1_l', # 5-bs9, 6-bs4 + 'pvt_v2_b0', 'pvt_v2_b1', # 7, 8 + 'pvt_v2_b2', 'pvt_v2_b5', # 9-bs10, 10-bs5 + ][6] + self.lateral_channels_in_collection = { + 'vgg16': [512, 256, 128, 64], 'vgg16bn': [512, 256, 128, 64], 'resnet50': [1024, 512, 256, 64], + 'pvt_v2_b2': [512, 320, 128, 64], 'pvt_v2_b5': [512, 320, 128, 64], + 'swin_v1_b': [1024, 512, 256, 128], 'swin_v1_l': [1536, 768, 384, 192], + 'swin_v1_t': [768, 384, 192, 96], 'swin_v1_s': [768, 384, 192, 96], + 'pvt_v2_b0': [256, 160, 64, 32], 'pvt_v2_b1': [512, 320, 128, 64], + }[self.bb] + if self.mul_scl_ipt == 'cat': + self.lateral_channels_in_collection = [channel * 2 for channel in self.lateral_channels_in_collection] + self.cxt = self.lateral_channels_in_collection[1:][::-1][-self.cxt_num:] if self.cxt_num else [] + + # MODEL settings - inactive + self.lat_blk = ['BasicLatBlk'][0] + self.dec_channels_inter = ['fixed', 'adap'][0] + self.refine = ['', 'itself', 'RefUNet', 'Refiner', 'RefinerPVTInChannels4'][0] + self.progressive_ref = self.refine and True + self.ender = self.progressive_ref and False + self.scale = self.progressive_ref and 2 + self.auxiliary_classification = False # Only for DIS5K, where class labels are saved in `dataset.py`. + self.refine_iteration = 1 + self.freeze_bb = False + self.model = [ + 'BiRefNet', + ][0] + if self.dec_blk == 'HierarAttDecBlk': + self.batch_size = 2 ** [0, 1, 2, 3, 4][2] + + # TRAINING settings - inactive + self.preproc_methods = ['flip', 'enhance', 'rotate', 'pepper', 'crop'][:4] + self.optimizer = ['Adam', 'AdamW'][1] + self.lr_decay_epochs = [1e5] # Set to negative N to decay the lr in the last N-th epoch. + self.lr_decay_rate = 0.5 + # Loss + self.lambdas_pix_last = { + # not 0 means opening this loss + # original rate -- 1 : 30 : 1.5 : 0.2, bce x 30 + 'bce': 30 * 1, # high performance + 'iou': 0.5 * 1, # 0 / 255 + 'iou_patch': 0.5 * 0, # 0 / 255, win_size = (64, 64) + 'mse': 150 * 0, # can smooth the saliency map + 'triplet': 3 * 0, + 'reg': 100 * 0, + 'ssim': 10 * 1, # help contours, + 'cnt': 5 * 0, # help contours + 'structure': 5 * 0, # structure loss from codes of MVANet. A little improvement on DIS-TE[1,2,3], a bit more decrease on DIS-TE4. + } + self.lambdas_cls = { + 'ce': 5.0 + } + # Adv + self.lambda_adv_g = 10. * 0 # turn to 0 to avoid adv training + self.lambda_adv_d = 3. * (self.lambda_adv_g > 0) + + # PATH settings - inactive + self.data_root_dir = os.path.join(self.sys_home_dir, 'datasets/dis') + self.weights_root_dir = os.path.join(self.sys_home_dir, 'weights') + self.weights = { + 'pvt_v2_b2': os.path.join(self.weights_root_dir, 'pvt_v2_b2.pth'), + 'pvt_v2_b5': os.path.join(self.weights_root_dir, ['pvt_v2_b5.pth', 'pvt_v2_b5_22k.pth'][0]), + 'swin_v1_b': os.path.join(self.weights_root_dir, ['swin_base_patch4_window12_384_22kto1k.pth', 'swin_base_patch4_window12_384_22k.pth'][0]), + 'swin_v1_l': os.path.join(self.weights_root_dir, ['swin_large_patch4_window12_384_22kto1k.pth', 'swin_large_patch4_window12_384_22k.pth'][0]), + 'swin_v1_t': os.path.join(self.weights_root_dir, ['swin_tiny_patch4_window7_224_22kto1k_finetune.pth'][0]), + 'swin_v1_s': os.path.join(self.weights_root_dir, ['swin_small_patch4_window7_224_22kto1k_finetune.pth'][0]), + 'pvt_v2_b0': os.path.join(self.weights_root_dir, ['pvt_v2_b0.pth'][0]), + 'pvt_v2_b1': os.path.join(self.weights_root_dir, ['pvt_v2_b1.pth'][0]), + } + + # Callbacks - inactive + self.verbose_eval = True + self.only_S_MAE = False + self.use_fp16 = False # Bugs. It may cause nan in training. + self.SDPA_enabled = False # Bugs. Slower and errors occur in multi-GPUs + + # others + self.device = [0, 'cpu'][0] # .to(0) == .to('cuda:0') + + self.batch_size_valid = 1 + self.rand_seed = 7 + # run_sh_file = [f for f in os.listdir('.') if 'train.sh' == f] + [os.path.join('..', f) for f in os.listdir('..') if 'train.sh' == f] + # with open(run_sh_file[0], 'r') as f: + # lines = f.readlines() + # self.save_last = int([l.strip() for l in lines if '"{}")'.format(self.task) in l and 'val_last=' in l][0].split('val_last=')[-1].split()[0]) + # self.save_step = int([l.strip() for l in lines if '"{}")'.format(self.task) in l and 'step=' in l][0].split('step=')[-1].split()[0]) + # self.val_step = [0, self.save_step][0] + + def print_task(self) -> None: + # Return task for choosing settings in shell scripts. + print(self.task) + + + +### models/backbones/pvt_v2.py + +import torch +import torch.nn as nn +from functools import partial + +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ +from timm.models.registry import register_model + +import math + +# from config import Config + +# config = Config() + +class Mlp(nn.Module): + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.dwconv = DWConv(hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + x = self.fc1(x) + x = self.dwconv(x, H, W) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +class Attention(nn.Module): + def __init__(self, dim, num_heads=8, qkv_bias=False, qk_scale=None, attn_drop=0., proj_drop=0., sr_ratio=1): + super().__init__() + assert dim % num_heads == 0, f"dim {dim} should be divided by num_heads {num_heads}." + + self.dim = dim + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + self.q = nn.Linear(dim, dim, bias=qkv_bias) + self.kv = nn.Linear(dim, dim * 2, bias=qkv_bias) + self.attn_drop_prob = attn_drop + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + self.sr_ratio = sr_ratio + if sr_ratio > 1: + self.sr = nn.Conv2d(dim, dim, kernel_size=sr_ratio, stride=sr_ratio) + self.norm = nn.LayerNorm(dim) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + B, N, C = x.shape + q = self.q(x).reshape(B, N, self.num_heads, C // self.num_heads).permute(0, 2, 1, 3) + + if self.sr_ratio > 1: + x_ = x.permute(0, 2, 1).reshape(B, C, H, W) + x_ = self.sr(x_).reshape(B, C, -1).permute(0, 2, 1) + x_ = self.norm(x_) + kv = self.kv(x_).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + else: + kv = self.kv(x).reshape(B, -1, 2, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + k, v = kv[0], kv[1] + + if config.SDPA_enabled: + x = torch.nn.functional.scaled_dot_product_attention( + q, k, v, + attn_mask=None, dropout_p=self.attn_drop_prob, is_causal=False + ).transpose(1, 2).reshape(B, N, C) + else: + attn = (q @ k.transpose(-2, -1)) * self.scale + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, C) + x = self.proj(x) + x = self.proj_drop(x) + + return x + + +class Block(nn.Module): + + def __init__(self, dim, num_heads, mlp_ratio=4., qkv_bias=False, qk_scale=None, drop=0., attn_drop=0., + drop_path=0., act_layer=nn.GELU, norm_layer=nn.LayerNorm, sr_ratio=1): + super().__init__() + self.norm1 = norm_layer(dim) + self.attn = Attention( + dim, + num_heads=num_heads, qkv_bias=qkv_bias, qk_scale=qk_scale, + attn_drop=attn_drop, proj_drop=drop, sr_ratio=sr_ratio) + # NOTE: drop path for stochastic depth, we shall see if this is better than dropout here + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x, H, W): + x = x + self.drop_path(self.attn(self.norm1(x), H, W)) + x = x + self.drop_path(self.mlp(self.norm2(x), H, W)) + + return x + + +class OverlapPatchEmbed(nn.Module): + """ Image to Patch Embedding + """ + + def __init__(self, img_size=224, patch_size=7, stride=4, in_channels=3, embed_dim=768): + super().__init__() + img_size = to_2tuple(img_size) + patch_size = to_2tuple(patch_size) + + self.img_size = img_size + self.patch_size = patch_size + self.H, self.W = img_size[0] // patch_size[0], img_size[1] // patch_size[1] + self.num_patches = self.H * self.W + self.proj = nn.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=stride, + padding=(patch_size[0] // 2, patch_size[1] // 2)) + self.norm = nn.LayerNorm(embed_dim) + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def forward(self, x): + x = self.proj(x) + _, _, H, W = x.shape + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + + return x, H, W + + +class PyramidVisionTransformerImpr(nn.Module): + def __init__(self, img_size=224, patch_size=16, in_channels=3, num_classes=1000, embed_dims=[64, 128, 256, 512], + num_heads=[1, 2, 4, 8], mlp_ratios=[4, 4, 4, 4], qkv_bias=False, qk_scale=None, drop_rate=0., + attn_drop_rate=0., drop_path_rate=0., norm_layer=nn.LayerNorm, + depths=[3, 4, 6, 3], sr_ratios=[8, 4, 2, 1]): + super().__init__() + self.num_classes = num_classes + self.depths = depths + + # patch_embed + self.patch_embed1 = OverlapPatchEmbed(img_size=img_size, patch_size=7, stride=4, in_channels=in_channels, + embed_dim=embed_dims[0]) + self.patch_embed2 = OverlapPatchEmbed(img_size=img_size // 4, patch_size=3, stride=2, in_channels=embed_dims[0], + embed_dim=embed_dims[1]) + self.patch_embed3 = OverlapPatchEmbed(img_size=img_size // 8, patch_size=3, stride=2, in_channels=embed_dims[1], + embed_dim=embed_dims[2]) + self.patch_embed4 = OverlapPatchEmbed(img_size=img_size // 16, patch_size=3, stride=2, in_channels=embed_dims[2], + embed_dim=embed_dims[3]) + + # transformer encoder + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule + cur = 0 + self.block1 = nn.ModuleList([Block( + dim=embed_dims[0], num_heads=num_heads[0], mlp_ratio=mlp_ratios[0], qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + i], norm_layer=norm_layer, + sr_ratio=sr_ratios[0]) + for i in range(depths[0])]) + self.norm1 = norm_layer(embed_dims[0]) + + cur += depths[0] + self.block2 = nn.ModuleList([Block( + dim=embed_dims[1], num_heads=num_heads[1], mlp_ratio=mlp_ratios[1], qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + i], norm_layer=norm_layer, + sr_ratio=sr_ratios[1]) + for i in range(depths[1])]) + self.norm2 = norm_layer(embed_dims[1]) + + cur += depths[1] + self.block3 = nn.ModuleList([Block( + dim=embed_dims[2], num_heads=num_heads[2], mlp_ratio=mlp_ratios[2], qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + i], norm_layer=norm_layer, + sr_ratio=sr_ratios[2]) + for i in range(depths[2])]) + self.norm3 = norm_layer(embed_dims[2]) + + cur += depths[2] + self.block4 = nn.ModuleList([Block( + dim=embed_dims[3], num_heads=num_heads[3], mlp_ratio=mlp_ratios[3], qkv_bias=qkv_bias, qk_scale=qk_scale, + drop=drop_rate, attn_drop=attn_drop_rate, drop_path=dpr[cur + i], norm_layer=norm_layer, + sr_ratio=sr_ratios[3]) + for i in range(depths[3])]) + self.norm4 = norm_layer(embed_dims[3]) + + # classification head + # self.head = nn.Linear(embed_dims[3], num_classes) if num_classes > 0 else nn.Identity() + + self.apply(self._init_weights) + + def _init_weights(self, m): + if isinstance(m, nn.Linear): + trunc_normal_(m.weight, std=.02) + if isinstance(m, nn.Linear) and m.bias is not None: + nn.init.constant_(m.bias, 0) + elif isinstance(m, nn.LayerNorm): + nn.init.constant_(m.bias, 0) + nn.init.constant_(m.weight, 1.0) + elif isinstance(m, nn.Conv2d): + fan_out = m.kernel_size[0] * m.kernel_size[1] * m.out_channels + fan_out //= m.groups + m.weight.data.normal_(0, math.sqrt(2.0 / fan_out)) + if m.bias is not None: + m.bias.data.zero_() + + def init_weights(self, pretrained=None): + if isinstance(pretrained, str): + logger = 1 + #load_checkpoint(self, pretrained, map_location='cpu', strict=False, logger=logger) + + def reset_drop_path(self, drop_path_rate): + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(self.depths))] + cur = 0 + for i in range(self.depths[0]): + self.block1[i].drop_path.drop_prob = dpr[cur + i] + + cur += self.depths[0] + for i in range(self.depths[1]): + self.block2[i].drop_path.drop_prob = dpr[cur + i] + + cur += self.depths[1] + for i in range(self.depths[2]): + self.block3[i].drop_path.drop_prob = dpr[cur + i] + + cur += self.depths[2] + for i in range(self.depths[3]): + self.block4[i].drop_path.drop_prob = dpr[cur + i] + + def freeze_patch_emb(self): + self.patch_embed1.requires_grad = False + + @torch.jit.ignore + def no_weight_decay(self): + return {'pos_embed1', 'pos_embed2', 'pos_embed3', 'pos_embed4', 'cls_token'} # has pos_embed may be better + + def get_classifier(self): + return self.head + + def reset_classifier(self, num_classes, global_pool=''): + self.num_classes = num_classes + self.head = nn.Linear(self.embed_dim, num_classes) if num_classes > 0 else nn.Identity() + + def forward_features(self, x): + B = x.shape[0] + outs = [] + + # stage 1 + x, H, W = self.patch_embed1(x) + for i, blk in enumerate(self.block1): + x = blk(x, H, W) + x = self.norm1(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + outs.append(x) + + # stage 2 + x, H, W = self.patch_embed2(x) + for i, blk in enumerate(self.block2): + x = blk(x, H, W) + x = self.norm2(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + outs.append(x) + + # stage 3 + x, H, W = self.patch_embed3(x) + for i, blk in enumerate(self.block3): + x = blk(x, H, W) + x = self.norm3(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + outs.append(x) + + # stage 4 + x, H, W = self.patch_embed4(x) + for i, blk in enumerate(self.block4): + x = blk(x, H, W) + x = self.norm4(x) + x = x.reshape(B, H, W, -1).permute(0, 3, 1, 2).contiguous() + outs.append(x) + + return outs + + # return x.mean(dim=1) + + def forward(self, x): + x = self.forward_features(x) + # x = self.head(x) + + return x + + +class DWConv(nn.Module): + def __init__(self, dim=768): + super(DWConv, self).__init__() + self.dwconv = nn.Conv2d(dim, dim, 3, 1, 1, bias=True, groups=dim) + + def forward(self, x, H, W): + B, N, C = x.shape + x = x.transpose(1, 2).view(B, C, H, W).contiguous() + x = self.dwconv(x) + x = x.flatten(2).transpose(1, 2) + + return x + + +def _conv_filter(state_dict, patch_size=16): + """ convert patch embedding weight from manual patchify + linear proj to conv""" + out_dict = {} + for k, v in state_dict.items(): + if 'patch_embed.proj.weight' in k: + v = v.reshape((v.shape[0], 3, patch_size, patch_size)) + out_dict[k] = v + + return out_dict + + +## @register_model +class pvt_v2_b0(PyramidVisionTransformerImpr): + def __init__(self, **kwargs): + super(pvt_v2_b0, self).__init__( + patch_size=4, embed_dims=[32, 64, 160, 256], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4], + qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[2, 2, 2, 2], sr_ratios=[8, 4, 2, 1], + drop_rate=0.0, drop_path_rate=0.1) + + + +## @register_model +class pvt_v2_b1(PyramidVisionTransformerImpr): + def __init__(self, **kwargs): + super(pvt_v2_b1, self).__init__( + patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4], + qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[2, 2, 2, 2], sr_ratios=[8, 4, 2, 1], + drop_rate=0.0, drop_path_rate=0.1) + +## @register_model +class pvt_v2_b2(PyramidVisionTransformerImpr): + def __init__(self, in_channels=3, **kwargs): + super(pvt_v2_b2, self).__init__( + patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4], + qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 6, 3], sr_ratios=[8, 4, 2, 1], + drop_rate=0.0, drop_path_rate=0.1, in_channels=in_channels) + +## @register_model +class pvt_v2_b3(PyramidVisionTransformerImpr): + def __init__(self, **kwargs): + super(pvt_v2_b3, self).__init__( + patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4], + qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 4, 18, 3], sr_ratios=[8, 4, 2, 1], + drop_rate=0.0, drop_path_rate=0.1) + +## @register_model +class pvt_v2_b4(PyramidVisionTransformerImpr): + def __init__(self, **kwargs): + super(pvt_v2_b4, self).__init__( + patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[8, 8, 4, 4], + qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 8, 27, 3], sr_ratios=[8, 4, 2, 1], + drop_rate=0.0, drop_path_rate=0.1) + + +## @register_model +class pvt_v2_b5(PyramidVisionTransformerImpr): + def __init__(self, **kwargs): + super(pvt_v2_b5, self).__init__( + patch_size=4, embed_dims=[64, 128, 320, 512], num_heads=[1, 2, 5, 8], mlp_ratios=[4, 4, 4, 4], + qkv_bias=True, norm_layer=partial(nn.LayerNorm, eps=1e-6), depths=[3, 6, 40, 3], sr_ratios=[8, 4, 2, 1], + drop_rate=0.0, drop_path_rate=0.1) + + + +### models/backbones/swin_v1.py + +# -------------------------------------------------------- +# Swin Transformer +# Copyright (c) 2021 Microsoft +# Licensed under The MIT License [see LICENSE for details] +# Written by Ze Liu, Yutong Lin, Yixuan Wei +# -------------------------------------------------------- + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +import numpy as np +from timm.models.layers import DropPath, to_2tuple, trunc_normal_ + +# from config import Config + + +# config = Config() + +class Mlp(nn.Module): + """ Multilayer perceptron.""" + + def __init__(self, in_features, hidden_features=None, out_features=None, act_layer=nn.GELU, drop=0.): + super().__init__() + out_features = out_features or in_features + hidden_features = hidden_features or in_features + self.fc1 = nn.Linear(in_features, hidden_features) + self.act = act_layer() + self.fc2 = nn.Linear(hidden_features, out_features) + self.drop = nn.Dropout(drop) + + def forward(self, x): + x = self.fc1(x) + x = self.act(x) + x = self.drop(x) + x = self.fc2(x) + x = self.drop(x) + return x + + +def window_partition(x, window_size): + """ + Args: + x: (B, H, W, C) + window_size (int): window size + + Returns: + windows: (num_windows*B, window_size, window_size, C) + """ + B, H, W, C = x.shape + x = x.view(B, H // window_size, window_size, W // window_size, window_size, C) + windows = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(-1, window_size, window_size, C) + return windows + + +def window_reverse(windows, window_size, H, W): + """ + Args: + windows: (num_windows*B, window_size, window_size, C) + window_size (int): Window size + H (int): Height of image + W (int): Width of image + + Returns: + x: (B, H, W, C) + """ + B = int(windows.shape[0] / (H * W / window_size / window_size)) + x = windows.view(B, H // window_size, W // window_size, window_size, window_size, -1) + x = x.permute(0, 1, 3, 2, 4, 5).contiguous().view(B, H, W, -1) + return x + + +class WindowAttention(nn.Module): + """ Window based multi-head self attention (W-MSA) module with relative position bias. + It supports both of shifted and non-shifted window. + + Args: + dim (int): Number of input channels. + window_size (tuple[int]): The height and width of the window. + num_heads (int): Number of attention heads. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set + attn_drop (float, optional): Dropout ratio of attention weight. Default: 0.0 + proj_drop (float, optional): Dropout ratio of output. Default: 0.0 + """ + + def __init__(self, dim, window_size, num_heads, qkv_bias=True, qk_scale=None, attn_drop=0., proj_drop=0.): + + super().__init__() + self.dim = dim + self.window_size = window_size # Wh, Ww + self.num_heads = num_heads + head_dim = dim // num_heads + self.scale = qk_scale or head_dim ** -0.5 + + # define a parameter table of relative position bias + self.relative_position_bias_table = nn.Parameter( + torch.zeros((2 * window_size[0] - 1) * (2 * window_size[1] - 1), num_heads)) # 2*Wh-1 * 2*Ww-1, nH + + # get pair-wise relative position index for each token inside the window + coords_h = torch.arange(self.window_size[0]) + coords_w = torch.arange(self.window_size[1]) + coords = torch.stack(torch.meshgrid([coords_h, coords_w], indexing='ij')) # 2, Wh, Ww + coords_flatten = torch.flatten(coords, 1) # 2, Wh*Ww + relative_coords = coords_flatten[:, :, None] - coords_flatten[:, None, :] # 2, Wh*Ww, Wh*Ww + relative_coords = relative_coords.permute(1, 2, 0).contiguous() # Wh*Ww, Wh*Ww, 2 + relative_coords[:, :, 0] += self.window_size[0] - 1 # shift to start from 0 + relative_coords[:, :, 1] += self.window_size[1] - 1 + relative_coords[:, :, 0] *= 2 * self.window_size[1] - 1 + relative_position_index = relative_coords.sum(-1) # Wh*Ww, Wh*Ww + self.register_buffer("relative_position_index", relative_position_index) + + self.qkv = nn.Linear(dim, dim * 3, bias=qkv_bias) + self.attn_drop_prob = attn_drop + self.attn_drop = nn.Dropout(attn_drop) + self.proj = nn.Linear(dim, dim) + self.proj_drop = nn.Dropout(proj_drop) + + trunc_normal_(self.relative_position_bias_table, std=.02) + self.softmax = nn.Softmax(dim=-1) + + def forward(self, x, mask=None): + """ Forward function. + + Args: + x: input features with shape of (num_windows*B, N, C) + mask: (0/-inf) mask with shape of (num_windows, Wh*Ww, Wh*Ww) or None + """ + B_, N, C = x.shape + qkv = self.qkv(x).reshape(B_, N, 3, self.num_heads, C // self.num_heads).permute(2, 0, 3, 1, 4) + q, k, v = qkv[0], qkv[1], qkv[2] # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + + if config.SDPA_enabled: + x = torch.nn.functional.scaled_dot_product_attention( + q, k, v, + attn_mask=None, dropout_p=self.attn_drop_prob, is_causal=False + ).transpose(1, 2).reshape(B_, N, C) + else: + attn = (q @ k.transpose(-2, -1)) + + relative_position_bias = self.relative_position_bias_table[self.relative_position_index.view(-1)].view( + self.window_size[0] * self.window_size[1], self.window_size[0] * self.window_size[1], -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + attn = attn + relative_position_bias.unsqueeze(0) + + if mask is not None: + nW = mask.shape[0] + attn = attn.view(B_ // nW, nW, self.num_heads, N, N) + mask.unsqueeze(1).unsqueeze(0) + attn = attn.view(-1, self.num_heads, N, N) + attn = self.softmax(attn) + else: + attn = self.softmax(attn) + + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B_, N, C) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +class SwinTransformerBlock(nn.Module): + """ Swin Transformer Block. + + Args: + dim (int): Number of input channels. + num_heads (int): Number of attention heads. + window_size (int): Window size. + shift_size (int): Shift size for SW-MSA. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float, optional): Stochastic depth rate. Default: 0.0 + act_layer (nn.Module, optional): Activation layer. Default: nn.GELU + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + + def __init__(self, dim, num_heads, window_size=7, shift_size=0, + mlp_ratio=4., qkv_bias=True, qk_scale=None, drop=0., attn_drop=0., drop_path=0., + act_layer=nn.GELU, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.window_size = window_size + self.shift_size = shift_size + self.mlp_ratio = mlp_ratio + assert 0 <= self.shift_size < self.window_size, "shift_size must in 0-window_size" + + self.norm1 = norm_layer(dim) + self.attn = WindowAttention( + dim, window_size=to_2tuple(self.window_size), num_heads=num_heads, + qkv_bias=qkv_bias, qk_scale=qk_scale, attn_drop=attn_drop, proj_drop=drop) + + self.drop_path = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.norm2 = norm_layer(dim) + mlp_hidden_dim = int(dim * mlp_ratio) + self.mlp = Mlp(in_features=dim, hidden_features=mlp_hidden_dim, act_layer=act_layer, drop=drop) + + self.H = None + self.W = None + + def forward(self, x, mask_matrix): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + mask_matrix: Attention mask for cyclic shift. + """ + B, L, C = x.shape + H, W = self.H, self.W + assert L == H * W, "input feature has wrong size" + + shortcut = x + x = self.norm1(x) + x = x.view(B, H, W, C) + + # pad feature maps to multiples of window size + pad_l = pad_t = 0 + pad_r = (self.window_size - W % self.window_size) % self.window_size + pad_b = (self.window_size - H % self.window_size) % self.window_size + x = F.pad(x, (0, 0, pad_l, pad_r, pad_t, pad_b)) + _, Hp, Wp, _ = x.shape + + # cyclic shift + if self.shift_size > 0: + shifted_x = torch.roll(x, shifts=(-self.shift_size, -self.shift_size), dims=(1, 2)) + attn_mask = mask_matrix + else: + shifted_x = x + attn_mask = None + + # partition windows + x_windows = window_partition(shifted_x, self.window_size) # nW*B, window_size, window_size, C + x_windows = x_windows.view(-1, self.window_size * self.window_size, C) # nW*B, window_size*window_size, C + + # W-MSA/SW-MSA + attn_windows = self.attn(x_windows, mask=attn_mask) # nW*B, window_size*window_size, C + + # merge windows + attn_windows = attn_windows.view(-1, self.window_size, self.window_size, C) + shifted_x = window_reverse(attn_windows, self.window_size, Hp, Wp) # B H' W' C + + # reverse cyclic shift + if self.shift_size > 0: + x = torch.roll(shifted_x, shifts=(self.shift_size, self.shift_size), dims=(1, 2)) + else: + x = shifted_x + + if pad_r > 0 or pad_b > 0: + x = x[:, :H, :W, :].contiguous() + + x = x.view(B, H * W, C) + + # FFN + x = shortcut + self.drop_path(x) + x = x + self.drop_path(self.mlp(self.norm2(x))) + + return x + + +class PatchMerging(nn.Module): + """ Patch Merging Layer + + Args: + dim (int): Number of input channels. + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + """ + def __init__(self, dim, norm_layer=nn.LayerNorm): + super().__init__() + self.dim = dim + self.reduction = nn.Linear(4 * dim, 2 * dim, bias=False) + self.norm = norm_layer(4 * dim) + + def forward(self, x, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + B, L, C = x.shape + assert L == H * W, "input feature has wrong size" + + x = x.view(B, H, W, C) + + # padding + pad_input = (H % 2 == 1) or (W % 2 == 1) + if pad_input: + x = F.pad(x, (0, 0, 0, W % 2, 0, H % 2)) + + x0 = x[:, 0::2, 0::2, :] # B H/2 W/2 C + x1 = x[:, 1::2, 0::2, :] # B H/2 W/2 C + x2 = x[:, 0::2, 1::2, :] # B H/2 W/2 C + x3 = x[:, 1::2, 1::2, :] # B H/2 W/2 C + x = torch.cat([x0, x1, x2, x3], -1) # B H/2 W/2 4*C + x = x.view(B, -1, 4 * C) # B H/2*W/2 4*C + + x = self.norm(x) + x = self.reduction(x) + + return x + + +class BasicLayer(nn.Module): + """ A basic Swin Transformer layer for one stage. + + Args: + dim (int): Number of feature channels + depth (int): Depths of this stage. + num_heads (int): Number of attention head. + window_size (int): Local window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool, optional): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float | None, optional): Override default qk scale of head_dim ** -0.5 if set. + drop (float, optional): Dropout rate. Default: 0.0 + attn_drop (float, optional): Attention dropout rate. Default: 0.0 + drop_path (float | tuple[float], optional): Stochastic depth rate. Default: 0.0 + norm_layer (nn.Module, optional): Normalization layer. Default: nn.LayerNorm + downsample (nn.Module | None, optional): Downsample layer at the end of the layer. Default: None + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + dim, + depth, + num_heads, + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop=0., + attn_drop=0., + drop_path=0., + norm_layer=nn.LayerNorm, + downsample=None, + use_checkpoint=False): + super().__init__() + self.window_size = window_size + self.shift_size = window_size // 2 + self.depth = depth + self.use_checkpoint = use_checkpoint + + # build blocks + self.blocks = nn.ModuleList([ + SwinTransformerBlock( + dim=dim, + num_heads=num_heads, + window_size=window_size, + shift_size=0 if (i % 2 == 0) else window_size // 2, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop, + attn_drop=attn_drop, + drop_path=drop_path[i] if isinstance(drop_path, list) else drop_path, + norm_layer=norm_layer) + for i in range(depth)]) + + # patch merging layer + if downsample is not None: + self.downsample = downsample(dim=dim, norm_layer=norm_layer) + else: + self.downsample = None + + def forward(self, x, H, W): + """ Forward function. + + Args: + x: Input feature, tensor size (B, H*W, C). + H, W: Spatial resolution of the input feature. + """ + + # calculate attention mask for SW-MSA + Hp = int(np.ceil(H / self.window_size)) * self.window_size + Wp = int(np.ceil(W / self.window_size)) * self.window_size + img_mask = torch.zeros((1, Hp, Wp, 1), device=x.device) # 1 Hp Wp 1 + h_slices = (slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None)) + w_slices = (slice(0, -self.window_size), + slice(-self.window_size, -self.shift_size), + slice(-self.shift_size, None)) + cnt = 0 + for h in h_slices: + for w in w_slices: + img_mask[:, h, w, :] = cnt + cnt += 1 + + mask_windows = window_partition(img_mask, self.window_size) # nW, window_size, window_size, 1 + mask_windows = mask_windows.view(-1, self.window_size * self.window_size) + attn_mask = mask_windows.unsqueeze(1) - mask_windows.unsqueeze(2) + attn_mask = attn_mask.masked_fill(attn_mask != 0, float(-100.0)).masked_fill(attn_mask == 0, float(0.0)) + + for blk in self.blocks: + blk.H, blk.W = H, W + if self.use_checkpoint: + x = checkpoint.checkpoint(blk, x, attn_mask) + else: + x = blk(x, attn_mask) + if self.downsample is not None: + x_down = self.downsample(x, H, W) + Wh, Ww = (H + 1) // 2, (W + 1) // 2 + return x, H, W, x_down, Wh, Ww + else: + return x, H, W, x, H, W + + +class PatchEmbed(nn.Module): + """ Image to Patch Embedding + + Args: + patch_size (int): Patch token size. Default: 4. + in_channels (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + norm_layer (nn.Module, optional): Normalization layer. Default: None + """ + + def __init__(self, patch_size=4, in_channels=3, embed_dim=96, norm_layer=None): + super().__init__() + patch_size = to_2tuple(patch_size) + self.patch_size = patch_size + + self.in_channels = in_channels + self.embed_dim = embed_dim + + self.proj = nn.Conv2d(in_channels, embed_dim, kernel_size=patch_size, stride=patch_size) + if norm_layer is not None: + self.norm = norm_layer(embed_dim) + else: + self.norm = None + + def forward(self, x): + """Forward function.""" + # padding + _, _, H, W = x.size() + if W % self.patch_size[1] != 0: + x = F.pad(x, (0, self.patch_size[1] - W % self.patch_size[1])) + if H % self.patch_size[0] != 0: + x = F.pad(x, (0, 0, 0, self.patch_size[0] - H % self.patch_size[0])) + + x = self.proj(x) # B C Wh Ww + if self.norm is not None: + Wh, Ww = x.size(2), x.size(3) + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + x = x.transpose(1, 2).view(-1, self.embed_dim, Wh, Ww) + + return x + + +class SwinTransformer(nn.Module): + """ Swin Transformer backbone. + A PyTorch impl of : `Swin Transformer: Hierarchical Vision Transformer using Shifted Windows` - + https://arxiv.org/pdf/2103.14030 + + Args: + pretrain_img_size (int): Input image size for training the pretrained model, + used in absolute postion embedding. Default 224. + patch_size (int | tuple(int)): Patch size. Default: 4. + in_channels (int): Number of input image channels. Default: 3. + embed_dim (int): Number of linear projection output channels. Default: 96. + depths (tuple[int]): Depths of each Swin Transformer stage. + num_heads (tuple[int]): Number of attention head of each stage. + window_size (int): Window size. Default: 7. + mlp_ratio (float): Ratio of mlp hidden dim to embedding dim. Default: 4. + qkv_bias (bool): If True, add a learnable bias to query, key, value. Default: True + qk_scale (float): Override default qk scale of head_dim ** -0.5 if set. + drop_rate (float): Dropout rate. + attn_drop_rate (float): Attention dropout rate. Default: 0. + drop_path_rate (float): Stochastic depth rate. Default: 0.2. + norm_layer (nn.Module): Normalization layer. Default: nn.LayerNorm. + ape (bool): If True, add absolute position embedding to the patch embedding. Default: False. + patch_norm (bool): If True, add normalization after patch embedding. Default: True. + out_indices (Sequence[int]): Output from which stages. + frozen_stages (int): Stages to be frozen (stop grad and set eval mode). + -1 means not freezing any parameters. + use_checkpoint (bool): Whether to use checkpointing to save memory. Default: False. + """ + + def __init__(self, + pretrain_img_size=224, + patch_size=4, + in_channels=3, + embed_dim=96, + depths=[2, 2, 6, 2], + num_heads=[3, 6, 12, 24], + window_size=7, + mlp_ratio=4., + qkv_bias=True, + qk_scale=None, + drop_rate=0., + attn_drop_rate=0., + drop_path_rate=0.2, + norm_layer=nn.LayerNorm, + ape=False, + patch_norm=True, + out_indices=(0, 1, 2, 3), + frozen_stages=-1, + use_checkpoint=False): + super().__init__() + + self.pretrain_img_size = pretrain_img_size + self.num_layers = len(depths) + self.embed_dim = embed_dim + self.ape = ape + self.patch_norm = patch_norm + self.out_indices = out_indices + self.frozen_stages = frozen_stages + + # split image into non-overlapping patches + self.patch_embed = PatchEmbed( + patch_size=patch_size, in_channels=in_channels, embed_dim=embed_dim, + norm_layer=norm_layer if self.patch_norm else None) + + # absolute position embedding + if self.ape: + pretrain_img_size = to_2tuple(pretrain_img_size) + patch_size = to_2tuple(patch_size) + patches_resolution = [pretrain_img_size[0] // patch_size[0], pretrain_img_size[1] // patch_size[1]] + + self.absolute_pos_embed = nn.Parameter(torch.zeros(1, embed_dim, patches_resolution[0], patches_resolution[1])) + trunc_normal_(self.absolute_pos_embed, std=.02) + + self.pos_drop = nn.Dropout(p=drop_rate) + + # stochastic depth + dpr = [x.item() for x in torch.linspace(0, drop_path_rate, sum(depths))] # stochastic depth decay rule + + # build layers + self.layers = nn.ModuleList() + for i_layer in range(self.num_layers): + layer = BasicLayer( + dim=int(embed_dim * 2 ** i_layer), + depth=depths[i_layer], + num_heads=num_heads[i_layer], + window_size=window_size, + mlp_ratio=mlp_ratio, + qkv_bias=qkv_bias, + qk_scale=qk_scale, + drop=drop_rate, + attn_drop=attn_drop_rate, + drop_path=dpr[sum(depths[:i_layer]):sum(depths[:i_layer + 1])], + norm_layer=norm_layer, + downsample=PatchMerging if (i_layer < self.num_layers - 1) else None, + use_checkpoint=use_checkpoint) + self.layers.append(layer) + + num_features = [int(embed_dim * 2 ** i) for i in range(self.num_layers)] + self.num_features = num_features + + # add a norm layer for each output + for i_layer in out_indices: + layer = norm_layer(num_features[i_layer]) + layer_name = f'norm{i_layer}' + self.add_module(layer_name, layer) + + self._freeze_stages() + + def _freeze_stages(self): + if self.frozen_stages >= 0: + self.patch_embed.eval() + for param in self.patch_embed.parameters(): + param.requires_grad = False + + if self.frozen_stages >= 1 and self.ape: + self.absolute_pos_embed.requires_grad = False + + if self.frozen_stages >= 2: + self.pos_drop.eval() + for i in range(0, self.frozen_stages - 1): + m = self.layers[i] + m.eval() + for param in m.parameters(): + param.requires_grad = False + + + def forward(self, x): + """Forward function.""" + x = self.patch_embed(x) + + Wh, Ww = x.size(2), x.size(3) + if self.ape: + # interpolate the position embedding to the corresponding size + absolute_pos_embed = F.interpolate(self.absolute_pos_embed, size=(Wh, Ww), mode='bicubic') + x = (x + absolute_pos_embed) # B Wh*Ww C + + outs = []#x.contiguous()] + x = x.flatten(2).transpose(1, 2) + x = self.pos_drop(x) + for i in range(self.num_layers): + layer = self.layers[i] + x_out, H, W, x, Wh, Ww = layer(x, Wh, Ww) + + if i in self.out_indices: + norm_layer = getattr(self, f'norm{i}') + x_out = norm_layer(x_out) + + out = x_out.view(-1, H, W, self.num_features[i]).permute(0, 3, 1, 2).contiguous() + outs.append(out) + + return tuple(outs) + + def train(self, mode=True): + """Convert the model into training mode while keep layers freezed.""" + super(SwinTransformer, self).train(mode) + self._freeze_stages() + +def swin_v1_t(): + model = SwinTransformer(embed_dim=96, depths=[2, 2, 6, 2], num_heads=[3, 6, 12, 24], window_size=7) + return model + +def swin_v1_s(): + model = SwinTransformer(embed_dim=96, depths=[2, 2, 18, 2], num_heads=[3, 6, 12, 24], window_size=7) + return model + +def swin_v1_b(): + model = SwinTransformer(embed_dim=128, depths=[2, 2, 18, 2], num_heads=[4, 8, 16, 32], window_size=12) + return model + +def swin_v1_l(): + model = SwinTransformer(embed_dim=192, depths=[2, 2, 18, 2], num_heads=[6, 12, 24, 48], window_size=12) + return model + + + +### models/modules/deform_conv.py + +import torch +import torch.nn as nn +from torchvision.ops import deform_conv2d + + +class DeformableConv2d(nn.Module): + def __init__(self, + in_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1, + bias=False): + + super(DeformableConv2d, self).__init__() + + assert type(kernel_size) == tuple or type(kernel_size) == int + + kernel_size = kernel_size if type(kernel_size) == tuple else (kernel_size, kernel_size) + self.stride = stride if type(stride) == tuple else (stride, stride) + self.padding = padding + + self.offset_conv = nn.Conv2d(in_channels, + 2 * kernel_size[0] * kernel_size[1], + kernel_size=kernel_size, + stride=stride, + padding=self.padding, + bias=True) + + nn.init.constant_(self.offset_conv.weight, 0.) + nn.init.constant_(self.offset_conv.bias, 0.) + + self.modulator_conv = nn.Conv2d(in_channels, + 1 * kernel_size[0] * kernel_size[1], + kernel_size=kernel_size, + stride=stride, + padding=self.padding, + bias=True) + + nn.init.constant_(self.modulator_conv.weight, 0.) + nn.init.constant_(self.modulator_conv.bias, 0.) + + self.regular_conv = nn.Conv2d(in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=self.padding, + bias=bias) + + def forward(self, x): + #h, w = x.shape[2:] + #max_offset = max(h, w)/4. + + offset = self.offset_conv(x)#.clamp(-max_offset, max_offset) + modulator = 2. * torch.sigmoid(self.modulator_conv(x)) + + x = deform_conv2d( + input=x, + offset=offset, + weight=self.regular_conv.weight, + bias=self.regular_conv.bias, + padding=self.padding, + mask=modulator, + stride=self.stride, + ) + return x + + + + +### utils.py + +import torch.nn as nn + + +def build_act_layer(act_layer): + if act_layer == 'ReLU': + return nn.ReLU(inplace=True) + elif act_layer == 'SiLU': + return nn.SiLU(inplace=True) + elif act_layer == 'GELU': + return nn.GELU() + + raise NotImplementedError(f'build_act_layer does not support {act_layer}') + + +def build_norm_layer(dim, + norm_layer, + in_format='channels_last', + out_format='channels_last', + eps=1e-6): + layers = [] + if norm_layer == 'BN': + if in_format == 'channels_last': + layers.append(to_channels_first()) + layers.append(nn.BatchNorm2d(dim)) + if out_format == 'channels_last': + layers.append(to_channels_last()) + elif norm_layer == 'LN': + if in_format == 'channels_first': + layers.append(to_channels_last()) + layers.append(nn.LayerNorm(dim, eps=eps)) + if out_format == 'channels_first': + layers.append(to_channels_first()) + else: + raise NotImplementedError( + f'build_norm_layer does not support {norm_layer}') + return nn.Sequential(*layers) + + +class to_channels_first(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 3, 1, 2) + + +class to_channels_last(nn.Module): + + def __init__(self): + super().__init__() + + def forward(self, x): + return x.permute(0, 2, 3, 1) + + + +### dataset.py + +_class_labels_TR_sorted = ( + 'Airplane, Ant, Antenna, Archery, Axe, BabyCarriage, Bag, BalanceBeam, Balcony, Balloon, Basket, BasketballHoop, Beatle, Bed, Bee, Bench, Bicycle, ' + 'BicycleFrame, BicycleStand, Boat, Bonsai, BoomLift, Bridge, BunkBed, Butterfly, Button, Cable, CableLift, Cage, Camcorder, Cannon, Canoe, Car, ' + 'CarParkDropArm, Carriage, Cart, Caterpillar, CeilingLamp, Centipede, Chair, Clip, Clock, Clothes, CoatHanger, Comb, ConcretePumpTruck, Crack, Crane, ' + 'Cup, DentalChair, Desk, DeskChair, Diagram, DishRack, DoorHandle, Dragonfish, Dragonfly, Drum, Earphone, Easel, ElectricIron, Excavator, Eyeglasses, ' + 'Fan, Fence, Fencing, FerrisWheel, FireExtinguisher, Fishing, Flag, FloorLamp, Forklift, GasStation, Gate, Gear, Goal, Golf, GymEquipment, Hammock, ' + 'Handcart, Handcraft, Handrail, HangGlider, Harp, Harvester, Headset, Helicopter, Helmet, Hook, HorizontalBar, Hydrovalve, IroningTable, Jewelry, Key, ' + 'KidsPlayground, Kitchenware, Kite, Knife, Ladder, LaundryRack, Lightning, Lobster, Locust, Machine, MachineGun, MagazineRack, Mantis, Medal, MemorialArchway, ' + 'Microphone, Missile, MobileHolder, Monitor, Mosquito, Motorcycle, MovingTrolley, Mower, MusicPlayer, MusicStand, ObservationTower, Octopus, OilWell, ' + 'OlympicLogo, OperatingTable, OutdoorFitnessEquipment, Parachute, Pavilion, Piano, Pipe, PlowHarrow, PoleVault, Punchbag, Rack, Racket, Rifle, Ring, Robot, ' + 'RockClimbing, Rope, Sailboat, Satellite, Scaffold, Scale, Scissor, Scooter, Sculpture, Seadragon, Seahorse, Seal, SewingMachine, Ship, Shoe, ShoppingCart, ' + 'ShoppingTrolley, Shower, Shrimp, Signboard, Skateboarding, Skeleton, Skiing, Spade, SpeedBoat, Spider, Spoon, Stair, Stand, Stationary, SteeringWheel, ' + 'Stethoscope, Stool, Stove, StreetLamp, SweetStand, Swing, Sword, TV, Table, TableChair, TableLamp, TableTennis, Tank, Tapeline, Teapot, Telescope, Tent, ' + 'TobaccoPipe, Toy, Tractor, TrafficLight, TrafficSign, Trampoline, TransmissionTower, Tree, Tricycle, TrimmerCover, Tripod, Trombone, Truck, Trumpet, Tuba, ' + 'UAV, Umbrella, UnevenBars, UtilityPole, VacuumCleaner, Violin, Wakesurfing, Watch, WaterTower, WateringPot, Well, WellLid, Wheel, Wheelchair, WindTurbine, Windmill, WineGlass, WireWhisk, Yacht' +) +class_labels_TR_sorted = _class_labels_TR_sorted.split(', ') + + +### models/backbones/build_backbones.py + +import torch +import torch.nn as nn +from collections import OrderedDict +from torchvision.models import vgg16, vgg16_bn, VGG16_Weights, VGG16_BN_Weights, resnet50, ResNet50_Weights +# from models.pvt_v2 import pvt_v2_b0, pvt_v2_b1, pvt_v2_b2, pvt_v2_b5 +# from models.swin_v1 import swin_v1_t, swin_v1_s, swin_v1_b, swin_v1_l +# from config import Config + + +config = Config() + +def build_backbone(bb_name, pretrained=True, params_settings=''): + if bb_name == 'vgg16': + bb_net = list(vgg16(pretrained=VGG16_Weights.DEFAULT if pretrained else None).children())[0] + bb = nn.Sequential(OrderedDict({'conv1': bb_net[:4], 'conv2': bb_net[4:9], 'conv3': bb_net[9:16], 'conv4': bb_net[16:23]})) + elif bb_name == 'vgg16bn': + bb_net = list(vgg16_bn(pretrained=VGG16_BN_Weights.DEFAULT if pretrained else None).children())[0] + bb = nn.Sequential(OrderedDict({'conv1': bb_net[:6], 'conv2': bb_net[6:13], 'conv3': bb_net[13:23], 'conv4': bb_net[23:33]})) + elif bb_name == 'resnet50': + bb_net = list(resnet50(pretrained=ResNet50_Weights.DEFAULT if pretrained else None).children()) + bb = nn.Sequential(OrderedDict({'conv1': nn.Sequential(*bb_net[0:3]), 'conv2': bb_net[4], 'conv3': bb_net[5], 'conv4': bb_net[6]})) + else: + bb = eval('{}({})'.format(bb_name, params_settings)) + if pretrained: + bb = load_weights(bb, bb_name) + return bb + +def load_weights(model, model_name): + save_model = torch.load(config.weights[model_name], map_location='cpu') + model_dict = model.state_dict() + state_dict = {k: v if v.size() == model_dict[k].size() else model_dict[k] for k, v in save_model.items() if k in model_dict.keys()} + # to ignore the weights with mismatched size when I modify the backbone itself. + if not state_dict: + save_model_keys = list(save_model.keys()) + sub_item = save_model_keys[0] if len(save_model_keys) == 1 else None + state_dict = {k: v if v.size() == model_dict[k].size() else model_dict[k] for k, v in save_model[sub_item].items() if k in model_dict.keys()} + if not state_dict or not sub_item: + print('Weights are not successully loaded. Check the state dict of weights file.') + return None + else: + print('Found correct weights in the "{}" item of loaded state_dict.'.format(sub_item)) + model_dict.update(state_dict) + model.load_state_dict(model_dict) + return model + + + +### models/modules/decoder_blocks.py + +import torch +import torch.nn as nn +# from models.aspp import ASPP, ASPPDeformable +# from config import Config + + +# config = Config() + + +class BasicDecBlk(nn.Module): + def __init__(self, in_channels=64, out_channels=64, inter_channels=64): + super(BasicDecBlk, self).__init__() + inter_channels = in_channels // 4 if config.dec_channels_inter == 'adap' else 64 + self.conv_in = nn.Conv2d(in_channels, inter_channels, 3, 1, padding=1) + self.relu_in = nn.ReLU(inplace=True) + if config.dec_att == 'ASPP': + self.dec_att = ASPP(in_channels=inter_channels) + elif config.dec_att == 'ASPPDeformable': + self.dec_att = ASPPDeformable(in_channels=inter_channels) + self.conv_out = nn.Conv2d(inter_channels, out_channels, 3, 1, padding=1) + self.bn_in = nn.BatchNorm2d(inter_channels) if config.batch_size > 1 else nn.Identity() + self.bn_out = nn.BatchNorm2d(out_channels) if config.batch_size > 1 else nn.Identity() + + def forward(self, x): + x = self.conv_in(x) + x = self.bn_in(x) + x = self.relu_in(x) + if hasattr(self, 'dec_att'): + x = self.dec_att(x) + x = self.conv_out(x) + x = self.bn_out(x) + return x + + +class ResBlk(nn.Module): + def __init__(self, in_channels=64, out_channels=None, inter_channels=64): + super(ResBlk, self).__init__() + if out_channels is None: + out_channels = in_channels + inter_channels = in_channels // 4 if config.dec_channels_inter == 'adap' else 64 + + self.conv_in = nn.Conv2d(in_channels, inter_channels, 3, 1, padding=1) + self.bn_in = nn.BatchNorm2d(inter_channels) if config.batch_size > 1 else nn.Identity() + self.relu_in = nn.ReLU(inplace=True) + + if config.dec_att == 'ASPP': + self.dec_att = ASPP(in_channels=inter_channels) + elif config.dec_att == 'ASPPDeformable': + self.dec_att = ASPPDeformable(in_channels=inter_channels) + + self.conv_out = nn.Conv2d(inter_channels, out_channels, 3, 1, padding=1) + self.bn_out = nn.BatchNorm2d(out_channels) if config.batch_size > 1 else nn.Identity() + + self.conv_resi = nn.Conv2d(in_channels, out_channels, 1, 1, 0) + + def forward(self, x): + _x = self.conv_resi(x) + x = self.conv_in(x) + x = self.bn_in(x) + x = self.relu_in(x) + if hasattr(self, 'dec_att'): + x = self.dec_att(x) + x = self.conv_out(x) + x = self.bn_out(x) + return x + _x + + + +### models/modules/lateral_blocks.py + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from functools import partial + +# from config import Config + + +# config = Config() + + +class BasicLatBlk(nn.Module): + def __init__(self, in_channels=64, out_channels=64, inter_channels=64): + super(BasicLatBlk, self).__init__() + inter_channels = in_channels // 4 if config.dec_channels_inter == 'adap' else 64 + self.conv = nn.Conv2d(in_channels, out_channels, 1, 1, 0) + + def forward(self, x): + x = self.conv(x) + return x + + + +### models/modules/aspp.py + +import torch +import torch.nn as nn +import torch.nn.functional as F +# from models.deform_conv import DeformableConv2d +# from config import Config + + +# config = Config() + + +class _ASPPModule(nn.Module): + def __init__(self, in_channels, planes, kernel_size, padding, dilation): + super(_ASPPModule, self).__init__() + self.atrous_conv = nn.Conv2d(in_channels, planes, kernel_size=kernel_size, + stride=1, padding=padding, dilation=dilation, bias=False) + self.bn = nn.BatchNorm2d(planes) if config.batch_size > 1 else nn.Identity() + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.atrous_conv(x) + x = self.bn(x) + + return self.relu(x) + + +class ASPP(nn.Module): + def __init__(self, in_channels=64, out_channels=None, output_stride=16): + super(ASPP, self).__init__() + self.down_scale = 1 + if out_channels is None: + out_channels = in_channels + self.in_channelster = 256 // self.down_scale + if output_stride == 16: + dilations = [1, 6, 12, 18] + elif output_stride == 8: + dilations = [1, 12, 24, 36] + else: + raise NotImplementedError + + self.aspp1 = _ASPPModule(in_channels, self.in_channelster, 1, padding=0, dilation=dilations[0]) + self.aspp2 = _ASPPModule(in_channels, self.in_channelster, 3, padding=dilations[1], dilation=dilations[1]) + self.aspp3 = _ASPPModule(in_channels, self.in_channelster, 3, padding=dilations[2], dilation=dilations[2]) + self.aspp4 = _ASPPModule(in_channels, self.in_channelster, 3, padding=dilations[3], dilation=dilations[3]) + + self.global_avg_pool = nn.Sequential(nn.AdaptiveAvgPool2d((1, 1)), + nn.Conv2d(in_channels, self.in_channelster, 1, stride=1, bias=False), + nn.BatchNorm2d(self.in_channelster) if config.batch_size > 1 else nn.Identity(), + nn.ReLU(inplace=True)) + self.conv1 = nn.Conv2d(self.in_channelster * 5, out_channels, 1, bias=False) + self.bn1 = nn.BatchNorm2d(out_channels) if config.batch_size > 1 else nn.Identity() + self.relu = nn.ReLU(inplace=True) + self.dropout = nn.Dropout(0.5) + + def forward(self, x): + x1 = self.aspp1(x) + x2 = self.aspp2(x) + x3 = self.aspp3(x) + x4 = self.aspp4(x) + x5 = self.global_avg_pool(x) + x5 = F.interpolate(x5, size=x1.size()[2:], mode='bilinear', align_corners=True) + x = torch.cat((x1, x2, x3, x4, x5), dim=1) + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + return self.dropout(x) + + +##################### Deformable +class _ASPPModuleDeformable(nn.Module): + def __init__(self, in_channels, planes, kernel_size, padding): + super(_ASPPModuleDeformable, self).__init__() + self.atrous_conv = DeformableConv2d(in_channels, planes, kernel_size=kernel_size, + stride=1, padding=padding, bias=False) + self.bn = nn.BatchNorm2d(planes) if config.batch_size > 1 else nn.Identity() + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + x = self.atrous_conv(x) + x = self.bn(x) + + return self.relu(x) + + +class ASPPDeformable(nn.Module): + def __init__(self, in_channels, out_channels=None, parallel_block_sizes=[1, 3, 7]): + super(ASPPDeformable, self).__init__() + self.down_scale = 1 + if out_channels is None: + out_channels = in_channels + self.in_channelster = 256 // self.down_scale + + self.aspp1 = _ASPPModuleDeformable(in_channels, self.in_channelster, 1, padding=0) + self.aspp_deforms = nn.ModuleList([ + _ASPPModuleDeformable(in_channels, self.in_channelster, conv_size, padding=int(conv_size//2)) for conv_size in parallel_block_sizes + ]) + + self.global_avg_pool = nn.Sequential(nn.AdaptiveAvgPool2d((1, 1)), + nn.Conv2d(in_channels, self.in_channelster, 1, stride=1, bias=False), + nn.BatchNorm2d(self.in_channelster) if config.batch_size > 1 else nn.Identity(), + nn.ReLU(inplace=True)) + self.conv1 = nn.Conv2d(self.in_channelster * (2 + len(self.aspp_deforms)), out_channels, 1, bias=False) + self.bn1 = nn.BatchNorm2d(out_channels) if config.batch_size > 1 else nn.Identity() + self.relu = nn.ReLU(inplace=True) + self.dropout = nn.Dropout(0.5) + + def forward(self, x): + x1 = self.aspp1(x) + x_aspp_deforms = [aspp_deform(x) for aspp_deform in self.aspp_deforms] + x5 = self.global_avg_pool(x) + x5 = F.interpolate(x5, size=x1.size()[2:], mode='bilinear', align_corners=True) + x = torch.cat((x1, *x_aspp_deforms, x5), dim=1) + + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + return self.dropout(x) + + + +### models/refinement/refiner.py + +import torch +import torch.nn as nn +from collections import OrderedDict +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision.models import vgg16, vgg16_bn +from torchvision.models import resnet50 + +# from config import Config +# from dataset import class_labels_TR_sorted +# from models.build_backbone import build_backbone +# from models.decoder_blocks import BasicDecBlk +# from models.lateral_blocks import BasicLatBlk +# from models.ing import * +# from models.stem_layer import StemLayer + + +class RefinerPVTInChannels4(nn.Module): + def __init__(self, in_channels=3+1): + super(RefinerPVTInChannels4, self).__init__() + self.config = Config() + self.epoch = 1 + self.bb = build_backbone(self.config.bb, params_settings='in_channels=4') + + lateral_channels_in_collection = { + 'vgg16': [512, 256, 128, 64], 'vgg16bn': [512, 256, 128, 64], 'resnet50': [1024, 512, 256, 64], + 'pvt_v2_b2': [512, 320, 128, 64], 'pvt_v2_b5': [512, 320, 128, 64], + 'swin_v1_b': [1024, 512, 256, 128], 'swin_v1_l': [1536, 768, 384, 192], + } + channels = lateral_channels_in_collection[self.config.bb] + self.squeeze_module = BasicDecBlk(channels[0], channels[0]) + + self.decoder = Decoder(channels) + + if 0: + for key, value in self.named_parameters(): + if 'bb.' in key: + value.requires_grad = False + + def forward(self, x): + if isinstance(x, list): + x = torch.cat(x, dim=1) + ########## Encoder ########## + if self.config.bb in ['vgg16', 'vgg16bn', 'resnet50']: + x1 = self.bb.conv1(x) + x2 = self.bb.conv2(x1) + x3 = self.bb.conv3(x2) + x4 = self.bb.conv4(x3) + else: + x1, x2, x3, x4 = self.bb(x) + + x4 = self.squeeze_module(x4) + + ########## Decoder ########## + + features = [x, x1, x2, x3, x4] + scaled_preds = self.decoder(features) + + return scaled_preds + + +class Refiner(nn.Module): + def __init__(self, in_channels=3+1): + super(Refiner, self).__init__() + self.config = Config() + self.epoch = 1 + self.stem_layer = StemLayer(in_channels=in_channels, inter_channels=48, out_channels=3, norm_layer='BN' if self.config.batch_size > 1 else 'LN') + self.bb = build_backbone(self.config.bb) + + lateral_channels_in_collection = { + 'vgg16': [512, 256, 128, 64], 'vgg16bn': [512, 256, 128, 64], 'resnet50': [1024, 512, 256, 64], + 'pvt_v2_b2': [512, 320, 128, 64], 'pvt_v2_b5': [512, 320, 128, 64], + 'swin_v1_b': [1024, 512, 256, 128], 'swin_v1_l': [1536, 768, 384, 192], + } + channels = lateral_channels_in_collection[self.config.bb] + self.squeeze_module = BasicDecBlk(channels[0], channels[0]) + + self.decoder = Decoder(channels) + + if 0: + for key, value in self.named_parameters(): + if 'bb.' in key: + value.requires_grad = False + + def forward(self, x): + if isinstance(x, list): + x = torch.cat(x, dim=1) + x = self.stem_layer(x) + ########## Encoder ########## + if self.config.bb in ['vgg16', 'vgg16bn', 'resnet50']: + x1 = self.bb.conv1(x) + x2 = self.bb.conv2(x1) + x3 = self.bb.conv3(x2) + x4 = self.bb.conv4(x3) + else: + x1, x2, x3, x4 = self.bb(x) + + x4 = self.squeeze_module(x4) + + ########## Decoder ########## + + features = [x, x1, x2, x3, x4] + scaled_preds = self.decoder(features) + + return scaled_preds + + +class Decoder(nn.Module): + def __init__(self, channels): + super(Decoder, self).__init__() + self.config = Config() + DecoderBlock = eval('BasicDecBlk') + LateralBlock = eval('BasicLatBlk') + + self.decoder_block4 = DecoderBlock(channels[0], channels[1]) + self.decoder_block3 = DecoderBlock(channels[1], channels[2]) + self.decoder_block2 = DecoderBlock(channels[2], channels[3]) + self.decoder_block1 = DecoderBlock(channels[3], channels[3]//2) + + self.lateral_block4 = LateralBlock(channels[1], channels[1]) + self.lateral_block3 = LateralBlock(channels[2], channels[2]) + self.lateral_block2 = LateralBlock(channels[3], channels[3]) + + if self.config.ms_supervision: + self.conv_ms_spvn_4 = nn.Conv2d(channels[1], 1, 1, 1, 0) + self.conv_ms_spvn_3 = nn.Conv2d(channels[2], 1, 1, 1, 0) + self.conv_ms_spvn_2 = nn.Conv2d(channels[3], 1, 1, 1, 0) + self.conv_out1 = nn.Sequential(nn.Conv2d(channels[3]//2, 1, 1, 1, 0)) + + def forward(self, features): + x, x1, x2, x3, x4 = features + outs = [] + p4 = self.decoder_block4(x4) + _p4 = F.interpolate(p4, size=x3.shape[2:], mode='bilinear', align_corners=True) + _p3 = _p4 + self.lateral_block4(x3) + + p3 = self.decoder_block3(_p3) + _p3 = F.interpolate(p3, size=x2.shape[2:], mode='bilinear', align_corners=True) + _p2 = _p3 + self.lateral_block3(x2) + + p2 = self.decoder_block2(_p2) + _p2 = F.interpolate(p2, size=x1.shape[2:], mode='bilinear', align_corners=True) + _p1 = _p2 + self.lateral_block2(x1) + + _p1 = self.decoder_block1(_p1) + _p1 = F.interpolate(_p1, size=x.shape[2:], mode='bilinear', align_corners=True) + p1_out = self.conv_out1(_p1) + + if self.config.ms_supervision: + outs.append(self.conv_ms_spvn_4(p4)) + outs.append(self.conv_ms_spvn_3(p3)) + outs.append(self.conv_ms_spvn_2(p2)) + outs.append(p1_out) + return outs + + +class RefUNet(nn.Module): + # Refinement + def __init__(self, in_channels=3+1): + super(RefUNet, self).__init__() + self.encoder_1 = nn.Sequential( + nn.Conv2d(in_channels, 64, 3, 1, 1), + nn.Conv2d(64, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + self.encoder_2 = nn.Sequential( + nn.MaxPool2d(2, 2, ceil_mode=True), + nn.Conv2d(64, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + self.encoder_3 = nn.Sequential( + nn.MaxPool2d(2, 2, ceil_mode=True), + nn.Conv2d(64, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + self.encoder_4 = nn.Sequential( + nn.MaxPool2d(2, 2, ceil_mode=True), + nn.Conv2d(64, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + self.pool4 = nn.MaxPool2d(2, 2, ceil_mode=True) + ##### + self.decoder_5 = nn.Sequential( + nn.Conv2d(64, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + ##### + self.decoder_4 = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + self.decoder_3 = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + self.decoder_2 = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + self.decoder_1 = nn.Sequential( + nn.Conv2d(128, 64, 3, 1, 1), + nn.BatchNorm2d(64), + nn.ReLU(inplace=True) + ) + + self.conv_d0 = nn.Conv2d(64, 1, 3, 1, 1) + + self.upscore2 = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True) + + def forward(self, x): + outs = [] + if isinstance(x, list): + x = torch.cat(x, dim=1) + hx = x + + hx1 = self.encoder_1(hx) + hx2 = self.encoder_2(hx1) + hx3 = self.encoder_3(hx2) + hx4 = self.encoder_4(hx3) + + hx = self.decoder_5(self.pool4(hx4)) + hx = torch.cat((self.upscore2(hx), hx4), 1) + + d4 = self.decoder_4(hx) + hx = torch.cat((self.upscore2(d4), hx3), 1) + + d3 = self.decoder_3(hx) + hx = torch.cat((self.upscore2(d3), hx2), 1) + + d2 = self.decoder_2(hx) + hx = torch.cat((self.upscore2(d2), hx1), 1) + + d1 = self.decoder_1(hx) + + x = self.conv_d0(d1) + outs.append(x) + return outs + + + +### models/stem_layer.py + +import torch.nn as nn +# from utils import build_act_layer, build_norm_layer + + +class StemLayer(nn.Module): + r""" Stem layer of InternImage + Args: + in_channels (int): number of input channels + out_channels (int): number of output channels + act_layer (str): activation layer + norm_layer (str): normalization layer + """ + + def __init__(self, + in_channels=3+1, + inter_channels=48, + out_channels=96, + act_layer='GELU', + norm_layer='BN'): + super().__init__() + self.conv1 = nn.Conv2d(in_channels, + inter_channels, + kernel_size=3, + stride=1, + padding=1) + self.norm1 = build_norm_layer( + inter_channels, norm_layer, 'channels_first', 'channels_first' + ) + self.act = build_act_layer(act_layer) + self.conv2 = nn.Conv2d(inter_channels, + out_channels, + kernel_size=3, + stride=1, + padding=1) + self.norm2 = build_norm_layer( + out_channels, norm_layer, 'channels_first', 'channels_first' + ) + + def forward(self, x): + x = self.conv1(x) + x = self.norm1(x) + x = self.act(x) + x = self.conv2(x) + x = self.norm2(x) + return x + + +### models/birefnet.py + +import torch +import torch.nn as nn +import torch.nn.functional as F +from kornia.filters import laplacian +from transformers import PreTrainedModel + +# from config import Config +# from dataset import class_labels_TR_sorted +# from models.build_backbone import build_backbone +# from models.decoder_blocks import BasicDecBlk, ResBlk, HierarAttDecBlk +# from models.lateral_blocks import BasicLatBlk +# from models.aspp import ASPP, ASPPDeformable +# from models.ing import * +# from models.refiner import Refiner, RefinerPVTInChannels4, RefUNet +# from models.stem_layer import StemLayer +from .BiRefNet_config import BiRefNetConfig + + +class BiRefNet( + PreTrainedModel +): + config_class = BiRefNetConfig + def __init__(self, bb_pretrained=True, config=BiRefNetConfig()): + super(BiRefNet, self).__init__(config) + bb_pretrained = config.bb_pretrained + self.config = Config() + self.epoch = 1 + self.bb = build_backbone(self.config.bb, pretrained=bb_pretrained) + + channels = self.config.lateral_channels_in_collection + + if self.config.auxiliary_classification: + self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) + self.cls_head = nn.Sequential( + nn.Linear(channels[0], len(class_labels_TR_sorted)) + ) + + if self.config.squeeze_block: + self.squeeze_module = nn.Sequential(*[ + eval(self.config.squeeze_block.split('_x')[0])(channels[0]+sum(self.config.cxt), channels[0]) + for _ in range(eval(self.config.squeeze_block.split('_x')[1])) + ]) + + self.decoder = Decoder(channels) + + if self.config.ender: + self.dec_end = nn.Sequential( + nn.Conv2d(1, 16, 3, 1, 1), + nn.Conv2d(16, 1, 3, 1, 1), + nn.ReLU(inplace=True), + ) + + # refine patch-level segmentation + if self.config.refine: + if self.config.refine == 'itself': + self.stem_layer = StemLayer(in_channels=3+1, inter_channels=48, out_channels=3, norm_layer='BN' if self.config.batch_size > 1 else 'LN') + else: + self.refiner = eval('{}({})'.format(self.config.refine, 'in_channels=3+1')) + + if self.config.freeze_bb: + # Freeze the backbone... + print(self.named_parameters()) + for key, value in self.named_parameters(): + if 'bb.' in key and 'refiner.' not in key: + value.requires_grad = False + + def forward_enc(self, x): + if self.config.bb in ['vgg16', 'vgg16bn', 'resnet50']: + x1 = self.bb.conv1(x); x2 = self.bb.conv2(x1); x3 = self.bb.conv3(x2); x4 = self.bb.conv4(x3) + else: + x1, x2, x3, x4 = self.bb(x) + if self.config.mul_scl_ipt == 'cat': + B, C, H, W = x.shape + x1_, x2_, x3_, x4_ = self.bb(F.interpolate(x, size=(H//2, W//2), mode='bilinear', align_corners=True)) + x1 = torch.cat([x1, F.interpolate(x1_, size=x1.shape[2:], mode='bilinear', align_corners=True)], dim=1) + x2 = torch.cat([x2, F.interpolate(x2_, size=x2.shape[2:], mode='bilinear', align_corners=True)], dim=1) + x3 = torch.cat([x3, F.interpolate(x3_, size=x3.shape[2:], mode='bilinear', align_corners=True)], dim=1) + x4 = torch.cat([x4, F.interpolate(x4_, size=x4.shape[2:], mode='bilinear', align_corners=True)], dim=1) + elif self.config.mul_scl_ipt == 'add': + B, C, H, W = x.shape + x1_, x2_, x3_, x4_ = self.bb(F.interpolate(x, size=(H//2, W//2), mode='bilinear', align_corners=True)) + x1 = x1 + F.interpolate(x1_, size=x1.shape[2:], mode='bilinear', align_corners=True) + x2 = x2 + F.interpolate(x2_, size=x2.shape[2:], mode='bilinear', align_corners=True) + x3 = x3 + F.interpolate(x3_, size=x3.shape[2:], mode='bilinear', align_corners=True) + x4 = x4 + F.interpolate(x4_, size=x4.shape[2:], mode='bilinear', align_corners=True) + class_preds = self.cls_head(self.avgpool(x4).view(x4.shape[0], -1)) if self.training and self.config.auxiliary_classification else None + if self.config.cxt: + x4 = torch.cat( + ( + *[ + F.interpolate(x1, size=x4.shape[2:], mode='bilinear', align_corners=True), + F.interpolate(x2, size=x4.shape[2:], mode='bilinear', align_corners=True), + F.interpolate(x3, size=x4.shape[2:], mode='bilinear', align_corners=True), + ][-len(self.config.cxt):], + x4 + ), + dim=1 + ) + return (x1, x2, x3, x4), class_preds + + def forward_ori(self, x): + ########## Encoder ########## + (x1, x2, x3, x4), class_preds = self.forward_enc(x) + if self.config.squeeze_block: + x4 = self.squeeze_module(x4) + ########## Decoder ########## + features = [x, x1, x2, x3, x4] + if self.training and self.config.out_ref: + features.append(laplacian(torch.mean(x, dim=1).unsqueeze(1), kernel_size=5)) + scaled_preds = self.decoder(features) + return scaled_preds, class_preds + + def forward(self, x): + scaled_preds, class_preds = self.forward_ori(x) + class_preds_lst = [class_preds] + return [scaled_preds, class_preds_lst] if self.training else scaled_preds + + +class Decoder(nn.Module): + def __init__(self, channels): + super(Decoder, self).__init__() + self.config = Config() + DecoderBlock = eval(self.config.dec_blk) + LateralBlock = eval(self.config.lat_blk) + + if self.config.dec_ipt: + self.split = self.config.dec_ipt_split + N_dec_ipt = 64 + DBlock = SimpleConvs + ic = 64 + ipt_cha_opt = 1 + self.ipt_blk5 = DBlock(2**10*3 if self.split else 3, [N_dec_ipt, channels[0]//8][ipt_cha_opt], inter_channels=ic) + self.ipt_blk4 = DBlock(2**8*3 if self.split else 3, [N_dec_ipt, channels[0]//8][ipt_cha_opt], inter_channels=ic) + self.ipt_blk3 = DBlock(2**6*3 if self.split else 3, [N_dec_ipt, channels[1]//8][ipt_cha_opt], inter_channels=ic) + self.ipt_blk2 = DBlock(2**4*3 if self.split else 3, [N_dec_ipt, channels[2]//8][ipt_cha_opt], inter_channels=ic) + self.ipt_blk1 = DBlock(2**0*3 if self.split else 3, [N_dec_ipt, channels[3]//8][ipt_cha_opt], inter_channels=ic) + else: + self.split = None + + self.decoder_block4 = DecoderBlock(channels[0]+([N_dec_ipt, channels[0]//8][ipt_cha_opt] if self.config.dec_ipt else 0), channels[1]) + self.decoder_block3 = DecoderBlock(channels[1]+([N_dec_ipt, channels[0]//8][ipt_cha_opt] if self.config.dec_ipt else 0), channels[2]) + self.decoder_block2 = DecoderBlock(channels[2]+([N_dec_ipt, channels[1]//8][ipt_cha_opt] if self.config.dec_ipt else 0), channels[3]) + self.decoder_block1 = DecoderBlock(channels[3]+([N_dec_ipt, channels[2]//8][ipt_cha_opt] if self.config.dec_ipt else 0), channels[3]//2) + self.conv_out1 = nn.Sequential(nn.Conv2d(channels[3]//2+([N_dec_ipt, channels[3]//8][ipt_cha_opt] if self.config.dec_ipt else 0), 1, 1, 1, 0)) + + self.lateral_block4 = LateralBlock(channels[1], channels[1]) + self.lateral_block3 = LateralBlock(channels[2], channels[2]) + self.lateral_block2 = LateralBlock(channels[3], channels[3]) + + if self.config.ms_supervision: + self.conv_ms_spvn_4 = nn.Conv2d(channels[1], 1, 1, 1, 0) + self.conv_ms_spvn_3 = nn.Conv2d(channels[2], 1, 1, 1, 0) + self.conv_ms_spvn_2 = nn.Conv2d(channels[3], 1, 1, 1, 0) + + if self.config.out_ref: + _N = 16 + self.gdt_convs_4 = nn.Sequential(nn.Conv2d(channels[1], _N, 3, 1, 1), nn.BatchNorm2d(_N) if self.config.batch_size > 1 else nn.Identity(), nn.ReLU(inplace=True)) + self.gdt_convs_3 = nn.Sequential(nn.Conv2d(channels[2], _N, 3, 1, 1), nn.BatchNorm2d(_N) if self.config.batch_size > 1 else nn.Identity(), nn.ReLU(inplace=True)) + self.gdt_convs_2 = nn.Sequential(nn.Conv2d(channels[3], _N, 3, 1, 1), nn.BatchNorm2d(_N) if self.config.batch_size > 1 else nn.Identity(), nn.ReLU(inplace=True)) + + self.gdt_convs_pred_4 = nn.Sequential(nn.Conv2d(_N, 1, 1, 1, 0)) + self.gdt_convs_pred_3 = nn.Sequential(nn.Conv2d(_N, 1, 1, 1, 0)) + self.gdt_convs_pred_2 = nn.Sequential(nn.Conv2d(_N, 1, 1, 1, 0)) + + self.gdt_convs_attn_4 = nn.Sequential(nn.Conv2d(_N, 1, 1, 1, 0)) + self.gdt_convs_attn_3 = nn.Sequential(nn.Conv2d(_N, 1, 1, 1, 0)) + self.gdt_convs_attn_2 = nn.Sequential(nn.Conv2d(_N, 1, 1, 1, 0)) + + def get_patches_batch(self, x, p): + _size_h, _size_w = p.shape[2:] + patches_batch = [] + for idx in range(x.shape[0]): + columns_x = torch.split(x[idx], split_size_or_sections=_size_w, dim=-1) + patches_x = [] + for column_x in columns_x: + patches_x += [p.unsqueeze(0) for p in torch.split(column_x, split_size_or_sections=_size_h, dim=-2)] + patch_sample = torch.cat(patches_x, dim=1) + patches_batch.append(patch_sample) + return torch.cat(patches_batch, dim=0) + + def forward(self, features): + if self.training and self.config.out_ref: + outs_gdt_pred = [] + outs_gdt_label = [] + x, x1, x2, x3, x4, gdt_gt = features + else: + x, x1, x2, x3, x4 = features + outs = [] + + if self.config.dec_ipt: + patches_batch = self.get_patches_batch(x, x4) if self.split else x + x4 = torch.cat((x4, self.ipt_blk5(F.interpolate(patches_batch, size=x4.shape[2:], mode='bilinear', align_corners=True))), 1) + p4 = self.decoder_block4(x4) + m4 = self.conv_ms_spvn_4(p4) if self.config.ms_supervision else None + if self.config.out_ref: + p4_gdt = self.gdt_convs_4(p4) + if self.training: + # >> GT: + m4_dia = m4 + gdt_label_main_4 = gdt_gt * F.interpolate(m4_dia, size=gdt_gt.shape[2:], mode='bilinear', align_corners=True) + outs_gdt_label.append(gdt_label_main_4) + # >> Pred: + gdt_pred_4 = self.gdt_convs_pred_4(p4_gdt) + outs_gdt_pred.append(gdt_pred_4) + gdt_attn_4 = self.gdt_convs_attn_4(p4_gdt).sigmoid() + # >> Finally: + p4 = p4 * gdt_attn_4 + _p4 = F.interpolate(p4, size=x3.shape[2:], mode='bilinear', align_corners=True) + _p3 = _p4 + self.lateral_block4(x3) + + if self.config.dec_ipt: + patches_batch = self.get_patches_batch(x, _p3) if self.split else x + _p3 = torch.cat((_p3, self.ipt_blk4(F.interpolate(patches_batch, size=x3.shape[2:], mode='bilinear', align_corners=True))), 1) + p3 = self.decoder_block3(_p3) + m3 = self.conv_ms_spvn_3(p3) if self.config.ms_supervision else None + if self.config.out_ref: + p3_gdt = self.gdt_convs_3(p3) + if self.training: + # >> GT: + # m3 --dilation--> m3_dia + # G_3^gt * m3_dia --> G_3^m, which is the label of gradient + m3_dia = m3 + gdt_label_main_3 = gdt_gt * F.interpolate(m3_dia, size=gdt_gt.shape[2:], mode='bilinear', align_corners=True) + outs_gdt_label.append(gdt_label_main_3) + # >> Pred: + # p3 --conv--BN--> F_3^G, where F_3^G predicts the \hat{G_3} with xx + # F_3^G --sigmoid--> A_3^G + gdt_pred_3 = self.gdt_convs_pred_3(p3_gdt) + outs_gdt_pred.append(gdt_pred_3) + gdt_attn_3 = self.gdt_convs_attn_3(p3_gdt).sigmoid() + # >> Finally: + # p3 = p3 * A_3^G + p3 = p3 * gdt_attn_3 + _p3 = F.interpolate(p3, size=x2.shape[2:], mode='bilinear', align_corners=True) + _p2 = _p3 + self.lateral_block3(x2) + + if self.config.dec_ipt: + patches_batch = self.get_patches_batch(x, _p2) if self.split else x + _p2 = torch.cat((_p2, self.ipt_blk3(F.interpolate(patches_batch, size=x2.shape[2:], mode='bilinear', align_corners=True))), 1) + p2 = self.decoder_block2(_p2) + m2 = self.conv_ms_spvn_2(p2) if self.config.ms_supervision else None + if self.config.out_ref: + p2_gdt = self.gdt_convs_2(p2) + if self.training: + # >> GT: + m2_dia = m2 + gdt_label_main_2 = gdt_gt * F.interpolate(m2_dia, size=gdt_gt.shape[2:], mode='bilinear', align_corners=True) + outs_gdt_label.append(gdt_label_main_2) + # >> Pred: + gdt_pred_2 = self.gdt_convs_pred_2(p2_gdt) + outs_gdt_pred.append(gdt_pred_2) + gdt_attn_2 = self.gdt_convs_attn_2(p2_gdt).sigmoid() + # >> Finally: + p2 = p2 * gdt_attn_2 + _p2 = F.interpolate(p2, size=x1.shape[2:], mode='bilinear', align_corners=True) + _p1 = _p2 + self.lateral_block2(x1) + + if self.config.dec_ipt: + patches_batch = self.get_patches_batch(x, _p1) if self.split else x + _p1 = torch.cat((_p1, self.ipt_blk2(F.interpolate(patches_batch, size=x1.shape[2:], mode='bilinear', align_corners=True))), 1) + _p1 = self.decoder_block1(_p1) + _p1 = F.interpolate(_p1, size=x.shape[2:], mode='bilinear', align_corners=True) + + if self.config.dec_ipt: + patches_batch = self.get_patches_batch(x, _p1) if self.split else x + _p1 = torch.cat((_p1, self.ipt_blk1(F.interpolate(patches_batch, size=x.shape[2:], mode='bilinear', align_corners=True))), 1) + p1_out = self.conv_out1(_p1) + + if self.config.ms_supervision: + outs.append(m4) + outs.append(m3) + outs.append(m2) + outs.append(p1_out) + return outs if not (self.config.out_ref and self.training) else ([outs_gdt_pred, outs_gdt_label], outs) + + +class SimpleConvs(nn.Module): + def __init__( + self, in_channels: int, out_channels: int, inter_channels=64 + ) -> None: + super().__init__() + self.conv1 = nn.Conv2d(in_channels, inter_channels, 3, 1, 1) + self.conv_out = nn.Conv2d(inter_channels, out_channels, 3, 1, 1) + + def forward(self, x): + return self.conv_out(self.conv1(x)) diff --git a/BiRefNet/RMBG-2.0/config.json b/BiRefNet/RMBG-2.0/config.json new file mode 100644 index 0000000000000000000000000000000000000000..85f92b5194dcf44bc1d59096b26389c6a386b22b --- /dev/null +++ b/BiRefNet/RMBG-2.0/config.json @@ -0,0 +1,21 @@ +{ + "_name_or_path": "ZhengPeng7/BiRefNet", + "architectures": [ + "BiRefNet" + ], + "auto_map": { + "AutoConfig": "BiRefNet_config.BiRefNetConfig", + "AutoModelForImageSegmentation": "birefnet.BiRefNet" + }, + "custom_pipelines": { + "image-segmentation": { + "pt": [ + "AutoModelForImageSegmentation" + ], + "tf": [], + "type": "image" + } + }, + "bb_pretrained": false, + "model_type": "birefnet" +} \ No newline at end of file diff --git a/BiRefNet/RMBG-2.0/diagram1.png b/BiRefNet/RMBG-2.0/diagram1.png new file mode 100644 index 0000000000000000000000000000000000000000..0a120d09a7fcbf76cf4812bbf86adc9ff94dd438 Binary files /dev/null and b/BiRefNet/RMBG-2.0/diagram1.png differ diff --git a/BiRefNet/RMBG-2.0/preprocessor_config.json b/BiRefNet/RMBG-2.0/preprocessor_config.json new file mode 100644 index 0000000000000000000000000000000000000000..825398cfd94a348babce456f2bcc8422c9bebb93 --- /dev/null +++ b/BiRefNet/RMBG-2.0/preprocessor_config.json @@ -0,0 +1,23 @@ +{ + "do_normalize": true, + "do_rescale": true, + "do_resize": true, + "feature_extractor_type": "ViTFeatureExtractor", + "image_mean": [ + 0.485, + 0.456, + 0.406 + ], + "image_processor_type": "ViTFeatureExtractor", + "image_std": [ + 0.229, + 0.224, + 0.225 + ], + "resample": 2, + "rescale_factor": 0.00392156862745098, + "size": { + "height": 1024, + "width": 1024 + } +} \ No newline at end of file diff --git a/Joy_caption/README.md b/Joy_caption/README.md new file mode 100644 index 0000000000000000000000000000000000000000..798324ab0fcb3b3908c36039effb2e418402f3a1 --- /dev/null +++ b/Joy_caption/README.md @@ -0,0 +1,79 @@ +--- +license: mit +language: +- en +--- +# Image Captioning App + +This is a mod of [Wi-zz/joy-caption-pre-alpha](https://huggingface.co/Wi-zz/joy-caption-pre-alpha) and [fancyfeast/joy-caption-alpha-two](https://huggingface.co/spaces/fancyfeast/joy-caption-alpha-two). Thanks to [dominic1021](https://huggingface.co/dominic1021), [IceHibiki](https://huggingface.co/IceHibiki), [BullseyeMxP](https://huggingface.co/BullseyeMxP), [Wakeme](https://huggingface.co/Wakeme). + +# Notice: I will contribute to Wi-zz after shaping the code. + +## Overview + +This application generates descriptive captions for images using advanced ML models. It processes single images or entire directories, leveraging CLIP and LLM models for accurate and contextual captions. It has NSFW captioning support with natural language. This is just an extension of the original author's efforts to improve performance. Their repo is located here: https://huggingface.co/spaces/fancyfeast/joy-caption-alpha-two. + +## Features + +- Single image and batch processing +- Multiple directory support +- Custom output directory +- Adjustable batch size +- Progress tracking + +## Usage + +| Command | Description | +|---------|-------------| +| `python app.py image.jpg` | Process a single image | +| `python app.py /path/to/directory` | Process all images in a directory | +| `python app.py /path/to/dir1 /path/to/dir2` | Process multiple directories | +| `python app.py /path/to/dir --output /path/to/output` | Specify output directory | +| `python app.py /path/to/dir --bs 8` | Set batch size (default: 4) | + +## Technical Details + +- **Models**: CLIP (vision), LLM (language), custom ImageAdapter +- **Optimization**: CUDA-enabled GPU support +- **Error Handling**: Skips problematic images in batch processing + +## Requirements + +- Python 3.x +- PyTorch +- Transformers library +- PEFT library +- CUDA-capable GPU (recommended) + +## Installation + +Windows + +```bash +git clone https://huggingface.co/John6666/joy-caption-alpha-two-cli-mod +cd joy-caption-alpha-two-cli-mod +python -m venv venv +.\venv\Scripts\activate +# Change as per https://pytorch.org/get-started/locally/ +pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu121 +pip install -r requirements.txt +``` + +Linux + +```bash +git clone https://huggingface.co/John6666/joy-caption-alpha-two-cli-mod +cd joy-caption-alpha-two-cli-mod +python3 -m venv venv +source venv/bin/activate +pip3 install torch torchvision torchaudio +pip3 install -r requirements.txt +``` + +## Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. + +## License + +This project is licensed under the [MIT License](LICENSE). \ No newline at end of file diff --git a/Joy_caption/app.py b/Joy_caption/app.py new file mode 100644 index 0000000000000000000000000000000000000000..ce979572cce6492414ae82431df4ef9b1194c570 --- /dev/null +++ b/Joy_caption/app.py @@ -0,0 +1,536 @@ +import torch +import torch.amp.autocast_mode +import os +import sys +import logging +import warnings +import argparse +from PIL import Image +from pathlib import Path +from tqdm import tqdm +from torch import nn +from transformers import AutoModel, AutoProcessor, AutoTokenizer, PreTrainedTokenizer, PreTrainedTokenizerFast, AutoModelForCausalLM +from typing import List, Union +import torchvision.transforms.functional as TVF +from peft import PeftModel +import gc +import sys +IS_COLAB = 'google.colab' in sys.modules + +# Constants +IMAGE_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.bmp', '.webp') +HF_TOKEN = os.environ.get("HF_TOKEN", None) +BASE_DIR = Path(__file__).resolve().parent # Define the base directory +CHECKPOINT_PATH = BASE_DIR / Path("cgrkzexw-599808") +CLIP_PATH = "google/siglip-so400m-patch14-384" +DEFAULT_MODEL_PATH = "unsloth/Meta-Llama-3.1-8B-bnb-4bit" +#DEFAULT_MODEL_PATH = "unsloth/Meta-Llama-3.1-8B-Instruct-bnb-4bit" # Default in Alpha One Two. +#DEFAULT_MODEL_PATH = "Orenguteng/Llama-3.1-8B-Lexi-Uncensored-V2" # Works better but full weight. +LORA_PATH = CHECKPOINT_PATH / "text_model" +CAPTION_TYPE_MAP = { + "Descriptive": [ + "Write a descriptive caption for this image in a formal tone.", + "Write a descriptive caption for this image in a formal tone within {word_count} words.", + "Write a {length} descriptive caption for this image in a formal tone.", + ], + "Descriptive (Informal)": [ + "Write a descriptive caption for this image in a casual tone.", + "Write a descriptive caption for this image in a casual tone within {word_count} words.", + "Write a {length} descriptive caption for this image in a casual tone.", + ], + "Training Prompt": [ + "Write a stable diffusion prompt for this image.", + "Write a stable diffusion prompt for this image within {word_count} words.", + "Write a {length} stable diffusion prompt for this image.", + ], + "MidJourney": [ + "Write a MidJourney prompt for this image.", + "Write a MidJourney prompt for this image within {word_count} words.", + "Write a {length} MidJourney prompt for this image.", + ], + "Booru tag list": [ + "Write a list of Booru tags for this image.", + "Write a list of Booru tags for this image within {word_count} words.", + "Write a {length} list of Booru tags for this image.", + ], + "Booru-like tag list": [ + "Write a list of Booru-like tags for this image.", + "Write a list of Booru-like tags for this image within {word_count} words.", + "Write a {length} list of Booru-like tags for this image.", + ], + "Art Critic": [ + "Analyze this image like an art critic would with information about its composition, style, symbolism, the use of color, light, any artistic movement it might belong to, etc.", + "Analyze this image like an art critic would with information about its composition, style, symbolism, the use of color, light, any artistic movement it might belong to, etc. Keep it within {word_count} words.", + "Analyze this image like an art critic would with information about its composition, style, symbolism, the use of color, light, any artistic movement it might belong to, etc. Keep it {length}.", + ], + "Product Listing": [ + "Write a caption for this image as though it were a product listing.", + "Write a caption for this image as though it were a product listing. Keep it under {word_count} words.", + "Write a {length} caption for this image as though it were a product listing.", + ], + "Social Media Post": [ + "Write a caption for this image as if it were being used for a social media post.", + "Write a caption for this image as if it were being used for a social media post. Limit the caption to {word_count} words.", + "Write a {length} caption for this image as if it were being used for a social media post.", + ], +} + +class ImageAdapter(nn.Module): + def __init__(self, input_features: int, output_features: int, ln1: bool, pos_emb: bool, num_image_tokens: int, deep_extract: bool): + super().__init__() + self.deep_extract = deep_extract + + if self.deep_extract: + input_features = input_features * 5 + + self.linear1 = nn.Linear(input_features, output_features) + self.activation = nn.GELU() + self.linear2 = nn.Linear(output_features, output_features) + self.ln1 = nn.Identity() if not ln1 else nn.LayerNorm(input_features) + self.pos_emb = None if not pos_emb else nn.Parameter(torch.zeros(num_image_tokens, input_features)) + + # Other tokens (<|image_start|>, <|image_end|>, <|eot_id|>) + self.other_tokens = nn.Embedding(3, output_features) + self.other_tokens.weight.data.normal_(mean=0.0, std=0.02) # Matches HF's implementation of llama3 + + def forward(self, vision_outputs: torch.Tensor): + if self.deep_extract: + x = torch.concat(( + vision_outputs[-2], + vision_outputs[3], + vision_outputs[7], + vision_outputs[13], + vision_outputs[20], + ), dim=-1) + assert len(x.shape) == 3, f"Expected 3, got {len(x.shape)}" # batch, tokens, features + assert x.shape[-1] == vision_outputs[-2].shape[-1] * 5, f"Expected {vision_outputs[-2].shape[-1] * 5}, got {x.shape[-1]}" + else: + x = vision_outputs[-2] + + x = self.ln1(x) + + if self.pos_emb is not None: + assert x.shape[-2:] == self.pos_emb.shape, f"Expected {self.pos_emb.shape}, got {x.shape[-2:]}" + x = x + self.pos_emb + + x = self.linear1(x) + x = self.activation(x) + x = self.linear2(x) + + # <|image_start|>, IMAGE, <|image_end|> + other_tokens = self.other_tokens(torch.tensor([0, 1], device=self.other_tokens.weight.device).expand(x.shape[0], -1)) + assert other_tokens.shape == (x.shape[0], 2, x.shape[2]), f"Expected {(x.shape[0], 2, x.shape[2])}, got {other_tokens.shape}" + x = torch.cat((other_tokens[:, 0:1], x, other_tokens[:, 1:2]), dim=1) + + return x + + def get_eot_embedding(self): + return self.other_tokens(torch.tensor([2], device=self.other_tokens.weight.device)).squeeze(0) + + +# Global Variables +IS_NF4 = True +IS_LORA = True +MODEL_PATH = DEFAULT_MODEL_PATH +device = "cuda" if torch.cuda.is_available() else "cpu" +print(f"Running on {device}") + +warnings.filterwarnings("ignore", category=UserWarning) +logging.getLogger("transformers").setLevel(logging.ERROR) + +class ImageAdapter(nn.Module): + def __init__(self, input_features: int, output_features: int, ln1: bool, pos_emb: bool, num_image_tokens: int, deep_extract: bool): + super().__init__() + self.deep_extract = deep_extract + + if self.deep_extract: + input_features = input_features * 5 + + self.linear1 = nn.Linear(input_features, output_features) + self.activation = nn.GELU() + self.linear2 = nn.Linear(output_features, output_features) + self.ln1 = nn.Identity() if not ln1 else nn.LayerNorm(input_features) + self.pos_emb = None if not pos_emb else nn.Parameter(torch.zeros(num_image_tokens, input_features)) + + # Mode token + #self.mode_token = nn.Embedding(n_modes, output_features) + #self.mode_token.weight.data.normal_(mean=0.0, std=0.02) # Matches HF's implementation of llama3 + + # Other tokens (<|image_start|>, <|image_end|>, <|eot_id|>) + self.other_tokens = nn.Embedding(3, output_features) + self.other_tokens.weight.data.normal_(mean=0.0, std=0.02) # Matches HF's implementation of llama3 + + def forward(self, vision_outputs: torch.Tensor): + if self.deep_extract: + x = torch.concat(( + vision_outputs[-2], + vision_outputs[3], + vision_outputs[7], + vision_outputs[13], + vision_outputs[20], + ), dim=-1) + assert len(x.shape) == 3, f"Expected 3, got {len(x.shape)}" # batch, tokens, features + assert x.shape[-1] == vision_outputs[-2].shape[-1] * 5, f"Expected {vision_outputs[-2].shape[-1] * 5}, got {x.shape[-1]}" + else: + x = vision_outputs[-2] + + x = self.ln1(x) + + if self.pos_emb is not None: + assert x.shape[-2:] == self.pos_emb.shape, f"Expected {self.pos_emb.shape}, got {x.shape[-2:]}" + x = x + self.pos_emb + + x = self.linear1(x) + x = self.activation(x) + x = self.linear2(x) + + # Mode token + #mode_token = self.mode_token(mode) + #assert mode_token.shape == (x.shape[0], mode_token.shape[1], x.shape[2]), f"Expected {(x.shape[0], 1, x.shape[2])}, got {mode_token.shape}" + #x = torch.cat((x, mode_token), dim=1) + + # <|image_start|>, IMAGE, <|image_end|> + other_tokens = self.other_tokens(torch.tensor([0, 1], device=self.other_tokens.weight.device).expand(x.shape[0], -1)) + assert other_tokens.shape == (x.shape[0], 2, x.shape[2]), f"Expected {(x.shape[0], 2, x.shape[2])}, got {other_tokens.shape}" + x = torch.cat((other_tokens[:, 0:1], x, other_tokens[:, 1:2]), dim=1) + + return x + + def get_eot_embedding(self): + return self.other_tokens(torch.tensor([2], device=self.other_tokens.weight.device)).squeeze(0) + +def load_models(): + global MODEL_PATH, IS_NF4, IS_LORA + try: + if IS_NF4: + from transformers import BitsAndBytesConfig + nf4_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", + bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16) + print("Loading in NF4") + print("Loading CLIP 📎") + clip_processor = AutoProcessor.from_pretrained(CLIP_PATH) + clip_model = AutoModel.from_pretrained(CLIP_PATH).vision_model + assert (CHECKPOINT_PATH / "clip_model.pt").exists() + if (CHECKPOINT_PATH / "clip_model.pt").exists(): + print("Loading VLM's custom vision model 📎") + checkpoint = torch.load(CHECKPOINT_PATH / "clip_model.pt", map_location='cpu', weights_only=False) + checkpoint = {k.replace("_orig_mod.module.", ""): v for k, v in checkpoint.items()} + clip_model.load_state_dict(checkpoint) + del checkpoint + clip_model.eval().requires_grad_(False).to(device) + + print("Loading tokenizer 🪙") + tokenizer = AutoTokenizer.from_pretrained(CHECKPOINT_PATH / "text_model", use_fast=True) + assert isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)), f"Tokenizer is of type {type(tokenizer)}" + + print(f"Loading LLM: {MODEL_PATH} 🤖") + text_model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, quantization_config=nf4_config).eval() + + if False and IS_LORA and LORA_PATH.exists(): # omitted + print("Loading VLM's custom text model 🤖") + text_model = PeftModel.from_pretrained(model=text_model, model_id=LORA_PATH, quantization_config=nf4_config) + text_model = text_model.merge_and_unload(safe_merge=True) # to avoid PEFT bug https://github.com/huggingface/transformers/issues/28515 + else: print("VLM's custom text model isn't loaded 🤖") + + print("Loading image adapter 🖼️") + image_adapter = ImageAdapter(clip_model.config.hidden_size, text_model.config.hidden_size, False, False, 38, False).eval().to("cpu") + image_adapter.load_state_dict(torch.load(CHECKPOINT_PATH / "image_adapter.pt", map_location="cpu", weights_only=False)) + image_adapter.eval().to(device) + else: + print("Loading in bfloat16") + print("Loading CLIP 📎") + clip_processor = AutoProcessor.from_pretrained(CLIP_PATH) + clip_model = AutoModel.from_pretrained(CLIP_PATH).vision_model + if (CHECKPOINT_PATH / "clip_model.pt").exists(): + print("Loading VLM's custom vision model 📎") + checkpoint = torch.load(CHECKPOINT_PATH / "clip_model.pt", map_location='cpu', weights_only=False) + checkpoint = {k.replace("_orig_mod.module.", ""): v for k, v in checkpoint.items()} + clip_model.load_state_dict(checkpoint) + del checkpoint + clip_model.eval().requires_grad_(False).to(device) + + print("Loading tokenizer 🪙") + tokenizer = AutoTokenizer.from_pretrained(CHECKPOINT_PATH / "text_model", use_fast=True) + assert isinstance(tokenizer, (PreTrainedTokenizer, PreTrainedTokenizerFast)), f"Tokenizer is of type {type(tokenizer)}" + + print(f"Loading LLM: {MODEL_PATH} 🤖") + text_model = AutoModelForCausalLM.from_pretrained(MODEL_PATH, device_map="auto", torch_dtype=torch.bfloat16).eval() # device_map="auto" may cause LoRA issue + + if IS_LORA and LORA_PATH.exists(): + print("Loading VLM's custom text model 🤖") + text_model = PeftModel.from_pretrained(model=text_model, model_id=LORA_PATH, device_map=device) + text_model = text_model.merge_and_unload(safe_merge=True) # to avoid PEFT bug https://github.com/huggingface/transformers/issues/28515 + else: print("VLM's custom text model isn't loaded 🤖") + + print("Loading image adapter 🖼️") + image_adapter = ImageAdapter(clip_model.config.hidden_size, text_model.config.hidden_size, False, False, 38, False).eval().to("cpu") + image_adapter.load_state_dict(torch.load(CHECKPOINT_PATH / "image_adapter.pt", map_location="cpu", weights_only=False)) + except Exception as e: + print(f"Error loading models: {e}") + sys.exit(1) + finally: + torch.cuda.empty_cache() + gc.collect() + return clip_processor, clip_model, tokenizer, text_model, image_adapter + +@torch.inference_mode() +def stream_chat(input_images: List[Image.Image], caption_type: str, caption_length: Union[str, int], extra_options: list[str], name_input: str, custom_prompt: str, + max_new_tokens: int, top_p: float, temperature: float, batch_size: int, pbar: tqdm, models: tuple) -> List[str]: + global MODEL_PATH + clip_processor, clip_model, tokenizer, text_model, image_adapter = models + torch.cuda.empty_cache() + all_captions = [] + + # 'any' means no length specified + length = None if caption_length == "any" else caption_length + + if isinstance(length, str): + try: + length = int(length) + except ValueError: + pass + + # Build prompt + if length is None: + map_idx = 0 + elif isinstance(length, int): + map_idx = 1 + elif isinstance(length, str): + map_idx = 2 + else: + raise ValueError(f"Invalid caption length: {length}") + + prompt_str = CAPTION_TYPE_MAP[caption_type][map_idx] + + # Add extra options + if len(extra_options) > 0: + prompt_str += " " + " ".join(extra_options) + + # Add name, length, word_count + prompt_str = prompt_str.format(name=name_input, length=caption_length, word_count=caption_length) + + if custom_prompt.strip() != "": + prompt_str = custom_prompt.strip() + + # For debugging + print(f"Prompt: {prompt_str}") + + for i in range(0, len(input_images), batch_size): + batch = input_images[i:i+batch_size] + + for input_image in input_images: + try: + # Preprocess image + # NOTE: I found the default processor for so400M to have worse results than just using PIL directly + #image = clip_processor(images=input_image, return_tensors='pt').pixel_values + image = input_image.resize((384, 384), Image.LANCZOS) + pixel_values = TVF.pil_to_tensor(image).unsqueeze(0) / 255.0 + pixel_values = TVF.normalize(pixel_values, [0.5], [0.5]) + pixel_values = pixel_values.to(device) + except ValueError as e: + print(f"Error processing image: {e}") + print("Skipping this image and continuing...") + continue + + # Embed image + # This results in Batch x Image Tokens x Features + with torch.amp.autocast_mode.autocast(device, enabled=True): + vision_outputs = clip_model(pixel_values=pixel_values, output_hidden_states=True) + image_features = vision_outputs.hidden_states + embedded_images = image_adapter(image_features).to(device) + + # Build the conversation + convo = [ + { + "role": "system", + "content": "You are a helpful image captioner.", + }, + { + "role": "user", + "content": prompt_str, + }, + ] + + # Format the conversation + convo_string = tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = True) + assert isinstance(convo_string, str) + + # Tokenize the conversation + # prompt_str is tokenized separately so we can do the calculations below + convo_tokens = tokenizer.encode(convo_string, return_tensors="pt", add_special_tokens=False, truncation=False) + prompt_tokens = tokenizer.encode(prompt_str, return_tensors="pt", add_special_tokens=False, truncation=False) + assert isinstance(convo_tokens, torch.Tensor) and isinstance(prompt_tokens, torch.Tensor) + convo_tokens = convo_tokens.squeeze(0) # Squeeze just to make the following easier + prompt_tokens = prompt_tokens.squeeze(0) + + # Calculate where to inject the image + eot_id_indices = (convo_tokens == tokenizer.convert_tokens_to_ids("<|eot_id|>")).nonzero(as_tuple=True)[0].tolist() + assert len(eot_id_indices) == 2, f"Expected 2 <|eot_id|> tokens, got {len(eot_id_indices)}" + + preamble_len = eot_id_indices[1] - prompt_tokens.shape[0] # Number of tokens before the prompt + + # Embed the tokens + convo_embeds = text_model.model.embed_tokens(convo_tokens.unsqueeze(0).to(device)) + + # Construct the input + input_embeds = torch.cat([ + convo_embeds[:, :preamble_len], # Part before the prompt + embedded_images.to(dtype=convo_embeds.dtype), # Image + convo_embeds[:, preamble_len:], # The prompt and anything after it + ], dim=1).to(device) + + input_ids = torch.cat([ + convo_tokens[:preamble_len].unsqueeze(0), + torch.zeros((1, embedded_images.shape[1]), dtype=torch.long), # Dummy tokens for the image (TODO: Should probably use a special token here so as not to confuse any generation algorithms that might be inspecting the input) + convo_tokens[preamble_len:].unsqueeze(0), + ], dim=1).to(device) + attention_mask = torch.ones_like(input_ids) + + # Debugging + #print(f"Input to model: {repr(tokenizer.decode(input_ids[0]))}") + + generate_ids = text_model.generate(input_ids=input_ids, inputs_embeds=input_embeds, attention_mask=attention_mask, do_sample=True, + suppress_tokens=None, max_new_tokens=max_new_tokens, top_p=top_p, temperature=temperature) + + # Trim off the prompt + generate_ids = generate_ids[:, input_ids.shape[1]:] + if generate_ids[0][-1] == tokenizer.eos_token_id or generate_ids[0][-1] == tokenizer.convert_tokens_to_ids("<|eot_id|>"): + generate_ids = generate_ids[:, :-1] + + caption = tokenizer.batch_decode(generate_ids, skip_special_tokens=False, clean_up_tokenization_spaces=False)[0] + all_captions.append(caption.strip()) + + if pbar: + pbar.update(len(batch)) + + return all_captions + +def process_directory(input_dir: Path, output_dir: Path, caption_type: str, caption_length: Union[str, int], extra_options: list[str], name_input: str, custom_prompt: str, + max_new_tokens: int, top_p: float, temperature: float, batch_size: int, models: tuple): + output_dir.mkdir(parents=True, exist_ok=True) + image_files = [f for f in input_dir.iterdir() if f.suffix.lower() in IMAGE_EXTENSIONS] + images_to_process = [f for f in image_files if not (output_dir / f"{f.stem}.txt").exists()] + + if not images_to_process: + print("No new images to process.") + return + + with tqdm(total=len(images_to_process), desc="Processing images", unit="image") as pbar: + for i in range(0, len(images_to_process), batch_size): + batch_files = images_to_process[i:i+batch_size] + batch_images = [Image.open(f).convert('RGB') for f in batch_files] + + captions = stream_chat(batch_images, caption_type, caption_length, extra_options, name_input, custom_prompt, + max_new_tokens, top_p, temperature, batch_size, pbar, models) + + for file, caption in zip(batch_files, captions): + with open(output_dir / f"{file.stem}.txt", 'w', encoding='utf-8') as f: + f.write(caption) + + for img in batch_images: + img.close() + +def parse_arguments(): + parser = argparse.ArgumentParser(description="Process images and generate captions.") + parser.add_argument("input", nargs='+', help="Input image file or directory (or multiple directories)") + parser.add_argument("--output", help="Output directory (optional)") + parser.add_argument("--bs", type=int, default=4, help="Batch size (default: 4)") + parser.add_argument("--type", type=str, default="Descriptive", + choices=["Descriptive", "Descriptive (Informal)", "Training Prompt", "MidJourney", "Booru tag list", "Booru-like tag list", "Art Critic", "Product Listing", "Social Media Post"], + help='Caption Type (default: "Descriptive")') + parser.add_argument("--len", default="long", + choices=["any", "very short", "short", "medium-length", "long", "very long"] + [str(i) for i in range(20, 261, 10)], + help='Caption Length (default: "long")') + parser.add_argument("--extra", default=[], type=list[str], help='Extra Options', + choices=[ + "If there is a person/character in the image you must refer to them as {name}.", + "Do NOT include information about people/characters that cannot be changed (like ethnicity, gender, etc), but do still include changeable attributes (like hair style).", + "Include information about lighting.", + "Include information about camera angle.", + "Include information about whether there is a watermark or not.", + "Include information about whether there are JPEG artifacts or not.", + "If it is a photo you MUST include information about what camera was likely used and details such as aperture, shutter speed, ISO, etc.", + "Do NOT include anything sexual; keep it PG.", + "Do NOT mention the image's resolution.", + "You MUST include information about the subjective aesthetic quality of the image from low to very high.", + "Include information on the image's composition style, such as leading lines, rule of thirds, or symmetry.", + "Do NOT mention any text that is in the image.", + "Specify the depth of field and whether the background is in focus or blurred.", + "If applicable, mention the likely use of artificial or natural lighting sources.", + "Do NOT use any ambiguous language.", + "Include whether the image is sfw, suggestive, or nsfw.", + "ONLY describe the most important elements of the image." + ]) + parser.add_argument("--name", type=str, default="", help='Person/Character Name (if applicable)') + parser.add_argument("--prompt", type=str, default="", help='Custom Prompt (optional, will override all other settings)') + parser.add_argument("--model", type=str, default=DEFAULT_MODEL_PATH, + help='Huggingface LLM repo (default: "unsloth/Meta-Llama-3.1-8B-bnb-4bit")') + parser.add_argument("--bf16", action="store_true", default=False, help="Use bfloat16 (default: NF4)") + parser.add_argument("--nolora", action="store_true", default=False, help="Disable VLM's custom text model (default: Enable)") + parser.add_argument("--tokens", type=int, default=300, help="Max tokens (default: 300)") + parser.add_argument("--topp", type=float, default=0.9, help="Top-P (default: 0.9)") + parser.add_argument("--temp", type=float, default=0.6, help="Temperature (default: 0.6)") + return parser.parse_args() + +def is_valid_repo(repo_id): + from huggingface_hub import HfApi + import re + try: + if not re.fullmatch(r'^[^/,\s\"\']+/[^/,\s\"\']+$', repo_id): return False + api = HfApi() + if api.repo_exists(repo_id=repo_id): return True + else: return False + except Exception as e: + print(f"Failed to connect {repo_id}. {e}") + return False + +def main(): + global MODEL_PATH, IS_NF4, IS_LORA + args = parse_arguments() + input_paths = [Path(input_path) for input_path in args.input] + batch_size = args.bs + caption_type = args.type + caption_length = args.len + extra_options = args.extra + name_input = args.name + custom_prompt = args.prompt + max_new_tokens = args.tokens + top_p = args.topp + temperature = args.temp + IS_NF4 = False if args.bf16 else True + IS_LORA = False if args.nolora else True + if is_valid_repo(args.model): MODEL_PATH = args.model + else: sys.exit(1) + models = load_models() + + for input_path in input_paths: + if input_path.is_file() and input_path.suffix.lower() in IMAGE_EXTENSIONS: + output_path = input_path.with_suffix('.txt') + print(f"Processing single image 🎞️: {input_path.name}") + with tqdm(total=1, desc="Processing image", unit="image") as pbar: + captions = stream_chat([Image.open(input_path).convert('RGB')], caption_type, caption_length, extra_options, name_input, custom_prompt, + max_new_tokens, top_p, temperature, 1, pbar, models) + with open(output_path, 'w', encoding='utf-8') as f: + f.write(captions[0]) + print(f"Output saved to {output_path}") + elif input_path.is_dir(): + output_path = Path(args.output) if args.output else input_path + print(f"Processing directory 📁: {input_path}") + print(f"Output directory 📦: {output_path}") + print(f"Batch size 🗄️: {batch_size}") + process_directory(input_path, output_path, caption_type, caption_length, extra_options, name_input, custom_prompt, + max_new_tokens, top_p, temperature, batch_size, models) + else: + print(f"Invalid input: {input_path}") + print("Skipping...") + + if not input_paths: + print("Usage:") + print("For single image: python app.py [image_file] [--bs batch_size]") + print("For directory (same input/output): python app.py [directory] [--bs batch_size]") + print("For directory (separate input/output): python app.py [directory] --output [output_directory] [--bs batch_size]") + print("For multiple directories: python app.py [directory1] [directory2] ... [--output output_directory] [--bs batch_size]") + sys.exit(1) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/Joy_caption/cgrkzexw-599808/config.yaml b/Joy_caption/cgrkzexw-599808/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a0bf05cd512c7ebba0eb11de730acc97c2dbe0d9 --- /dev/null +++ b/Joy_caption/cgrkzexw-599808/config.yaml @@ -0,0 +1,39 @@ +wandb_project: joy-caption-1 +device_batch_size: 2 +batch_size: 256 +learning_rate: 0.0002 +warmup_samples: 18000 +max_samples: 600000 +save_every: 50000 +test_every: 50000 +use_amp: true +grad_scaler: true +lr_scheduler_type: cosine +min_lr_ratio: 0.0 +allow_tf32: true +seed: 69 +num_workers: 8 +optimizer_type: adamw +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_eps: 1.0e-08 +adam_weight_decay: 0.0 +clip_grad_norm: 1.0 +dataset: fancyfeast/joy-captioning-20240924a +clip_model: google/siglip-so400m-patch14-384 +text_model: ../lora-train/lora_model_vwbzycxh +resume: null +gradient_checkpointing: false +test_size: 2048 +grad_scaler_init: 65536.0 +max_caption_length: 257 +num_image_tokens: 32 +adapter_type: mlp +text_model_dtype: bfloat16 +pre_test: false +train_image_model: true +image_model_lr: null +train_lora: true +lora_r: 64 +lora_alpha: 16 +lora_dropout: 0.1 diff --git a/Joy_caption/cgrkzexw-599808/text_model/README.md b/Joy_caption/cgrkzexw-599808/text_model/README.md new file mode 100644 index 0000000000000000000000000000000000000000..7b065b8e81a5ec1ad09d6c9e8e8a80b650f1b122 --- /dev/null +++ b/Joy_caption/cgrkzexw-599808/text_model/README.md @@ -0,0 +1,202 @@ +--- +base_model: unsloth/Meta-Llama-3.1-8B-Instruct +library_name: peft +--- + +# Model Card for Model ID + + + + + +## Model Details + +### Model Description + + + + + +- **Developed by:** [More Information Needed] +- **Funded by [optional]:** [More Information Needed] +- **Shared by [optional]:** [More Information Needed] +- **Model type:** [More Information Needed] +- **Language(s) (NLP):** [More Information Needed] +- **License:** [More Information Needed] +- **Finetuned from model [optional]:** [More Information Needed] + +### Model Sources [optional] + + + +- **Repository:** [More Information Needed] +- **Paper [optional]:** [More Information Needed] +- **Demo [optional]:** [More Information Needed] + +## Uses + + + +### Direct Use + + + +[More Information Needed] + +### Downstream Use [optional] + + + +[More Information Needed] + +### Out-of-Scope Use + + + +[More Information Needed] + +## Bias, Risks, and Limitations + + + +[More Information Needed] + +### Recommendations + + + +Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations. + +## How to Get Started with the Model + +Use the code below to get started with the model. + +[More Information Needed] + +## Training Details + +### Training Data + + + +[More Information Needed] + +### Training Procedure + + + +#### Preprocessing [optional] + +[More Information Needed] + + +#### Training Hyperparameters + +- **Training regime:** [More Information Needed] + +#### Speeds, Sizes, Times [optional] + + + +[More Information Needed] + +## Evaluation + + + +### Testing Data, Factors & Metrics + +#### Testing Data + + + +[More Information Needed] + +#### Factors + + + +[More Information Needed] + +#### Metrics + + + +[More Information Needed] + +### Results + +[More Information Needed] + +#### Summary + + + +## Model Examination [optional] + + + +[More Information Needed] + +## Environmental Impact + + + +Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700). + +- **Hardware Type:** [More Information Needed] +- **Hours used:** [More Information Needed] +- **Cloud Provider:** [More Information Needed] +- **Compute Region:** [More Information Needed] +- **Carbon Emitted:** [More Information Needed] + +## Technical Specifications [optional] + +### Model Architecture and Objective + +[More Information Needed] + +### Compute Infrastructure + +[More Information Needed] + +#### Hardware + +[More Information Needed] + +#### Software + +[More Information Needed] + +## Citation [optional] + + + +**BibTeX:** + +[More Information Needed] + +**APA:** + +[More Information Needed] + +## Glossary [optional] + + + +[More Information Needed] + +## More Information [optional] + +[More Information Needed] + +## Model Card Authors [optional] + +[More Information Needed] + +## Model Card Contact + +[More Information Needed] +### Framework versions + +- PEFT 0.12.0 \ No newline at end of file diff --git a/Joy_caption/cgrkzexw-599808/text_model/adapter_config.json b/Joy_caption/cgrkzexw-599808/text_model/adapter_config.json new file mode 100644 index 0000000000000000000000000000000000000000..3690c721ad21000a2977e64f919f03b654c0e46b --- /dev/null +++ b/Joy_caption/cgrkzexw-599808/text_model/adapter_config.json @@ -0,0 +1,34 @@ +{ + "alpha_pattern": {}, + "auto_mapping": null, + "base_model_name_or_path": "unsloth/Meta-Llama-3.1-8B-Instruct", + "bias": "none", + "fan_in_fan_out": false, + "inference_mode": true, + "init_lora_weights": true, + "layer_replication": null, + "layers_pattern": null, + "layers_to_transform": null, + "loftq_config": {}, + "lora_alpha": 16, + "lora_dropout": 0, + "megatron_config": null, + "megatron_core": "megatron.core", + "modules_to_save": null, + "peft_type": "LORA", + "r": 64, + "rank_pattern": {}, + "revision": null, + "target_modules": [ + "q_proj", + "v_proj", + "gate_proj", + "down_proj", + "o_proj", + "k_proj", + "up_proj" + ], + "task_type": "CAUSAL_LM", + "use_dora": false, + "use_rslora": false +} \ No newline at end of file diff --git a/Joy_caption/cgrkzexw-599808/text_model/tokenizer.json b/Joy_caption/cgrkzexw-599808/text_model/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..79ca62f28109c3f40505d0e1e71a3a2d85634777 --- /dev/null +++ b/Joy_caption/cgrkzexw-599808/text_model/tokenizer.json @@ -0,0 +1,410563 @@ +{ + "version": "1.0", + "truncation": null, + "padding": null, + "added_tokens": [ + { + "id": 128000, + "content": "<|begin_of_text|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128001, + "content": "<|end_of_text|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128002, + "content": "<|reserved_special_token_0|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128003, + "content": "<|reserved_special_token_1|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128004, + "content": "<|finetune_right_pad_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128005, + "content": "<|reserved_special_token_2|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128006, + "content": "<|start_header_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128007, + "content": "<|end_header_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128008, + "content": "<|eom_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128009, + "content": "<|eot_id|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128010, + "content": "<|python_tag|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128011, + "content": "<|reserved_special_token_3|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128012, + "content": "<|reserved_special_token_4|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128013, + "content": "<|reserved_special_token_5|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128014, + "content": "<|reserved_special_token_6|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128015, + "content": "<|reserved_special_token_7|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128016, + "content": "<|reserved_special_token_8|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128017, + "content": "<|reserved_special_token_9|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128018, + "content": "<|reserved_special_token_10|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128019, + "content": "<|reserved_special_token_11|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128020, + "content": "<|reserved_special_token_12|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128021, + "content": "<|reserved_special_token_13|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128022, + "content": "<|reserved_special_token_14|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128023, + "content": "<|reserved_special_token_15|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128024, + "content": "<|reserved_special_token_16|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128025, + "content": "<|reserved_special_token_17|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128026, + "content": "<|reserved_special_token_18|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128027, + "content": "<|reserved_special_token_19|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128028, + "content": "<|reserved_special_token_20|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128029, + "content": "<|reserved_special_token_21|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128030, + "content": "<|reserved_special_token_22|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128031, + "content": "<|reserved_special_token_23|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128032, + "content": "<|reserved_special_token_24|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128033, + "content": "<|reserved_special_token_25|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128034, + "content": "<|reserved_special_token_26|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128035, + "content": "<|reserved_special_token_27|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128036, + "content": "<|reserved_special_token_28|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128037, + "content": "<|reserved_special_token_29|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128038, + "content": "<|reserved_special_token_30|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128039, + "content": "<|reserved_special_token_31|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128040, + "content": "<|reserved_special_token_32|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128041, + "content": "<|reserved_special_token_33|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128042, + "content": "<|reserved_special_token_34|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128043, + "content": "<|reserved_special_token_35|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128044, + "content": "<|reserved_special_token_36|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128045, + "content": "<|reserved_special_token_37|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128046, + "content": "<|reserved_special_token_38|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128047, + "content": "<|reserved_special_token_39|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128048, + "content": "<|reserved_special_token_40|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128049, + "content": "<|reserved_special_token_41|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128050, + "content": "<|reserved_special_token_42|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128051, + "content": "<|reserved_special_token_43|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128052, + "content": "<|reserved_special_token_44|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128053, + "content": "<|reserved_special_token_45|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128054, + "content": "<|reserved_special_token_46|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128055, + "content": "<|reserved_special_token_47|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128056, + "content": "<|reserved_special_token_48|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128057, + "content": "<|reserved_special_token_49|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128058, + "content": "<|reserved_special_token_50|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128059, + "content": "<|reserved_special_token_51|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128060, + "content": "<|reserved_special_token_52|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128061, + "content": "<|reserved_special_token_53|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128062, + "content": "<|reserved_special_token_54|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128063, + "content": "<|reserved_special_token_55|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128064, + "content": "<|reserved_special_token_56|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128065, + "content": "<|reserved_special_token_57|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128066, + "content": "<|reserved_special_token_58|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128067, + "content": "<|reserved_special_token_59|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128068, + "content": "<|reserved_special_token_60|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128069, + "content": "<|reserved_special_token_61|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128070, + "content": "<|reserved_special_token_62|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128071, + "content": "<|reserved_special_token_63|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128072, + "content": "<|reserved_special_token_64|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128073, + "content": "<|reserved_special_token_65|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128074, + "content": "<|reserved_special_token_66|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128075, + "content": "<|reserved_special_token_67|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128076, + "content": "<|reserved_special_token_68|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128077, + "content": "<|reserved_special_token_69|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128078, + "content": "<|reserved_special_token_70|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128079, + "content": "<|reserved_special_token_71|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128080, + "content": "<|reserved_special_token_72|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128081, + "content": "<|reserved_special_token_73|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128082, + "content": "<|reserved_special_token_74|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128083, + "content": "<|reserved_special_token_75|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128084, + "content": "<|reserved_special_token_76|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128085, + "content": "<|reserved_special_token_77|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128086, + "content": "<|reserved_special_token_78|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128087, + "content": "<|reserved_special_token_79|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128088, + "content": "<|reserved_special_token_80|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128089, + "content": "<|reserved_special_token_81|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128090, + "content": "<|reserved_special_token_82|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128091, + "content": "<|reserved_special_token_83|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128092, + "content": "<|reserved_special_token_84|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128093, + "content": "<|reserved_special_token_85|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128094, + "content": "<|reserved_special_token_86|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128095, + "content": "<|reserved_special_token_87|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128096, + "content": "<|reserved_special_token_88|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128097, + "content": "<|reserved_special_token_89|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128098, + "content": "<|reserved_special_token_90|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128099, + "content": "<|reserved_special_token_91|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128100, + "content": "<|reserved_special_token_92|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128101, + "content": "<|reserved_special_token_93|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128102, + "content": "<|reserved_special_token_94|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128103, + "content": "<|reserved_special_token_95|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128104, + "content": "<|reserved_special_token_96|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128105, + "content": "<|reserved_special_token_97|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128106, + "content": "<|reserved_special_token_98|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128107, + "content": "<|reserved_special_token_99|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128108, + "content": "<|reserved_special_token_100|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128109, + "content": "<|reserved_special_token_101|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128110, + "content": "<|reserved_special_token_102|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128111, + "content": "<|reserved_special_token_103|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128112, + "content": "<|reserved_special_token_104|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128113, + "content": "<|reserved_special_token_105|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128114, + "content": "<|reserved_special_token_106|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128115, + "content": "<|reserved_special_token_107|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128116, + "content": "<|reserved_special_token_108|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128117, + "content": "<|reserved_special_token_109|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128118, + "content": "<|reserved_special_token_110|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128119, + "content": "<|reserved_special_token_111|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128120, + "content": "<|reserved_special_token_112|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128121, + "content": "<|reserved_special_token_113|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128122, + "content": "<|reserved_special_token_114|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128123, + "content": "<|reserved_special_token_115|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128124, + "content": "<|reserved_special_token_116|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128125, + "content": "<|reserved_special_token_117|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128126, + "content": "<|reserved_special_token_118|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128127, + "content": "<|reserved_special_token_119|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128128, + "content": "<|reserved_special_token_120|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128129, + "content": "<|reserved_special_token_121|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128130, + "content": "<|reserved_special_token_122|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128131, + "content": "<|reserved_special_token_123|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128132, + "content": "<|reserved_special_token_124|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128133, + "content": "<|reserved_special_token_125|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128134, + "content": "<|reserved_special_token_126|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128135, + "content": "<|reserved_special_token_127|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128136, + "content": "<|reserved_special_token_128|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128137, + "content": "<|reserved_special_token_129|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128138, + "content": "<|reserved_special_token_130|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128139, + "content": "<|reserved_special_token_131|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128140, + "content": "<|reserved_special_token_132|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128141, + "content": "<|reserved_special_token_133|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128142, + "content": "<|reserved_special_token_134|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128143, + "content": "<|reserved_special_token_135|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128144, + "content": "<|reserved_special_token_136|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128145, + "content": "<|reserved_special_token_137|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128146, + "content": "<|reserved_special_token_138|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128147, + "content": "<|reserved_special_token_139|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128148, + "content": "<|reserved_special_token_140|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128149, + "content": "<|reserved_special_token_141|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128150, + "content": "<|reserved_special_token_142|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128151, + "content": "<|reserved_special_token_143|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128152, + "content": "<|reserved_special_token_144|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128153, + "content": "<|reserved_special_token_145|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128154, + "content": "<|reserved_special_token_146|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128155, + "content": "<|reserved_special_token_147|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128156, + "content": "<|reserved_special_token_148|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128157, + "content": "<|reserved_special_token_149|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128158, + "content": "<|reserved_special_token_150|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128159, + "content": "<|reserved_special_token_151|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128160, + "content": "<|reserved_special_token_152|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128161, + "content": "<|reserved_special_token_153|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128162, + "content": "<|reserved_special_token_154|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128163, + "content": "<|reserved_special_token_155|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128164, + "content": "<|reserved_special_token_156|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128165, + "content": "<|reserved_special_token_157|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128166, + "content": "<|reserved_special_token_158|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128167, + "content": "<|reserved_special_token_159|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128168, + "content": "<|reserved_special_token_160|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128169, + "content": "<|reserved_special_token_161|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128170, + "content": "<|reserved_special_token_162|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128171, + "content": "<|reserved_special_token_163|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128172, + "content": "<|reserved_special_token_164|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128173, + "content": "<|reserved_special_token_165|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128174, + "content": "<|reserved_special_token_166|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128175, + "content": "<|reserved_special_token_167|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128176, + "content": "<|reserved_special_token_168|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128177, + "content": "<|reserved_special_token_169|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128178, + "content": "<|reserved_special_token_170|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128179, + "content": "<|reserved_special_token_171|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128180, + "content": "<|reserved_special_token_172|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128181, + "content": "<|reserved_special_token_173|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128182, + "content": "<|reserved_special_token_174|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128183, + "content": "<|reserved_special_token_175|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128184, + "content": "<|reserved_special_token_176|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128185, + "content": "<|reserved_special_token_177|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128186, + "content": "<|reserved_special_token_178|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128187, + "content": "<|reserved_special_token_179|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128188, + "content": "<|reserved_special_token_180|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128189, + "content": "<|reserved_special_token_181|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128190, + "content": "<|reserved_special_token_182|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128191, + "content": "<|reserved_special_token_183|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128192, + "content": "<|reserved_special_token_184|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128193, + "content": "<|reserved_special_token_185|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128194, + "content": "<|reserved_special_token_186|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128195, + "content": "<|reserved_special_token_187|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128196, + "content": "<|reserved_special_token_188|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128197, + "content": "<|reserved_special_token_189|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128198, + "content": "<|reserved_special_token_190|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128199, + "content": "<|reserved_special_token_191|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128200, + "content": "<|reserved_special_token_192|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128201, + "content": "<|reserved_special_token_193|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128202, + "content": "<|reserved_special_token_194|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128203, + "content": "<|reserved_special_token_195|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128204, + "content": "<|reserved_special_token_196|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128205, + "content": "<|reserved_special_token_197|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128206, + "content": "<|reserved_special_token_198|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128207, + "content": "<|reserved_special_token_199|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128208, + "content": "<|reserved_special_token_200|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128209, + "content": "<|reserved_special_token_201|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128210, + "content": "<|reserved_special_token_202|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128211, + "content": "<|reserved_special_token_203|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128212, + "content": "<|reserved_special_token_204|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128213, + "content": "<|reserved_special_token_205|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128214, + "content": "<|reserved_special_token_206|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128215, + "content": "<|reserved_special_token_207|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128216, + "content": "<|reserved_special_token_208|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128217, + "content": "<|reserved_special_token_209|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128218, + "content": "<|reserved_special_token_210|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128219, + "content": "<|reserved_special_token_211|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128220, + "content": "<|reserved_special_token_212|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128221, + "content": "<|reserved_special_token_213|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128222, + "content": "<|reserved_special_token_214|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128223, + "content": "<|reserved_special_token_215|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128224, + "content": "<|reserved_special_token_216|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128225, + "content": "<|reserved_special_token_217|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128226, + "content": "<|reserved_special_token_218|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128227, + "content": "<|reserved_special_token_219|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128228, + "content": "<|reserved_special_token_220|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128229, + "content": "<|reserved_special_token_221|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128230, + "content": "<|reserved_special_token_222|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128231, + "content": "<|reserved_special_token_223|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128232, + "content": "<|reserved_special_token_224|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128233, + "content": "<|reserved_special_token_225|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128234, + "content": "<|reserved_special_token_226|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128235, + "content": "<|reserved_special_token_227|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128236, + "content": "<|reserved_special_token_228|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128237, + "content": "<|reserved_special_token_229|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128238, + "content": "<|reserved_special_token_230|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128239, + "content": "<|reserved_special_token_231|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128240, + "content": "<|reserved_special_token_232|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128241, + "content": "<|reserved_special_token_233|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128242, + "content": "<|reserved_special_token_234|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128243, + "content": "<|reserved_special_token_235|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128244, + "content": "<|reserved_special_token_236|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128245, + "content": "<|reserved_special_token_237|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128246, + "content": "<|reserved_special_token_238|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128247, + "content": "<|reserved_special_token_239|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128248, + "content": "<|reserved_special_token_240|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128249, + "content": "<|reserved_special_token_241|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128250, + "content": "<|reserved_special_token_242|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128251, + "content": "<|reserved_special_token_243|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128252, + "content": "<|reserved_special_token_244|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128253, + "content": "<|reserved_special_token_245|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128254, + "content": "<|reserved_special_token_246|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + }, + { + "id": 128255, + "content": "<|reserved_special_token_247|>", + "single_word": false, + "lstrip": false, + "rstrip": false, + "normalized": false, + "special": true + } + ], + "normalizer": null, + "pre_tokenizer": { + "type": "Sequence", + "pretokenizers": [ + { + "type": "Split", + "pattern": { + "Regex": "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+" + }, + "behavior": "Isolated", + "invert": false + }, + { + "type": "ByteLevel", + "add_prefix_space": false, + "trim_offsets": true, + "use_regex": false + } + ] + }, + "post_processor": { + "type": "Sequence", + "processors": [ + { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": false, + "use_regex": true + }, + { + "type": "TemplateProcessing", + "single": [ + { + "SpecialToken": { + "id": "<|begin_of_text|>", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "A", + "type_id": 0 + } + } + ], + "pair": [ + { + "SpecialToken": { + "id": "<|begin_of_text|>", + "type_id": 0 + } + }, + { + "Sequence": { + "id": "A", + "type_id": 0 + } + }, + { + "SpecialToken": { + "id": "<|begin_of_text|>", + "type_id": 1 + } + }, + { + "Sequence": { + "id": "B", + "type_id": 1 + } + } + ], + "special_tokens": { + "<|begin_of_text|>": { + "id": "<|begin_of_text|>", + "ids": [ + 128000 + ], + "tokens": [ + "<|begin_of_text|>" + ] + } + } + } + ] + }, + "decoder": { + "type": "ByteLevel", + "add_prefix_space": true, + "trim_offsets": true, + "use_regex": true + }, + "model": { + "type": "BPE", + "dropout": null, + "unk_token": null, + "continuing_subword_prefix": null, + "end_of_word_suffix": null, + "fuse_unk": false, + "byte_fallback": false, + "ignore_merges": true, + "vocab": { + "!": 0, + "\"": 1, + "#": 2, + "$": 3, + "%": 4, + "&": 5, + "'": 6, + "(": 7, + ")": 8, + "*": 9, + "+": 10, + ",": 11, + "-": 12, + ".": 13, + "/": 14, + "0": 15, + "1": 16, + "2": 17, + "3": 18, + "4": 19, + "5": 20, + "6": 21, + "7": 22, + "8": 23, + "9": 24, + ":": 25, + ";": 26, + "<": 27, + "=": 28, + ">": 29, + "?": 30, + "@": 31, + "A": 32, + "B": 33, + "C": 34, + "D": 35, + "E": 36, + "F": 37, + "G": 38, + "H": 39, + "I": 40, + "J": 41, + "K": 42, + "L": 43, + "M": 44, + "N": 45, + "O": 46, + "P": 47, + "Q": 48, + "R": 49, + "S": 50, + "T": 51, + "U": 52, + "V": 53, + "W": 54, + "X": 55, + "Y": 56, + "Z": 57, + "[": 58, + "\\": 59, + "]": 60, + "^": 61, + "_": 62, + "`": 63, + "a": 64, + "b": 65, + "c": 66, + "d": 67, + "e": 68, + "f": 69, + "g": 70, + "h": 71, + "i": 72, + "j": 73, + "k": 74, + "l": 75, + "m": 76, + "n": 77, + "o": 78, + "p": 79, + "q": 80, + "r": 81, + "s": 82, + "t": 83, + "u": 84, + "v": 85, + "w": 86, + "x": 87, + "y": 88, + "z": 89, + "{": 90, + "|": 91, + "}": 92, + "~": 93, + "¡": 94, + "¢": 95, + "£": 96, + "¤": 97, + "¥": 98, + "¦": 99, + "§": 100, + "¨": 101, + "©": 102, + "ª": 103, + "«": 104, + "¬": 105, + "®": 106, + "¯": 107, + "°": 108, + "±": 109, + "²": 110, + "³": 111, + "´": 112, + "µ": 113, + "¶": 114, + "·": 115, + "¸": 116, + "¹": 117, + "º": 118, + "»": 119, + "¼": 120, + "½": 121, + "¾": 122, + "¿": 123, + "À": 124, + "Á": 125, + "Â": 126, + "Ã": 127, + "Ä": 128, + "Å": 129, + "Æ": 130, + "Ç": 131, + "È": 132, + "É": 133, + "Ê": 134, + "Ë": 135, + "Ì": 136, + "Í": 137, + "Î": 138, + "Ï": 139, + "Ð": 140, + "Ñ": 141, + "Ò": 142, + "Ó": 143, + "Ô": 144, + "Õ": 145, + "Ö": 146, + "×": 147, + "Ø": 148, + "Ù": 149, + "Ú": 150, + "Û": 151, + "Ü": 152, + "Ý": 153, + "Þ": 154, + "ß": 155, + "à": 156, + "á": 157, + "â": 158, + "ã": 159, + "ä": 160, + "å": 161, + "æ": 162, + "ç": 163, + "è": 164, + "é": 165, + "ê": 166, + "ë": 167, + "ì": 168, + "í": 169, + "î": 170, + "ï": 171, + "ð": 172, + "ñ": 173, + "ò": 174, + "ó": 175, + "ô": 176, + "õ": 177, + "ö": 178, + "÷": 179, + "ø": 180, + "ù": 181, + "ú": 182, + "û": 183, + "ü": 184, + "ý": 185, + "þ": 186, + "ÿ": 187, + "Ā": 188, + "ā": 189, + "Ă": 190, + "ă": 191, + "Ą": 192, + "ą": 193, + "Ć": 194, + "ć": 195, + "Ĉ": 196, + "ĉ": 197, + "Ċ": 198, + "ċ": 199, + "Č": 200, + "č": 201, + "Ď": 202, + "ď": 203, + "Đ": 204, + "đ": 205, + "Ē": 206, + "ē": 207, + "Ĕ": 208, + "ĕ": 209, + "Ė": 210, + "ė": 211, + "Ę": 212, + "ę": 213, + "Ě": 214, + "ě": 215, + "Ĝ": 216, + "ĝ": 217, + "Ğ": 218, + "ğ": 219, + "Ġ": 220, + "ġ": 221, + "Ģ": 222, + "ģ": 223, + "Ĥ": 224, + "ĥ": 225, + "Ħ": 226, + "ħ": 227, + "Ĩ": 228, + "ĩ": 229, + "Ī": 230, + "ī": 231, + "Ĭ": 232, + "ĭ": 233, + "Į": 234, + "į": 235, + "İ": 236, + "ı": 237, + "IJ": 238, + "ij": 239, + "Ĵ": 240, + "ĵ": 241, + "Ķ": 242, + "ķ": 243, + "ĸ": 244, + "Ĺ": 245, + "ĺ": 246, + "Ļ": 247, + "ļ": 248, + "Ľ": 249, + "ľ": 250, + "Ŀ": 251, + "ŀ": 252, + "Ł": 253, + "ł": 254, + "Ń": 255, + "ĠĠ": 256, + "ĠĠĠĠ": 257, + "in": 258, + "Ġt": 259, + "ĠĠĠĠĠĠĠĠ": 260, + "er": 261, + "ĠĠĠ": 262, + "on": 263, + "Ġa": 264, + "re": 265, + "at": 266, + "st": 267, + "en": 268, + "or": 269, + "Ġth": 270, + "ĊĊ": 271, + "Ġc": 272, + "le": 273, + "Ġs": 274, + "it": 275, + "an": 276, + "ar": 277, + "al": 278, + "Ġthe": 279, + ";Ċ": 280, + "Ġp": 281, + "Ġf": 282, + "ou": 283, + "Ġ=": 284, + "is": 285, + "ĠĠĠĠĠĠĠ": 286, + "ing": 287, + "es": 288, + "Ġw": 289, + "ion": 290, + "ed": 291, + "ic": 292, + "Ġb": 293, + "Ġd": 294, + "et": 295, + "Ġm": 296, + "Ġo": 297, + "ĉĉ": 298, + "ro": 299, + "as": 300, + "el": 301, + "ct": 302, + "nd": 303, + "Ġin": 304, + "Ġh": 305, + "ent": 306, + "id": 307, + "Ġn": 308, + "am": 309, + "ĠĠĠĠĠĠĠĠĠĠĠ": 310, + "Ġto": 311, + "Ġre": 312, + "--": 313, + "Ġ{": 314, + "Ġof": 315, + "om": 316, + ");Ċ": 317, + "im": 318, + "čĊ": 319, + "Ġ(": 320, + "il": 321, + "//": 322, + "Ġand": 323, + "ur": 324, + "se": 325, + "Ġl": 326, + "ex": 327, + "ĠS": 328, + "ad": 329, + "Ġ\"": 330, + "ch": 331, + "ut": 332, + "if": 333, + "**": 334, + "Ġ}": 335, + "em": 336, + "ol": 337, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 338, + "th": 339, + ")Ċ": 340, + "Ġ{Ċ": 341, + "Ġg": 342, + "ig": 343, + "iv": 344, + ",Ċ": 345, + "ce": 346, + "od": 347, + "Ġv": 348, + "ate": 349, + "ĠT": 350, + "ag": 351, + "ay": 352, + "Ġ*": 353, + "ot": 354, + "us": 355, + "ĠC": 356, + "Ġst": 357, + "ĠI": 358, + "un": 359, + "ul": 360, + "ue": 361, + "ĠA": 362, + "ow": 363, + "Ġ'": 364, + "ew": 365, + "Ġ<": 366, + "ation": 367, + "()": 368, + "Ġfor": 369, + "ab": 370, + "ort": 371, + "um": 372, + "ame": 373, + "Ġis": 374, + "pe": 375, + "tr": 376, + "ck": 377, + "âĢ": 378, + "Ġy": 379, + "ist": 380, + "----": 381, + ".ĊĊ": 382, + "he": 383, + "Ġe": 384, + "lo": 385, + "ĠM": 386, + "Ġbe": 387, + "ers": 388, + "Ġon": 389, + "Ġcon": 390, + "ap": 391, + "ub": 392, + "ĠP": 393, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 394, + "ass": 395, + "int": 396, + ">Ċ": 397, + "ly": 398, + "urn": 399, + "Ġ$": 400, + ";ĊĊ": 401, + "av": 402, + "port": 403, + "ir": 404, + "->": 405, + "nt": 406, + "ction": 407, + "end": 408, + "Ġde": 409, + "00": 410, + "ith": 411, + "out": 412, + "turn": 413, + "our": 414, + "ĠĠĠĠĠ": 415, + "lic": 416, + "res": 417, + "pt": 418, + "==": 419, + "Ġthis": 420, + "Ġwh": 421, + "Ġif": 422, + "ĠD": 423, + "ver": 424, + "age": 425, + "ĠB": 426, + "ht": 427, + "ext": 428, + "=\"": 429, + "Ġthat": 430, + "****": 431, + "ĠR": 432, + "Ġit": 433, + "ess": 434, + "ĠF": 435, + "Ġr": 436, + "os": 437, + "and": 438, + "Ġas": 439, + "ect": 440, + "ke": 441, + "rom": 442, + "Ġ//": 443, + "con": 444, + "ĠL": 445, + "(\"": 446, + "qu": 447, + "lass": 448, + "Ġwith": 449, + "iz": 450, + "de": 451, + "ĠN": 452, + "Ġal": 453, + "op": 454, + "up": 455, + "get": 456, + "Ġ}Ċ": 457, + "ile": 458, + "Ġan": 459, + "ata": 460, + "ore": 461, + "ri": 462, + "Ġpro": 463, + ";čĊ": 464, + "ĉĉĉĉ": 465, + "ter": 466, + "ain": 467, + "ĠW": 468, + "ĠE": 469, + "Ġcom": 470, + "Ġreturn": 471, + "art": 472, + "ĠH": 473, + "ack": 474, + "import": 475, + "ublic": 476, + "Ġor": 477, + "est": 478, + "ment": 479, + "ĠG": 480, + "able": 481, + "Ġ-": 482, + "ine": 483, + "ill": 484, + "ind": 485, + "ere": 486, + "::": 487, + "ity": 488, + "Ġ+": 489, + "Ġtr": 490, + "elf": 491, + "ight": 492, + "('": 493, + "orm": 494, + "ult": 495, + "str": 496, + "..": 497, + "\",": 498, + "Ġyou": 499, + "ype": 500, + "pl": 501, + "Ġnew": 502, + "Ġj": 503, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 504, + "Ġfrom": 505, + "Ġex": 506, + "ĠO": 507, + "20": 508, + "ld": 509, + "Ġ[": 510, + "oc": 511, + ":Ċ": 512, + "Ġse": 513, + "Ġle": 514, + "--------": 515, + ".s": 516, + "{Ċ": 517, + "',": 518, + "ant": 519, + "Ġat": 520, + "ase": 521, + ".c": 522, + "Ġch": 523, + "": 524, + "ave": 525, + "ang": 526, + "Ġare": 527, + "Ġint": 528, + "âĢĻ": 529, + "_t": 530, + "ert": 531, + "ial": 532, + "act": 533, + "}Ċ": 534, + "ive": 535, + "ode": 536, + "ost": 537, + "Ġclass": 538, + "Ġnot": 539, + "og": 540, + "ord": 541, + "alue": 542, + "all": 543, + "ff": 544, + "();Ċ": 545, + "ont": 546, + "ime": 547, + "are": 548, + "ĠU": 549, + "Ġpr": 550, + "Ġ:": 551, + "ies": 552, + "ize": 553, + "ure": 554, + "Ġby": 555, + "ire": 556, + "Ġ}ĊĊ": 557, + ".p": 558, + "Ġsh": 559, + "ice": 560, + "ast": 561, + "ption": 562, + "tring": 563, + "ok": 564, + "__": 565, + "cl": 566, + "##": 567, + "Ġhe": 568, + "ard": 569, + ").": 570, + "Ġ@": 571, + "iew": 572, + "ĉĉĉ": 573, + "Ġwas": 574, + "ip": 575, + "this": 576, + "Ġu": 577, + "ĠThe": 578, + "ide": 579, + "ace": 580, + "ib": 581, + "ac": 582, + "rou": 583, + "Ġwe": 584, + "ject": 585, + "Ġpublic": 586, + "ak": 587, + "ve": 588, + "ath": 589, + "oid": 590, + "Ġ=>": 591, + "ust": 592, + "que": 593, + "Ġres": 594, + "))": 595, + "'s": 596, + "Ġk": 597, + "ans": 598, + "yst": 599, + "unction": 600, + "********": 601, + "Ġi": 602, + "Ġus": 603, + "pp": 604, + "10": 605, + "one": 606, + "ail": 607, + "====": 608, + "name": 609, + "Ġstr": 610, + "Ġ/": 611, + "Ġ&": 612, + "ach": 613, + "div": 614, + "ystem": 615, + "ell": 616, + "Ġhave": 617, + "err": 618, + "ould": 619, + "ull": 620, + "pon": 621, + "ĠJ": 622, + "_p": 623, + "Ġ==": 624, + "ign": 625, + "St": 626, + ".Ċ": 627, + "Ġpl": 628, + ");ĊĊ": 629, + "form": 630, + "put": 631, + "ount": 632, + "}ĊĊ": 633, + "dd": 634, + "ite": 635, + "Ġget": 636, + "rr": 637, + "ome": 638, + "ĠâĢ": 639, + "aram": 640, + "cc": 641, + "Ġ*/": 642, + "ER": 643, + "In": 644, + "les": 645, + "_s": 646, + "ong": 647, + "ie": 648, + "Ġcan": 649, + "ĠV": 650, + "erv": 651, + "pr": 652, + "Ġun": 653, + "row": 654, + "ber": 655, + "Ġdo": 656, + "ll": 657, + "Ġel": 658, + "Ġself": 659, + "ated": 660, + "ary": 661, + "Ġ.": 662, + "']": 663, + "ud": 664, + "Ġen": 665, + "ĠTh": 666, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 667, + "te": 668, + "_c": 669, + "uct": 670, + "Ġab": 671, + "ork": 672, + ".get": 673, + "Ġ#": 674, + "aw": 675, + "ress": 676, + "ob": 677, + "Name": 678, + "201": 679, + "app": 680, + "['": 681, + "Ġall": 682, + "ory": 683, + "ition": 684, + "ance": 685, + "ear": 686, + "Ġcont": 687, + "vent": 688, + "ia": 689, + "Ġwill": 690, + "IN": 691, + "ĠĠĠĠĠĠĠĠĠ": 692, + "return": 693, + "Ġ": 694, + "data": 695, + ")ĊĊ": 696, + "Re": 697, + "ple": 698, + "ild": 699, + "ther": 700, + "Ġyour": 701, + "\"Ċ": 702, + "($": 703, + "Ġout": 704, + "),": 705, + "Ġhas": 706, + "String": 707, + "so": 708, + "Ġup": 709, + "ax": 710, + "Ġdef": 711, + "Ġbo": 712, + "ge": 713, + "alse": 714, + "ON": 715, + "per": 716, + "12": 717, + "ich": 718, + "Ġbut": 719, + "ĠĊ": 720, + "Ġ_": 721, + "_m": 722, + "add": 723, + "quest": 724, + "odel": 725, + "self": 726, + "ery": 727, + "ft": 728, + "ens": 729, + "////": 730, + "ake": 731, + ".C": 732, + "Ġgo": 733, + "Ġfunction": 734, + "ĠK": 735, + "ivate": 736, + "Ġim": 737, + "Ġconst": 738, + ".t": 739, + "Ġ*/Ċ": 740, + ");čĊ": 741, + "Ġvoid": 742, + "Ġset": 743, + "ĠSystem": 744, + "cri": 745, + "()Ċ": 746, + "li": 747, + "ĉif": 748, + ".m": 749, + "ally": 750, + "set": 751, + "ep": 752, + "âĢĻs": 753, + "bo": 754, + "def": 755, + "',Ċ": 756, + "Ġme": 757, + "Ġ!": 758, + "atch": 759, + "\">": 760, + "\",Ċ": 761, + "ec": 762, + "ĠIn": 763, + "ph": 764, + "Ġ|": 765, + "_f": 766, + "Ġvar": 767, + "ence": 768, + "Id": 769, + "ree": 770, + "ink": 771, + "lect": 772, + "ug": 773, + "eth": 774, + "Ġelse": 775, + "----------------": 776, + "19": 777, + "cont": 778, + "Ġso": 779, + "atic": 780, + "Ġlo": 781, + "pro": 782, + "ton": 783, + "ss": 784, + "own": 785, + "abel": 786, + "oint": 787, + "ous": 788, + "eld": 789, + "ST": 790, + "The": 791, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 792, + "RE": 793, + "\":": 794, + "olor": 795, + "tp": 796, + "eg": 797, + "key": 798, + "ude": 799, + "ĠSt": 800, + "ound": 801, + "Ġar": 802, + "\");Ċ": 803, + "ener": 804, + "ser": 805, + "11": 806, + "bject": 807, + "essage": 808, + "fer": 809, + "Ġmore": 810, + "ations": 811, + "ents": 812, + "Ġhis": 813, + "Ġthey": 814, + ".S": 815, + "ĠY": 816, + "use": 817, + "ne": 818, + "ish": 819, + "old": 820, + "_d": 821, + "io": 822, + "ield": 823, + "Ġper": 824, + "Cont": 825, + "ings": 826, + "####": 827, + "Ġdata": 828, + "Ġsa": 829, + "ef": 830, + "fo": 831, + "Ġone": 832, + "eng": 833, + "Ġdis": 834, + "AT": 835, + "Ġname": 836, + "Ġtrue": 837, + "val": 838, + "led": 839, + ".f": 840, + "Ġne": 841, + "Ġend": 842, + "32": 843, + ".T": 844, + "16": 845, + "cre": 846, + "ark": 847, + "log": 848, + "Ex": 849, + "error": 850, + "_id": 851, + "urre": 852, + "ange": 853, + "Ġnull": 854, + "rray": 855, + "Ġmy": 856, + "pan": 857, + "ict": 858, + "ator": 859, + "View": 860, + "List": 861, + "ĉreturn": 862, + "âĢĿ": 863, + "Ġpre": 864, + "Ġx": 865, + "clude": 866, + "arg": 867, + "15": 868, + "ov": 869, + ".h": 870, + "Ġ>": 871, + "Ġtheir": 872, + "')": 873, + "irst": 874, + "ick": 875, + "gh": 876, + "LE": 877, + "OR": 878, + "Ġprivate": 879, + "tem": 880, + "čĊčĊ": 881, + "user": 882, + "Ġ)": 883, + "com": 884, + ".A": 885, + "\";Ċ": 886, + "Ġid": 887, + "read": 888, + "Ġwho": 889, + "_b": 890, + "\">Ċ": 891, + "Ġtime": 892, + "Ġman": 893, + "ry": 894, + "========": 895, + "roup": 896, + "rop": 897, + "public": 898, + "vel": 899, + "umber": 900, + "ble": 901, + "Ġwhich": 902, + "****************": 903, + "Ġany": 904, + "Ġfalse": 905, + "we": 906, + "Ġvalue": 907, + "Ġli": 908, + "\")": 909, + "nder": 910, + "gr": 911, + "Ġno": 912, + "param": 913, + "25": 914, + "fig": 915, + ".com": 916, + "Ġapp": 917, + "_l": 918, + "ions": 919, + ".D": 920, + "ĠCh": 921, + "Ġabout": 922, + "Ġadd": 923, + "Ġsu": 924, + "Ġstring": 925, + "ID": 926, + "Ġover": 927, + "string": 928, + ".l": 929, + "ource": 930, + "000": 931, + "_C": 932, + "]Ċ": 933, + "Ġqu": 934, + "ĠString": 935, + "ca": 936, + "SE": 937, + "Ġro": 938, + "sh": 939, + "ual": 940, + "Type": 941, + "son": 942, + "new": 943, + "ern": 944, + "Ġag": 945, + "AR": 946, + "];Ċ": 947, + "].": 948, + "Ġ?": 949, + "ical": 950, + "Ġdes": 951, + "uth": 952, + "ix": 953, + "ays": 954, + "Ġtype": 955, + "'t": 956, + "ault": 957, + "Ġinter": 958, + "var": 959, + ".b": 960, + "Ġpart": 961, + ".d": 962, + "urrent": 963, + "IT": 964, + "EN": 965, + "30": 966, + "enc": 967, + "(f": 968, + "ra": 969, + "value": 970, + "cho": 971, + "18": 972, + "utton": 973, + "ose": 974, + "14": 975, + "Ġ!=": 976, + "ater": 977, + "é": 978, + "reate": 979, + "oll": 980, + "pos": 981, + "yle": 982, + "ng": 983, + "AL": 984, + "using": 985, + "ames": 986, + "Ġ{čĊ": 987, + "ates": 988, + "ely": 989, + "Ġwork": 990, + "Ġem": 991, + "inal": 992, + "Ġsp": 993, + "Ġwhen": 994, + ".set": 995, + "ĠĠĠĠĠĠ": 996, + "):Ċ": 997, + "to": 998, + "quire": 999, + "indow": 1000, + "lement": 1001, + "pect": 1002, + "ash": 1003, + "[i": 1004, + "Ġuse": 1005, + ".F": 1006, + "pec": 1007, + "Ġad": 1008, + "ove": 1009, + "ception": 1010, + "ength": 1011, + "include": 1012, + "ader": 1013, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1014, + "atus": 1015, + "Th": 1016, + "itle": 1017, + "rit": 1018, + "void": 1019, + "().": 1020, + "(Ċ": 1021, + "Ġoff": 1022, + "Ġother": 1023, + "Ġ&&": 1024, + "';Ċ": 1025, + "ms": 1026, + "Ġbeen": 1027, + "Ġte": 1028, + "ml": 1029, + "co": 1030, + "nc": 1031, + "13": 1032, + "ervice": 1033, + "Ġ%": 1034, + "**Ċ": 1035, + "ann": 1036, + "ade": 1037, + "ĊĊĊĊ": 1038, + "lock": 1039, + "const": 1040, + "100": 1041, + "ponse": 1042, + "Ġsup": 1043, + "++": 1044, + "date": 1045, + "Ġacc": 1046, + "Ġhad": 1047, + "Ġbu": 1048, + "200": 1049, + "ĠRe": 1050, + "Ġwere": 1051, + "Ġfile": 1052, + "Ġwould": 1053, + "ĠâĢľ": 1054, + "ven": 1055, + "iss": 1056, + "Ġour": 1057, + "class": 1058, + "raw": 1059, + "Ġyear": 1060, + "Data": 1061, + "Ġval": 1062, + "Ġsome": 1063, + "fter": 1064, + "ys": 1065, + "Ġ///": 1066, + "round": 1067, + "view": 1068, + "Ġpe": 1069, + "Ġthere": 1070, + "Ġsaid": 1071, + "du": 1072, + "of": 1073, + "line": 1074, + "/*": 1075, + "duct": 1076, + "Ġher": 1077, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1078, + "Res": 1079, + "Ġco": 1080, + "Ġcomm": 1081, + "ise": 1082, + "min": 1083, + "ĠĠĠĠĊ": 1084, + "#include": 1085, + "ethod": 1086, + ".P": 1087, + "ute": 1088, + "Ġass": 1089, + "Int": 1090, + "ask": 1091, + "loc": 1092, + "Ġlike": 1093, + "ody": 1094, + "Ġlet": 1095, + "load": 1096, + "Ġam": 1097, + "rol": 1098, + "Ġgr": 1099, + "yp": 1100, + "Ġalso": 1101, + "ĠIt": 1102, + "url": 1103, + "ific": 1104, + "ors": 1105, + "_P": 1106, + "_n": 1107, + "igh": 1108, + "Ġthan": 1109, + "Com": 1110, + "AN": 1111, + "UL": 1112, + "ating": 1113, + "17": 1114, + "ĠThis": 1115, + "ref": 1116, + "_S": 1117, + "Ġstatic": 1118, + "roll": 1119, + "Ġjust": 1120, + "Ġresult": 1121, + "ian": 1122, + "idth": 1123, + "Ġthem": 1124, + "));Ċ": 1125, + "der": 1126, + "reak": 1127, + "Con": 1128, + "://": 1129, + "ule": 1130, + "...": 1131, + "arch": 1132, + "ement": 1133, + "Ġ<<": 1134, + "50": 1135, + "ush": 1136, + "ense": 1137, + "arr": 1138, + "Ġinto": 1139, + "cess": 1140, + "amp": 1141, + "ied": 1142, + "ument": 1143, + "Ġ\\": 1144, + "],": 1145, + "wo": 1146, + "als": 1147, + "Ġwhat": 1148, + "anc": 1149, + "Value": 1150, + "='": 1151, + "olum": 1152, + "Ġpos": 1153, + "ages": 1154, + "ayer": 1155, + "Ġsc": 1156, + "ues": 1157, + "\")Ċ": 1158, + "_T": 1159, + "Ġlist": 1160, + "(s": 1161, + "Ġcase": 1162, + "Ch": 1163, + "ĉĉĉĉĉ": 1164, + "////////": 1165, + "ponent": 1166, + "Ġz": 1167, + "Ġkn": 1168, + "let": 1169, + "DE": 1170, + "red": 1171, + "Ġfe": 1172, + "Ġ},Ċ": 1173, + "Ġ,": 1174, + "(t": 1175, + "Ġfirst": 1176, + "');Ċ": 1177, + "word": 1178, + "Ġimport": 1179, + "Ġact": 1180, + "Ġchar": 1181, + "CT": 1182, + "ĠTr": 1183, + "ople": 1184, + "={": 1185, + "ĉf": 1186, + "24": 1187, + "ient": 1188, + "cent": 1189, + ".j": 1190, + "lection": 1191, + "))Ċ": 1192, + "Ġonly": 1193, + "Ġprint": 1194, + "mer": 1195, + ".W": 1196, + "ock": 1197, + "Ġ--": 1198, + "Text": 1199, + "Ġop": 1200, + "ank": 1201, + "Ġits": 1202, + "Ġback": 1203, + "[\"": 1204, + "Ġneed": 1205, + "Ġcl": 1206, + "Ġsub": 1207, + "Ġla": 1208, + "((": 1209, + ".\"": 1210, + "Object": 1211, + "Ġstart": 1212, + "file": 1213, + "(self": 1214, + "ner": 1215, + "ey": 1216, + "Ġuser": 1217, + "Ġent": 1218, + "ĠCom": 1219, + "its": 1220, + "ĠCon": 1221, + "ouble": 1222, + "ower": 1223, + "item": 1224, + "very": 1225, + "ĠWe": 1226, + "64": 1227, + "lick": 1228, + "ĠQ": 1229, + "php": 1230, + "ttp": 1231, + "':": 1232, + "ics": 1233, + "Ġunder": 1234, + "Ġ*Ċ": 1235, + ".L": 1236, + ");": 1237, + "ices": 1238, + "Ġreg": 1239, + ")čĊ": 1240, + "ĉpublic": 1241, + "SS": 1242, + "Ġthen": 1243, + "reat": 1244, + "ious": 1245, + ".G": 1246, + "ek": 1247, + "irect": 1248, + "heck": 1249, + "cript": 1250, + "ning": 1251, + "ĠUn": 1252, + "Ġmay": 1253, + "ĠWh": 1254, + "Bo": 1255, + "Item": 1256, + "struct": 1257, + ".st": 1258, + "ream": 1259, + "ible": 1260, + "loat": 1261, + "Ġorg": 1262, + "und": 1263, + "sum": 1264, + "_in": 1265, + "../": 1266, + "_M": 1267, + "Ġhow": 1268, + "rite": 1269, + "'Ċ": 1270, + "To": 1271, + "40": 1272, + "ww": 1273, + "Ġpeople": 1274, + "index": 1275, + ".n": 1276, + "http": 1277, + "(m": 1278, + "ector": 1279, + "Ġind": 1280, + "Ġjav": 1281, + "],Ċ": 1282, + "ĠHe": 1283, + "_st": 1284, + "ful": 1285, + "ole": 1286, + "){Ċ": 1287, + "Ġshould": 1288, + "opy": 1289, + "elp": 1290, + "ier": 1291, + "_name": 1292, + "erson": 1293, + "ION": 1294, + "ote": 1295, + "Ġtest": 1296, + "Ġbet": 1297, + "rror": 1298, + "ular": 1299, + "ãĢ": 1300, + "ĠÐ": 1301, + "bs": 1302, + "ting": 1303, + "Ġmake": 1304, + "Tr": 1305, + "Ġafter": 1306, + "arget": 1307, + "RO": 1308, + "olumn": 1309, + "rc": 1310, + "_re": 1311, + "define": 1312, + "22": 1313, + "Ġright": 1314, + "right": 1315, + "day": 1316, + "Ġlong": 1317, + "[]": 1318, + "(p": 1319, + "td": 1320, + "cond": 1321, + "ĠPro": 1322, + "Ġrem": 1323, + "ptions": 1324, + "vid": 1325, + ".g": 1326, + "Ġext": 1327, + "Ġ__": 1328, + "')Ċ": 1329, + "pace": 1330, + "mp": 1331, + "Ġmin": 1332, + "stance": 1333, + "air": 1334, + "action": 1335, + "wh": 1336, + "type": 1337, + "util": 1338, + "ait": 1339, + "": 1340, + "IC": 1341, + "text": 1342, + "Ġph": 1343, + "Ġfl": 1344, + ".M": 1345, + "ccess": 1346, + "br": 1347, + "fore": 1348, + "ersion": 1349, + "),Ċ": 1350, + ".re": 1351, + "ateg": 1352, + "Ġloc": 1353, + "ins": 1354, + "-s": 1355, + "trib": 1356, + "ĠInt": 1357, + "Ġarray": 1358, + ",\"": 1359, + "Pro": 1360, + "(c": 1361, + "ession": 1362, + ">ĊĊ": 1363, + "Ġshe": 1364, + "\"]": 1365, + "aph": 1366, + "Ġexp": 1367, + "erty": 1368, + "ĠSe": 1369, + "Ġpar": 1370, + "unc": 1371, + "ET": 1372, + "Ġread": 1373, + "print": 1374, + "Ġrel": 1375, + "Ġform": 1376, + "Ġdr": 1377, + "Exception": 1378, + "input": 1379, + "Ġtrans": 1380, + "########": 1381, + "order": 1382, + "By": 1383, + "Ġaw": 1384, + "ities": 1385, + "uff": 1386, + "play": 1387, + ".add": 1388, + "ĠâĢĵ": 1389, + "Ġwant": 1390, + "Ġcomp": 1391, + "ments": 1392, + "Ġ||": 1393, + "az": 1394, + "be": 1395, + "Ġnumber": 1396, + "Ġrequire": 1397, + "ĠEx": 1398, + "60": 1399, + "Ġcol": 1400, + "Ġkey": 1401, + "ember": 1402, + "Ġtwo": 1403, + "Ġsize": 1404, + "Ġwhere": 1405, + "UT": 1406, + "result": 1407, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1408, + "ough": 1409, + "orld": 1410, + "ood": 1411, + "uch": 1412, + "ative": 1413, + "ger": 1414, + "arent": 1415, + "Ġ/*": 1416, + "Ġarg": 1417, + "Ġwhile": 1418, + "23": 1419, + "(this": 1420, + "Ġrec": 1421, + "Ġdif": 1422, + "State": 1423, + "Ġspec": 1424, + "ride": 1425, + "_F": 1426, + "Ġlook": 1427, + "AM": 1428, + "ility": 1429, + "eter": 1430, + "âĢĻt": 1431, + "ĊĊĊ": 1432, + "ayout": 1433, + "--------------------------------": 1434, + "ager": 1435, + "Ġcould": 1436, + "Ġbr": 1437, + "ends": 1438, + "ures": 1439, + "Ġknow": 1440, + "ets": 1441, + "ĠIf": 1442, + "ĠSh": 1443, + ".w": 1444, + "back": 1445, + "Ġser": 1446, + "Ġ+=": 1447, + "Ġfr": 1448, + "());Ċ": 1449, + "Ġhand": 1450, + "Ind": 1451, + "ULL": 1452, + "Im": 1453, + "();ĊĊ": 1454, + "Ġmost": 1455, + "Ġtry": 1456, + "Ġnow": 1457, + "rough": 1458, + ">čĊ": 1459, + "ackage": 1460, + "Ġhim": 1461, + "._": 1462, + "ify": 1463, + "Ġbreak": 1464, + "Ġ);Ċ": 1465, + "ren": 1466, + "#define": 1467, + "itt": 1468, + "Ġap": 1469, + "ĉc": 1470, + "(n": 1471, + "ĠYou": 1472, + ":ĊĊ": 1473, + "-m": 1474, + "Ġevery": 1475, + "ustom": 1476, + "lient": 1477, + "ocument": 1478, + "cription": 1479, + "Error": 1480, + "-b": 1481, + "о": 1482, + "][": 1483, + "99": 1484, + "trans": 1485, + "Ġpoint": 1486, + "Ġstd": 1487, + "Ġfil": 1488, + "Time": 1489, + "80": 1490, + "Ġmod": 1491, + "Ġ->": 1492, + "Ġerror": 1493, + "ah": 1494, + "Ġtext": 1495, + "roller": 1496, + "lose": 1497, + "ql": 1498, + "Ġpol": 1499, + ">": 1500, + "Ġshow": 1501, + "User": 1502, + "ased": 1503, + "Ġ{ĊĊ": 1504, + "Ġfind": 1505, + "а": 1506, + "ED": 1507, + "span": 1508, + "enu": 1509, + "Ġcurrent": 1510, + "Ġused": 1511, + "cept": 1512, + "clud": 1513, + "Ġplay": 1514, + "Ġlog": 1515, + "ution": 1516, + "fl": 1517, + "Ġsee": 1518, + "indows": 1519, + "Ġhelp": 1520, + "Ġthese": 1521, + "Ġpass": 1522, + "Ġdown": 1523, + "Ġeven": 1524, + "ason": 1525, + "uild": 1526, + "from": 1527, + "(d": 1528, + "Ġbl": 1529, + "label": 1530, + "else": 1531, + "е": 1532, + "Ġ(!": 1533, + "ized": 1534, + "(),": 1535, + "Ġob": 1536, + "Ġitem": 1537, + "ump": 1538, + "UR": 1539, + "orn": 1540, + "Ġdon": 1541, + "Se": 1542, + "man": 1543, + "27": 1544, + "ample": 1545, + "tn": 1546, + "================": 1547, + "He": 1548, + "gram": 1549, + "Ġdid": 1550, + "wn": 1551, + "_h": 1552, + "iver": 1553, + "Ġsm": 1554, + "Ġthrough": 1555, + "ĠAn": 1556, + "che": 1557, + "Ġinv": 1558, + "ouse": 1559, + "Ġes": 1560, + "ĠNew": 1561, + "export": 1562, + "mary": 1563, + "uto": 1564, + "ler": 1565, + "Ġlast": 1566, + "Ġevent": 1567, + "try": 1568, + "ï¼": 1569, + "ily": 1570, + "igned": 1571, + "ines": 1572, + "ollow": 1573, + "icense": 1574, + "sole": 1575, + "lear": 1576, + "(int": 1577, + "Ġagain": 1578, + "Ġhigh": 1579, + "html": 1580, + "Index": 1581, + "uthor": 1582, + "Ġ/**Ċ": 1583, + "Ġline": 1584, + "Event": 1585, + "_D": 1586, + "Ġdoes": 1587, + "itial": 1588, + "Ġcr": 1589, + "ars": 1590, + "28": 1591, + "Ġtem": 1592, + "cause": 1593, + "face": 1594, + "Ġ`": 1595, + "_A": 1596, + "Button": 1597, + "ature": 1598, + "ected": 1599, + "ES": 1600, + "ister": 1601, + "ĉĊ": 1602, + "Ġbefore": 1603, + "ale": 1604, + "other": 1605, + "Ġbecause": 1606, + "roid": 1607, + "Ġed": 1608, + "ik": 1609, + "reg": 1610, + "ĠDe": 1611, + "Ġdist": 1612, + "},Ċ": 1613, + "Ġstate": 1614, + "Ġcons": 1615, + "rint": 1616, + "att": 1617, + "Ġhere": 1618, + "ined": 1619, + "Ġfinal": 1620, + "Ġ\"\"": 1621, + "Key": 1622, + "LO": 1623, + "Ġdel": 1624, + "pty": 1625, + "thing": 1626, + "26": 1627, + "ĠAnd": 1628, + "Ġrun": 1629, + "ĠX": 1630, + "ym": 1631, + ".app": 1632, + "Ġvery": 1633, + "ces": 1634, + "_N": 1635, + "ared": 1636, + "ward": 1637, + "list": 1638, + "ited": 1639, + "olog": 1640, + "itch": 1641, + "Box": 1642, + "ife": 1643, + "33": 1644, + "Ġac": 1645, + "Ġmodel": 1646, + "Ġmon": 1647, + "Ġway": 1648, + "lete": 1649, + "Ġcall": 1650, + "Ġatt": 1651, + "Ġcal": 1652, + "vert": 1653, + "Ġdec": 1654, + "lease": 1655, + "oun": 1656, + "Ġ});Ċ": 1657, + "fr": 1658, + "formation": 1659, + "etail": 1660, + "Ġnum": 1661, + "aj": 1662, + "query": 1663, + "Ġwell": 1664, + "Ġobject": 1665, + "ĠAs": 1666, + "Ġyears": 1667, + "Color": 1668, + "IS": 1669, + "Ġdefault": 1670, + "Wh": 1671, + "Ġins": 1672, + "aint": 1673, + "Ġjava": 1674, + "Ġsim": 1675, + "ĠAr": 1676, + "mon": 1677, + "til": 1678, + "();čĊ": 1679, + "):": 1680, + "Set": 1681, + "29": 1682, + "atter": 1683, + "Ġview": 1684, + "Ġpres": 1685, + "array": 1686, + "We": 1687, + "At": 1688, + "Ġbel": 1689, + "Ġmany": 1690, + "21": 1691, + "Man": 1692, + "ender": 1693, + "Ġbeing": 1694, + "Ġgood": 1695, + "ĉĉĉĉĉĉ": 1696, + "ational": 1697, + "ware": 1698, + ".log": 1699, + "{čĊ": 1700, + "Ġusing": 1701, + "_B": 1702, + "Ġ:=": 1703, + "_w": 1704, + "ists": 1705, + "lish": 1706, + "Ġstud": 1707, + "ĠAl": 1708, + "Ġgu": 1709, + "config": 1710, + "uring": 1711, + "time": 1712, + "oken": 1713, + "amespace": 1714, + "Ġrequest": 1715, + "Ġchild": 1716, + "ĠÃ": 1717, + "lob": 1718, + "Ġparam": 1719, + "Ġ}čĊ": 1720, + "01": 1721, + "Ġecho": 1722, + "function": 1723, + "********************************": 1724, + "ps": 1725, + "Element": 1726, + "alk": 1727, + "lication": 1728, + "by": 1729, + "Size": 1730, + "rawing": 1731, + "Ġperson": 1732, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1733, + "\\n": 1734, + "object": 1735, + "ince": 1736, + "En": 1737, + "File": 1738, + "uf": 1739, + "ffect": 1740, + "AC": 1741, + "Ġstyle": 1742, + "summary": 1743, + "Ġque": 1744, + "_r": 1745, + "Ġ($": 1746, + "Model": 1747, + "ident": 1748, + "Ġmethod": 1749, + "IL": 1750, + "ott": 1751, + "less": 1752, + "ING": 1753, + "Ġ()": 1754, + "Ġexpect": 1755, + "ync": 1756, + "package": 1757, + "35": 1758, + "urs": 1759, + "Ġprot": 1760, + "./": 1761, + "pre": 1762, + "Ġ)Ċ": 1763, + "ma": 1764, + "Ġsur": 1765, + "Ġfound": 1766, + "Info": 1767, + "par": 1768, + "imes": 1769, + ".e": 1770, + "ains": 1771, + "Ġpost": 1772, + "-d": 1773, + "45": 1774, + "olean": 1775, + "Ġsl": 1776, + "PE": 1777, + "Ġsuch": 1778, + "select": 1779, + "ainer": 1780, + "Ġthink": 1781, + "Ġdiffer": 1782, + ".r": 1783, + "/**Ċ": 1784, + "FF": 1785, + "ool": 1786, + "plate": 1787, + "qual": 1788, + "ĠFor": 1789, + "Ġmuch": 1790, + "uc": 1791, + "(new": 1792, + "odule": 1793, + "Ġsom": 1794, + "Ġhttp": 1795, + "ĠList": 1796, + "Ġcount": 1797, + "Ġinst": 1798, + "char": 1799, + "mit": 1800, + ".id": 1801, + "aking": 1802, + "Ġgener": 1803, + "px": 1804, + "vice": 1805, + "37": 1806, + "_data": 1807, + "ĠNULL": 1808, + "}čĊ": 1809, + "idd": 1810, + "ãĢĤ": 1811, + "Ġmed": 1812, + "org": 1813, + "ider": 1814, + "ache": 1815, + "work": 1816, + "Ġcheck": 1817, + "ween": 1818, + "Ġ((": 1819, + "the": 1820, + "ants": 1821, + "><": 1822, + ".B": 1823, + "-c": 1824, + "Ġopen": 1825, + "Ġest": 1826, + "ĠĠĠĠĠĠĠĠĊ": 1827, + "Ġnext": 1828, + "IM": 1829, + "ÑĤ": 1830, + "OT": 1831, + "ó": 1832, + "Ġfollow": 1833, + "content": 1834, + "ĠĠĠĠĠĠĠĠĠĠĠĠ": 1835, + "Ġinclud": 1836, + "HE": 1837, + "ĠRes": 1838, + "Ġhref": 1839, + "и": 1840, + "Ġcar": 1841, + "ypes": 1842, + "image": 1843, + "Un": 1844, + "Ġbool": 1845, + "AD": 1846, + "Ġgame": 1847, + ".Form": 1848, + "rows": 1849, + "*/": 1850, + "velop": 1851, + ".Drawing": 1852, + "Ġpath": 1853, + "ision": 1854, + "Ġeach": 1855, + "ĠPl": 1856, + "_type": 1857, + "Path": 1858, + "nection": 1859, + "Ġav": 1860, + "').": 1861, + "Ġsupport": 1862, + "ENT": 1863, + "rem": 1864, + "\").": 1865, + "Ġown": 1866, + "Ġcor": 1867, + "count": 1868, + "miss": 1869, + "ually": 1870, + "Ġmem": 1871, + "std": 1872, + "ience": 1873, + "search": 1874, + "\"ĊĊ": 1875, + "Form": 1876, + "Ġsex": 1877, + "ename": 1878, + "Ġsign": 1879, + "Ġet": 1880, + "ĠĠĠĠĠĠĠĠĠĠ": 1881, + "','": 1882, + "ĠApp": 1883, + "Ġthose": 1884, + "off": 1885, + "Ġerr": 1886, + "Ġsystem": 1887, + "Ġbest": 1888, + "code": 1889, + "Ġsame": 1890, + "Ġdi": 1891, + "uss": 1892, + "Ġcreate": 1893, + "ather": 1894, + "Array": 1895, + ".in": 1896, + "fe": 1897, + "Service": 1898, + "UN": 1899, + "ats": 1900, + "ĠZ": 1901, + "alth": 1902, + "Ġmade": 1903, + "true": 1904, + "AB": 1905, + "Ġmark": 1906, + "rid": 1907, + "ified": 1908, + ",čĊ": 1909, + "yn": 1910, + "press": 1911, + "Ġgroup": 1912, + "Ġfin": 1913, + "ĠLicense": 1914, + "Field": 1915, + "eger": 1916, + "Ġworld": 1917, + "iness": 1918, + "ty": 1919, + "Ġprocess": 1920, + "(b": 1921, + "Ġcre": 1922, + "arn": 1923, + "ives": 1924, + "Ġmain": 1925, + "ideo": 1926, + "36": 1927, + "_g": 1928, + "AG": 1929, + "valid": 1930, + "img": 1931, + "PI": 1932, + "Ġcolor": 1933, + "Ġreport": 1934, + "Ġtake": 1935, + "rib": 1936, + "OM": 1937, + "Ġday": 1938, + "Request": 1939, + "Ġsk": 1940, + "bers": 1941, + "ĉs": 1942, + ".Add": 1943, + "oot": 1944, + "Image": 1945, + "Ġcomple": 1946, + "ollection": 1947, + "Ġtop": 1948, + "Ġfree": 1949, + "AS": 1950, + "De": 1951, + "ĠOn": 1952, + "IG": 1953, + "90": 1954, + "eta": 1955, + "Date": 1956, + "Ġaction": 1957, + "34": 1958, + "Over": 1959, + "itor": 1960, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 1961, + "not": 1962, + "Ġindex": 1963, + "her": 1964, + "icon": 1965, + "On": 1966, + ";čĊčĊ": 1967, + "ivity": 1968, + "mand": 1969, + ".Windows": 1970, + "OL": 1971, + "Ġreal": 1972, + "Ġmax": 1973, + "land": 1974, + "....": 1975, + "raph": 1976, + "Ġbuild": 1977, + "leg": 1978, + "assword": 1979, + "?ĊĊ": 1980, + "â̦": 1981, + "ook": 1982, + "uck": 1983, + "Ġmessage": 1984, + "test": 1985, + "ivers": 1986, + "38": 1987, + "Ġinput": 1988, + "Ġart": 1989, + "Ġbetween": 1990, + "Get": 1991, + "enter": 1992, + "ground": 1993, + "ene": 1994, + "á": 1995, + ".length": 1996, + "Node": 1997, + "(i": 1998, + "Class": 1999, + "for": 2000, + "ĠâĢĶ": 2001, + "ten": 2002, + "oin": 2003, + "Ġke": 2004, + "ui": 2005, + "ĠIN": 2006, + "Ġtable": 2007, + "sub": 2008, + "ĠLe": 2009, + "Ġhead": 2010, + "Ġmust": 2011, + "////////////////": 2012, + ".util": 2013, + "Context": 2014, + "Ġorder": 2015, + "Ġmov": 2016, + "over": 2017, + "Ġcontin": 2018, + "Ġsay": 2019, + "static": 2020, + ".Text": 2021, + "ĠclassName": 2022, + "pany": 2023, + "Ġter": 2024, + "head": 2025, + "rg": 2026, + "Ġproduct": 2027, + "This": 2028, + ".âĢĿ": 2029, + "ĠBut": 2030, + "70": 2031, + "loy": 2032, + "Ġdouble": 2033, + "sg": 2034, + "Ġplace": 2035, + ".x": 2036, + "message": 2037, + "Ġinformation": 2038, + "private": 2039, + "Ġoper": 2040, + "ced": 2041, + "db": 2042, + "\">": 2043, + "Param": 2044, + "icle": 2045, + "Ġweek": 2046, + "Ġprop": 2047, + "table": 2048, + "idget": 2049, + "place": 2050, + "Prop": 2051, + "ĠAll": 2052, + "els": 2053, + "box": 2054, + ".ĊĊĊĊ": 2055, + ".R": 2056, + "ĠTo": 2057, + "iter": 2058, + "Sh": 2059, + "uration": 2060, + "older": 2061, + "_list": 2062, + "come": 2063, + "Ġsw": 2064, + "ization": 2065, + "ĉfor": 2066, + "bl": 2067, + "Ġprogram": 2068, + "(e": 2069, + "ape": 2070, + "check": 2071, + ".Forms": 2072, + "Ġund": 2073, + "ategory": 2074, + "75": 2075, + "ags": 2076, + "Ġresponse": 2077, + "US": 2078, + "request": 2079, + "Ġstruct": 2080, + "escription": 2081, + "Ġcode": 2082, + "_H": 2083, + "uffer": 2084, + "Ġwithout": 2085, + "lobal": 2086, + "Manager": 2087, + "ilter": 2088, + "PO": 2089, + "ĉthis": 2090, + "option": 2091, + "Ġsol": 2092, + "Ġ===": 2093, + "akes": 2094, + "Controller": 2095, + "44": 2096, + "Message": 2097, + "Ġref": 2098, + "ever": 2099, + "ĠSo": 2100, + "aining": 2101, + ".append": 2102, + "Ġstill": 2103, + "Ġprovid": 2104, + "Ġassert": 2105, + "med": 2106, + "Ġcap": 2107, + "usiness": 2108, + "Ġrep": 2109, + "tings": 2110, + "ved": 2111, + ".N": 2112, + "api": 2113, + "OD": 2114, + "Ġfield": 2115, + "iven": 2116, + "oto": 2117, + "âĢľ": 2118, + "col": 2119, + "(x": 2120, + "ght": 2121, + "Result": 2122, + "Code": 2123, + ".is": 2124, + "link": 2125, + "Ġcour": 2126, + "An": 2127, + "Ġteam": 2128, + "ĉint": 2129, + "ift": 2130, + "55": 2131, + "Ġsecond": 2132, + "Ġgoing": 2133, + "Ġrange": 2134, + "_E": 2135, + "ness": 2136, + "39": 2137, + "Ġfam": 2138, + "Ġnil": 2139, + "ĠCont": 2140, + "ailable": 2141, + "utes": 2142, + "atab": 2143, + "Ġfact": 2144, + "Ġvis": 2145, + "(&": 2146, + "ĠAN": 2147, + "31": 2148, + "Al": 2149, + "title": 2150, + "Ġandroid": 2151, + "CE": 2152, + "\\\"": 2153, + "irt": 2154, + "Ġwrit": 2155, + "н": 2156, + "ĉm": 2157, + "ftware": 2158, + "ond": 2159, + "Ġret": 2160, + "osition": 2161, + "Ġhome": 2162, + "Ġleft": 2163, + "args": 2164, + "meric": 2165, + "48": 2166, + "Ġdirect": 2167, + "oci": 2168, + "Pl": 2169, + "As": 2170, + "ret": 2171, + "ado": 2172, + "Of": 2173, + "chn": 2174, + "ĠGet": 2175, + "ee": 2176, + "ross": 2177, + "();": 2178, + "____": 2179, + ".ph": 2180, + "It": 2181, + "oute": 2182, + "Ġexper": 2183, + "chool": 2184, + "www": 2185, + "},": 2186, + "Ġallow": 2187, + "ĠÂ": 2188, + "())": 2189, + "size": 2190, + "ism": 2191, + "ai": 2192, + "tract": 2193, + "ane": 2194, + "...ĊĊ": 2195, + "context": 2196, + "Ġbeg": 2197, + "CH": 2198, + "Ġpage": 2199, + "hip": 2200, + "no": 2201, + "core": 2202, + "sp": 2203, + "Ġdifferent": 2204, + "iable": 2205, + "ĠMe": 2206, + "_IN": 2207, + "button": 2208, + "ĠIs": 2209, + "ervices": 2210, + "Ġca": 2211, + "Ġaround": 2212, + "App": 2213, + "ration": 2214, + "Ġrece": 2215, + "Ġreally": 2216, + "Ġimage": 2217, + "Ġtarget": 2218, + "Ġdep": 2219, + "opyright": 2220, + "tra": 2221, + "ingle": 2222, + "ital": 2223, + "Layout": 2224, + "Ġboth": 2225, + "Override": 2226, + "arm": 2227, + "=>": 2228, + "aterial": 2229, + "iled": 2230, + "Ġput": 2231, + "Qu": 2232, + "ÑĢ": 2233, + "ung": 2234, + "map": 2235, + "ĉĉĉĉĉĉĉĉ": 2236, + "Ġlevel": 2237, + "Component": 2238, + "book": 2239, + "creen": 2240, + "_RE": 2241, + "Ġconfig": 2242, + "ãģ": 2243, + "Or": 2244, + ".data": 2245, + "Ġdocument": 2246, + "\",\"": 2247, + "tribute": 2248, + "ux": 2249, + "Log": 2250, + "ference": 2251, + "post": 2252, + "_e": 2253, + "Ġlocal": 2254, + "andom": 2255, + "assert": 2256, + "Val": 2257, + "lected": 2258, + "ina": 2259, + "atabase": 2260, + "Add": 2261, + "Ġcontent": 2262, + ".print": 2263, + "signed": 2264, + "ric": 2265, + ".\"ĊĊ": 2266, + "Ġfa": 2267, + "!ĊĊ": 2268, + "-f": 2269, + "ived": 2270, + "Ġquest": 2271, + ".ex": 2272, + "Ġfloat": 2273, + "Ġdevelop": 2274, + "оÐ": 2275, + "Map": 2276, + "ading": 2277, + "Ġposs": 2278, + "UE": 2279, + "namespace": 2280, + "_O": 2281, + "ĉb": 2282, + ".Get": 2283, + ">(": 2284, + "json": 2285, + "etails": 2286, + "66": 2287, + "Ġtoo": 2288, + "Ġextends": 2289, + "ĠNone": 2290, + "Ġfore": 2291, + "(String": 2292, + "format": 2293, + "Ġgreat": 2294, + "inter": 2295, + "cale": 2296, + "Ñģ": 2297, + "ron": 2298, + "iving": 2299, + "Ent": 2300, + "ency": 2301, + "xt": 2302, + "oy": 2303, + "05": 2304, + "Ġmonth": 2305, + "Ġhapp": 2306, + "Ġsuper": 2307, + "bar": 2308, + "default": 2309, + "_de": 2310, + "ords": 2311, + "ln": 2312, + "({Ċ": 2313, + "ĠInd": 2314, + "ases": 2315, + "Ġtitle": 2316, + "Ġcontext": 2317, + "08": 2318, + "oh": 2319, + "-p": 2320, + "Em": 2321, + "Ġmet": 2322, + "Test": 2323, + "Ġlife": 2324, + "_v": 2325, + "ĠUS": 2326, + "UI": 2327, + "ocation": 2328, + "md": 2329, + "Ġ[Ċ": 2330, + "Ġ]": 2331, + "sw": 2332, + "Ġincre": 2333, + "script": 2334, + "ential": 2335, + "ways": 2336, + ".de": 2337, + "Ġsrc": 2338, + "Ġcatch": 2339, + "ĠAmeric": 2340, + "//Ċ": 2341, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 2342, + "Ġpay": 2343, + "plit": 2344, + "âĢĶ": 2345, + "Ġcoun": 2346, + "obj": 2347, + ".php": 2348, + "Ġchange": 2349, + "ething": 2350, + "'re": 2351, + "aster": 2352, + "los": 2353, + "lation": 2354, + "ĠĠĊ": 2355, + "Le": 2356, + "ä": 2357, + "({": 2358, + "ready": 2359, + "ĠNo": 2360, + "Ġposition": 2361, + "Ġold": 2362, + "Ġbook": 2363, + "abled": 2364, + "bug": 2365, + "202": 2366, + "Hand": 2367, + "};ĊĊ": 2368, + "isplay": 2369, + "aving": 2370, + "04": 2371, + "Ġgover": 2372, + "Ġversion": 2373, + "System": 2374, + "nect": 2375, + "response": 2376, + "Style": 2377, + "Up": 2378, + "angu": 2379, + "Ġthree": 2380, + "init": 2381, + "ero": 2382, + "Ġlaw": 2383, + "endif": 2384, + "Ġbase": 2385, + "email": 2386, + "(l": 2387, + "_V": 2388, + "Ġconf": 2389, + "ATE": 2390, + "Ġduring": 2391, + "tes": 2392, + "Ġconsole": 2393, + "ĠPr": 2394, + "Ġspe": 2395, + "ves": 2396, + "65": 2397, + "path": 2398, + "ialog": 2399, + "dition": 2400, + "_to": 2401, + "ards": 2402, + "Ġagainst": 2403, + "etwork": 2404, + "ĠPh": 2405, + "_L": 2406, + "cur": 2407, + "imit": 2408, + "With": 2409, + "Ġpower": 2410, + "ium": 2411, + "';ĊĊ": 2412, + "Ġwom": 2413, + "left": 2414, + "ources": 2415, + "atri": 2416, + "ĠIm": 2417, + "ĠMan": 2418, + "orth": 2419, + "${": 2420, + "88": 2421, + "quals": 2422, + "ese": 2423, + "_size": 2424, + "Ġiss": 2425, + "otal": 2426, + "-g": 2427, + "ique": 2428, + "rame": 2429, + "Ġwidth": 2430, + "erg": 2431, + ")(": 2432, + "ittle": 2433, + "TR": 2434, + "ĠThey": 2435, + "ences": 2436, + "02": 2437, + "rl": 2438, + "ons": 2439, + "Ġlabel": 2440, + ".y": 2441, + "-t": 2442, + "update": 2443, + "anel": 2444, + "sc": 2445, + ".to": 2446, + "Ġproject": 2447, + "ü": 2448, + "Ġelement": 2449, + "Ġsuccess": 2450, + "ĉĉĊ": 2451, + ".sh": 2452, + "ram": 2453, + "ched": 2454, + "())Ċ": 2455, + "Ġ(Ċ": 2456, + "Ġdate": 2457, + "Ġtot": 2458, + "_ST": 2459, + "All": 2460, + "ification": 2461, + "ĉvar": 2462, + "Ġtri": 2463, + "chem": 2464, + "my": 2465, + "Ġbig": 2466, + "ĠAd": 2467, + "ĠAt": 2468, + "ots": 2469, + "num": 2470, + "Act": 2471, + "Ġmap": 2472, + "era": 2473, + "cope": 2474, + ".$": 2475, + ",âĢĿ": 2476, + "Ġpop": 2477, + "Ġfew": 2478, + "Ġlen": 2479, + "uid": 2480, + "eters": 2481, + "ules": 2482, + "ÃŃ": 2483, + "source": 2484, + "https": 2485, + "Ġdem": 2486, + "Ġear": 2487, + "################": 2488, + "Ġmatch": 2489, + "ories": 2490, + "49": 2491, + "aces": 2492, + "ĠCl": 2493, + "Ġnode": 2494, + "78": 2495, + "irc": 2496, + "local": 2497, + "unity": 2498, + "};Ċ": 2499, + "Ġanother": 2500, + "<<": 2501, + "ogle": 2502, + "Ġsit": 2503, + "ework": 2504, + "TE": 2505, + ".I": 2506, + "NS": 2507, + "ology": 2508, + "ought": 2509, + ".Cont": 2510, + ">>": 2511, + "Ġcare": 2512, + "state": 2513, + "ĉprivate": 2514, + "Ġeffect": 2515, + "++)": 2516, + "_file": 2517, + "ending": 2518, + "Line": 2519, + "For": 2520, + "ior": 2521, + "ĠSc": 2522, + "Ġfun": 2523, + ".Size": 2524, + "ĉelse": 2525, + "])": 2526, + "start": 2527, + "vious": 2528, + "Ġ},": 2529, + "ours": 2530, + "Ġleg": 2531, + "Ġservice": 2532, + "Ġsince": 2533, + "iron": 2534, + "Label": 2535, + "Ġnon": 2536, + "Ġlos": 2537, + "iction": 2538, + "Ġfull": 2539, + "acter": 2540, + "board": 2541, + "gress": 2542, + "Ġturn": 2543, + "ither": 2544, + "09": 2545, + ".size": 2546, + "Ġbody": 2547, + "resh": 2548, + "eturn": 2549, + "199": 2550, + "(_": 2551, + "yles": 2552, + "ormal": 2553, + "pi": 2554, + "Ġsomething": 2555, + "!--": 2556, + "uint": 2557, + "Ġprodu": 2558, + "Ġstand": 2559, + "Ġproble": 2560, + "Ġavailable": 2561, + "mt": 2562, + "ĠBl": 2563, + "Ġ...": 2564, + "Ġblock": 2565, + "Input": 2566, + "Ġkeep": 2567, + "Count": 2568, + "open": 2569, + "Ġ['": 2570, + "Ġthrow": 2571, + "uilder": 2572, + "Action": 2573, + "Ġthings": 2574, + "True": 2575, + "Ġurl": 2576, + "ĠBo": 2577, + "printf": 2578, + "Ġred": 2579, + "js": 2580, + ".create": 2581, + "ĠOr": 2582, + "Status": 2583, + "Instance": 2584, + "Ġcontrol": 2585, + "Ġcome": 2586, + "Ġcustom": 2587, + "location": 2588, + "07": 2589, + "model": 2590, + "ĠčĊ": 2591, + "Ġsource": 2592, + "Ġeas": 2593, + ".out": 2594, + "]ĊĊ": 2595, + "oney": 2596, + "Ġawait": 2597, + "Ġpartic": 2598, + "AP": 2599, + "ublish": 2600, + "odes": 2601, + "_pro": 2602, + "ply": 2603, + "riter": 2604, + "Ġprov": 2605, + "Ġmill": 2606, + "HT": 2607, + "])Ċ": 2608, + "Ġchang": 2609, + "Ġask": 2610, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 2611, + "Ġoutput": 2612, + "Ġemail": 2613, + "68": 2614, + ".push": 2615, + "Ġ}čĊčĊ": 2616, + "ination": 2617, + "47": 2618, + "atrix": 2619, + "Table": 2620, + "uccess": 2621, + "]);Ċ": 2622, + "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 2623, + "Ġdisc": 2624, + "([": 2625, + "Ġbusiness": 2626, + "height": 2627, + ".html": 2628, + "ta": 2629, + "field": 2630, + "Ġrequired": 2631, + "_R": 2632, + "Ġgovern": 2633, + "}čĊčĊ": 2634, + "lex": 2635, + "500": 2636, + ".,": 2637, + "ĠSet": 2638, + "urch": 2639, + "///": 2640, + "ts": 2641, + "af": 2642, + "Ġmight": 2643, + "istory": 2644, + "Str": 2645, + "Ġnever": 2646, + "Response": 2647, + "arse": 2648, + "ada": 2649, + "ĠHow": 2650, + "Ġ*)": 2651, + "Ġ;": 2652, + "Ġhard": 2653, + "Ad": 2654, + "Ġintern": 2655, + "used": 2656, + "(data": 2657, + "mod": 2658, + "annel": 2659, + "Ġnp": 2660, + "ugg": 2661, + "Ġ/>Ċ": 2662, + "Ġcalled": 2663, + "body": 2664, + "Ġcho": 2665, + "(r": 2666, + "_set": 2667, + "ird": 2668, + "Ġ>=": 2669, + "Ġ};Ċ": 2670, + "Ġoptions": 2671, + "ĠGener": 2672, + "Ġheight": 2673, + "Point": 2674, + "You": 2675, + "ety": 2676, + "Click": 2677, + "Ġsmall": 2678, + "Ġide": 2679, + "Ġaccess": 2680, + "anguage": 2681, + "Ġprotected": 2682, + "Ġjob": 2683, + "ĠThere": 2684, + "Def": 2685, + "Ġaddress": 2686, + "Ġuint": 2687, + "Not": 2688, + "oo": 2689, + "aps": 2690, + "
&": 5909,
+ "CON": 5910,
+ "Ġrepl": 5911,
+ "Ġregular": 5912,
+ "Storage": 5913,
+ "ramework": 5914,
+ "Ġgoal": 5915,
+ "Ġtouch": 5916,
+ ".widget": 5917,
+ "Ġbuilt": 5918,
+ "des": 5919,
+ "Part": 5920,
+ "(re": 5921,
+ "Ġworth": 5922,
+ "hib": 5923,
+ "game": 5924,
+ "91": 5925,
+ "192": 5926,
+ "Ġв": 5927,
+ "acion": 5928,
+ "ĠWhite": 5929,
+ "(type": 5930,
+ "(`": 5931,
+ "81": 5932,
+ "Ġnatural": 5933,
+ "Ġinj": 5934,
+ "Ġcalcul": 5935,
+ "ĠApril": 5936,
+ ".List": 5937,
+ "Ġassociated": 5938,
+ "ĉSystem": 5939,
+ "~~": 5940,
+ "=[": 5941,
+ "Ġstorage": 5942,
+ "Ġbytes": 5943,
+ "Ġtravel": 5944,
+ "Ġsou": 5945,
+ "Ġpassed": 5946,
+ "!=": 5947,
+ "ascript": 5948,
+ ".open": 5949,
+ "Ġgrid": 5950,
+ "Ġbus": 5951,
+ "Ġrecogn": 5952,
+ "Ab": 5953,
+ "Ġhon": 5954,
+ "ĠCenter": 5955,
+ "Ġprec": 5956,
+ "build": 5957,
+ "73": 5958,
+ "HTML": 5959,
+ "ĠSan": 5960,
+ "Ġcountries": 5961,
+ "aled": 5962,
+ "token": 5963,
+ "kt": 5964,
+ "Ġqual": 5965,
+ "Last": 5966,
+ "adow": 5967,
+ "Ġmanufact": 5968,
+ "idad": 5969,
+ "jango": 5970,
+ "Next": 5971,
+ "xf": 5972,
+ ".a": 5973,
+ "Ġporno": 5974,
+ "ĠPM": 5975,
+ "erve": 5976,
+ "iting": 5977,
+ "_th": 5978,
+ "ci": 5979,
+ "=None": 5980,
+ "gs": 5981,
+ "Ġlogin": 5982,
+ "atives": 5983,
+ "']);Ċ": 5984,
+ "Äħ": 5985,
+ "Ġill": 5986,
+ "IA": 5987,
+ "children": 5988,
+ "DO": 5989,
+ "Ġlevels": 5990,
+ "Ġ{{": 5991,
+ "Ġlooks": 5992,
+ "Ġ\"#": 5993,
+ "ToString": 5994,
+ "Ġnecessary": 5995,
+ "ĠĠĠĊ": 5996,
+ "cell": 5997,
+ "Entry": 5998,
+ "Ġ'#": 5999,
+ "Ġextrem": 6000,
+ "Selector": 6001,
+ "Ġplaceholder": 6002,
+ "Load": 6003,
+ "Ġreleased": 6004,
+ "ORE": 6005,
+ "Enumer": 6006,
+ "ĠTV": 6007,
+ "SET": 6008,
+ "inq": 6009,
+ "Press": 6010,
+ "ĠDepartment": 6011,
+ "Ġproperties": 6012,
+ "Ġrespond": 6013,
+ "Search": 6014,
+ "ael": 6015,
+ "Ġrequ": 6016,
+ "ĠBook": 6017,
+ "/Ċ": 6018,
+ "(st": 6019,
+ "Ġfinancial": 6020,
+ "icket": 6021,
+ "_input": 6022,
+ "Ġthreat": 6023,
+ "(in": 6024,
+ "Strip": 6025,
+ "ìĿ": 6026,
+ "ção": 6027,
+ "71": 6028,
+ "Ġevidence": 6029,
+ "));": 6030,
+ "ĠBro": 6031,
+ "Ġ[];Ċ": 6032,
+ "Ġou": 6033,
+ "buf": 6034,
+ "Script": 6035,
+ "dat": 6036,
+ "Ġrule": 6037,
+ "#import": 6038,
+ "=\"/": 6039,
+ "Serial": 6040,
+ "Ġstarting": 6041,
+ "[index": 6042,
+ "ae": 6043,
+ "Ġcontrib": 6044,
+ "session": 6045,
+ "_new": 6046,
+ "utable": 6047,
+ "ober": 6048,
+ "Ġ\"./": 6049,
+ "Ġlogger": 6050,
+ "Ġrecently": 6051,
+ "Ġreturned": 6052,
+ "ččĊ": 6053,
+ ")))Ċ": 6054,
+ "itions": 6055,
+ "Ġseek": 6056,
+ "Ġcommunic": 6057,
+ "Ġ\".": 6058,
+ "Ġusername": 6059,
+ "ECT": 6060,
+ "DS": 6061,
+ "Ġotherwise": 6062,
+ "ĠGerman": 6063,
+ ".aw": 6064,
+ "Adapter": 6065,
+ "ixel": 6066,
+ "Ġsystems": 6067,
+ "Ġdrop": 6068,
+ "83": 6069,
+ "Ġstructure": 6070,
+ "Ġ$(\"#": 6071,
+ "encies": 6072,
+ "anning": 6073,
+ "ĠLink": 6074,
+ "ĠResponse": 6075,
+ "Ġstri": 6076,
+ "ż": 6077,
+ "ĠDB": 6078,
+ "æĹ": 6079,
+ "android": 6080,
+ "submit": 6081,
+ "otion": 6082,
+ "92": 6083,
+ "(@": 6084,
+ ".test": 6085,
+ "82": 6086,
+ "ĊĊĊĊĊĊĊĊ": 6087,
+ "];čĊ": 6088,
+ "Ġdirectly": 6089,
+ "Ġ\"%": 6090,
+ "ris": 6091,
+ "elta": 6092,
+ "AIL": 6093,
+ "){čĊ": 6094,
+ "mine": 6095,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 6096,
+ "(k": 6097,
+ "bon": 6098,
+ "asic": 6099,
+ "pite": 6100,
+ "___": 6101,
+ "Max": 6102,
+ "Ġerrors": 6103,
+ "ĠWhile": 6104,
+ "Ġarguments": 6105,
+ "Ġensure": 6106,
+ "Right": 6107,
+ "-based": 6108,
+ "Web": 6109,
+ "Ġ-=": 6110,
+ "Ġintrodu": 6111,
+ "ĠInst": 6112,
+ "ĠWash": 6113,
+ "ordin": 6114,
+ "join": 6115,
+ "Database": 6116,
+ "Ġgrad": 6117,
+ "Ġusually": 6118,
+ "ITE": 6119,
+ "Props": 6120,
+ "?>Ċ": 6121,
+ "ĠGo": 6122,
+ "@Override": 6123,
+ "REF": 6124,
+ "Ġip": 6125,
+ "ĠAustral": 6126,
+ "Ġist": 6127,
+ "ViewById": 6128,
+ "Ġserious": 6129,
+ "Ġcustomer": 6130,
+ ".prototype": 6131,
+ "odo": 6132,
+ "cor": 6133,
+ "Ġdoor": 6134,
+ "ĠWITHOUT": 6135,
+ "Ġplant": 6136,
+ "Ġbegan": 6137,
+ "Ġdistance": 6138,
+ "()).": 6139,
+ "Ġchance": 6140,
+ "Ġord": 6141,
+ "came": 6142,
+ "pragma": 6143,
+ "Ġprotect": 6144,
+ "ragment": 6145,
+ "ĠNode": 6146,
+ "ening": 6147,
+ "Ñĩ": 6148,
+ "Ġroute": 6149,
+ "ĠSchool": 6150,
+ "hi": 6151,
+ "Ġneighb": 6152,
+ "After": 6153,
+ "licit": 6154,
+ "Ġcontr": 6155,
+ "Ġprimary": 6156,
+ "AA": 6157,
+ ".WriteLine": 6158,
+ "utils": 6159,
+ "Ġbi": 6160,
+ "Red": 6161,
+ ".Linq": 6162,
+ ".object": 6163,
+ "Ġleaders": 6164,
+ "unities": 6165,
+ "Ġgun": 6166,
+ "onth": 6167,
+ "ĠDev": 6168,
+ "FILE": 6169,
+ "Ġcomments": 6170,
+ "_len": 6171,
+ "arrow": 6172,
+ "amount": 6173,
+ "Range": 6174,
+ "sert": 6175,
+ "GridView": 6176,
+ "Ġupdated": 6177,
+ "ĠMo": 6178,
+ "Ġinform": 6179,
+ "ociety": 6180,
+ "ala": 6181,
+ "Access": 6182,
+ "Ġhab": 6183,
+ "Ġcreat": 6184,
+ "_arg": 6185,
+ "ĠJanuary": 6186,
+ "ĠDay": 6187,
+ "\")čĊ": 6188,
+ "uple": 6189,
+ "document": 6190,
+ "gorith": 6191,
+ "menu": 6192,
+ "ĠOver": 6193,
+ "bb": 6194,
+ ".title": 6195,
+ "_out": 6196,
+ "Ġled": 6197,
+ "uri": 6198,
+ "Ġ?>": 6199,
+ "gl": 6200,
+ "Ġbank": 6201,
+ "ayment": 6202,
+ "ĉprintf": 6203,
+ "MD": 6204,
+ "Ġsample": 6205,
+ "Ġhands": 6206,
+ "ĠVersion": 6207,
+ "uario": 6208,
+ "Ġoffers": 6209,
+ "ityEngine": 6210,
+ "Ġshape": 6211,
+ "Ġsleep": 6212,
+ "_point": 6213,
+ "Settings": 6214,
+ "Ġachie": 6215,
+ "Ġsold": 6216,
+ "ota": 6217,
+ ".bind": 6218,
+ "Am": 6219,
+ "Ġsafe": 6220,
+ "Store": 6221,
+ "Ġshared": 6222,
+ "Ġpriv": 6223,
+ "_VAL": 6224,
+ "Ġsens": 6225,
+ "){": 6226,
+ "Ġremember": 6227,
+ "shared": 6228,
+ "element": 6229,
+ "Ġshoot": 6230,
+ "Vert": 6231,
+ "cout": 6232,
+ "Ġenv": 6233,
+ "_label": 6234,
+ "Ġ>Ċ": 6235,
+ "run": 6236,
+ "Ġscene": 6237,
+ "(array": 6238,
+ "device": 6239,
+ "_title": 6240,
+ "agon": 6241,
+ "]čĊ": 6242,
+ "aby": 6243,
+ "Ġbecame": 6244,
+ "boolean": 6245,
+ "Ġpark": 6246,
+ "ĠCode": 6247,
+ "upload": 6248,
+ "riday": 6249,
+ "ĠSeptember": 6250,
+ "Fe": 6251,
+ "Ġsen": 6252,
+ "cing": 6253,
+ "FL": 6254,
+ "Col": 6255,
+ "uts": 6256,
+ "_page": 6257,
+ "inn": 6258,
+ "Ġimplied": 6259,
+ "aling": 6260,
+ "Ġyourself": 6261,
+ ".Count": 6262,
+ "conf": 6263,
+ "Ġaud": 6264,
+ "_init": 6265,
+ ".)": 6266,
+ "Ġwrote": 6267,
+ "003": 6268,
+ "NG": 6269,
+ ".Error": 6270,
+ "ä»": 6271,
+ ".for": 6272,
+ "Ġequal": 6273,
+ "ĠRequest": 6274,
+ "Ġserial": 6275,
+ "Ġallows": 6276,
+ "XX": 6277,
+ "Ġmiddle": 6278,
+ "chor": 6279,
+ "195": 6280,
+ "94": 6281,
+ "ø": 6282,
+ "erval": 6283,
+ ".Column": 6284,
+ "reading": 6285,
+ "Ġescort": 6286,
+ "ĠAugust": 6287,
+ "Ġquickly": 6288,
+ "Ġweap": 6289,
+ "ĠCG": 6290,
+ "ropri": 6291,
+ "ho": 6292,
+ "Ġcop": 6293,
+ "(struct": 6294,
+ "ĠBig": 6295,
+ "Ġvs": 6296,
+ "Ġfrequ": 6297,
+ ".Value": 6298,
+ "Ġactions": 6299,
+ "Ġproper": 6300,
+ "Ġinn": 6301,
+ "Ġobjects": 6302,
+ "Ġmatrix": 6303,
+ "avascript": 6304,
+ "Ġones": 6305,
+ ".group": 6306,
+ "Ġgreen": 6307,
+ "Ġpaint": 6308,
+ "ools": 6309,
+ "ycl": 6310,
+ "encode": 6311,
+ "olt": 6312,
+ "comment": 6313,
+ ".api": 6314,
+ "Dir": 6315,
+ "Ġune": 6316,
+ "izont": 6317,
+ ".position": 6318,
+ "Ġdesigned": 6319,
+ "_val": 6320,
+ "avi": 6321,
+ "iring": 6322,
+ "tab": 6323,
+ "Ġlayer": 6324,
+ "Ġviews": 6325,
+ "Ġreve": 6326,
+ "rael": 6327,
+ "ĠON": 6328,
+ "rics": 6329,
+ "160": 6330,
+ "np": 6331,
+ "Ġcore": 6332,
+ "());čĊ": 6333,
+ "Main": 6334,
+ "Ġexpert": 6335,
+ "ĉĉčĊ": 6336,
+ "_en": 6337,
+ "Ġ/>": 6338,
+ "utter": 6339,
+ "IAL": 6340,
+ "ails": 6341,
+ "ĠKing": 6342,
+ "*/ĊĊ": 6343,
+ "ĠMet": 6344,
+ "_end": 6345,
+ "addr": 6346,
+ "ora": 6347,
+ "Ġir": 6348,
+ "Min": 6349,
+ "Ġsurpr": 6350,
+ "Ġrepe": 6351,
+ "Ġdirectory": 6352,
+ "PUT": 6353,
+ "-S": 6354,
+ "Ġelection": 6355,
+ "haps": 6356,
+ ".pre": 6357,
+ "cm": 6358,
+ "Values": 6359,
+ "Ġ\"Ċ": 6360,
+ "column": 6361,
+ "ivil": 6362,
+ "Login": 6363,
+ "inue": 6364,
+ "93": 6365,
+ "Ġbeautiful": 6366,
+ "Ġsecret": 6367,
+ "(event": 6368,
+ "Ġchat": 6369,
+ "ums": 6370,
+ "Ġorigin": 6371,
+ "Ġeffects": 6372,
+ "Ġmanagement": 6373,
+ "illa": 6374,
+ "tk": 6375,
+ "Ġsetting": 6376,
+ "ĠCour": 6377,
+ "Ġmassage": 6378,
+ "ĉend": 6379,
+ "Ġhappy": 6380,
+ "Ġfinish": 6381,
+ "Ġcamera": 6382,
+ "ĠVer": 6383,
+ "ĠDemocr": 6384,
+ "ĠHer": 6385,
+ "(Q": 6386,
+ "cons": 6387,
+ "ita": 6388,
+ "Ġ'.": 6389,
+ "{}": 6390,
+ "ĉC": 6391,
+ "Ġstuff": 6392,
+ "194": 6393,
+ "Ġ:Ċ": 6394,
+ "ĠAR": 6395,
+ "Task": 6396,
+ "hidden": 6397,
+ "eros": 6398,
+ "IGN": 6399,
+ "atio": 6400,
+ "ĠHealth": 6401,
+ "olute": 6402,
+ "Enter": 6403,
+ "'>": 6404,
+ "ĠTwitter": 6405,
+ "ĠCounty": 6406,
+ "scribe": 6407,
+ "Ġ=>Ċ": 6408,
+ "Ġhy": 6409,
+ "fit": 6410,
+ "Ġmilitary": 6411,
+ "Ġsale": 6412,
+ "required": 6413,
+ "non": 6414,
+ "bootstrap": 6415,
+ "hold": 6416,
+ "rim": 6417,
+ "-old": 6418,
+ "ĠDown": 6419,
+ "Ġmention": 6420,
+ "contact": 6421,
+ "_group": 6422,
+ "oday": 6423,
+ "Ġtown": 6424,
+ "Ġsolution": 6425,
+ "uate": 6426,
+ "elling": 6427,
+ "]->": 6428,
+ "otes": 6429,
+ "ental": 6430,
+ "omen": 6431,
+ "ospital": 6432,
+ "ĠSup": 6433,
+ "_EN": 6434,
+ "Ġslow": 6435,
+ "SESSION": 6436,
+ "Ġblue": 6437,
+ "ago": 6438,
+ "Ġlives": 6439,
+ "Ġ^": 6440,
+ ".un": 6441,
+ "inst": 6442,
+ "enge": 6443,
+ "Ġcustomers": 6444,
+ "Ġcast": 6445,
+ "udget": 6446,
+ "ï¼ģ": 6447,
+ "icens": 6448,
+ "Ġdetermin": 6449,
+ "Selected": 6450,
+ "_pl": 6451,
+ "ueue": 6452,
+ "Ġdark": 6453,
+ "//ĊĊ": 6454,
+ "si": 6455,
+ "thern": 6456,
+ "ĠJapan": 6457,
+ "/w": 6458,
+ "PU": 6459,
+ "ĠEast": 6460,
+ "ovie": 6461,
+ "Ġpackage": 6462,
+ "Ġnor": 6463,
+ "Ġapi": 6464,
+ "bot": 6465,
+ "\"];Ċ": 6466,
+ "_post": 6467,
+ "ulate": 6468,
+ "Ġclub": 6469,
+ "'));Ċ": 6470,
+ "Ġloop": 6471,
+ "PIO": 6472,
+ "ione": 6473,
+ "shot": 6474,
+ "Initial": 6475,
+ "Ġplayed": 6476,
+ "register": 6477,
+ "rought": 6478,
+ "_max": 6479,
+ "acement": 6480,
+ "match": 6481,
+ "raphics": 6482,
+ "AST": 6483,
+ "Ġexisting": 6484,
+ "Ġcomplex": 6485,
+ "DA": 6486,
+ ".Ch": 6487,
+ ".common": 6488,
+ "mo": 6489,
+ "Ġ'../../": 6490,
+ "ito": 6491,
+ "Ġanalysis": 6492,
+ "Ġdeliver": 6493,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ": 6494,
+ "idx": 6495,
+ "Ãł": 6496,
+ "ongo": 6497,
+ "ĠEnglish": 6498,
+ "Ċ": 10197,
+ "_default": 10198,
+ "ĠDatabase": 10199,
+ "rep": 10200,
+ "ESS": 10201,
+ "nergy": 10202,
+ ".Find": 10203,
+ "_mask": 10204,
+ "Ġrise": 10205,
+ "Ġkernel": 10206,
+ "::$": 10207,
+ ".Q": 10208,
+ "Ġoffering": 10209,
+ "decl": 10210,
+ "ĠCS": 10211,
+ "Ġlisted": 10212,
+ "Ġmostly": 10213,
+ "enger": 10214,
+ "Ġblocks": 10215,
+ "olo": 10216,
+ "Ġgoverning": 10217,
+ "\\F": 10218,
+ "Ġconcent": 10219,
+ ".getText": 10220,
+ "Ġmb": 10221,
+ "Ġoccurred": 10222,
+ "Ġchanging": 10223,
+ "Scene": 10224,
+ "_CODE": 10225,
+ "Beh": 10226,
+ "\"The": 10227,
+ "Ġtile": 10228,
+ "ĠAssociation": 10229,
+ "ĉP": 10230,
+ "alty": 10231,
+ "_ad": 10232,
+ "odies": 10233,
+ "iated": 10234,
+ "Ġprepared": 10235,
+ "possible": 10236,
+ "Ġmort": 10237,
+ "TEST": 10238,
+ "142": 10239,
+ "Ġignore": 10240,
+ "Ġcalc": 10241,
+ "Ġrs": 10242,
+ "ĠassertEquals": 10243,
+ "Ġsz": 10244,
+ "ĠTHIS": 10245,
+ ".\"Ċ": 10246,
+ "Ġcanvas": 10247,
+ "java": 10248,
+ "Ġdut": 10249,
+ "VALID": 10250,
+ ".sql": 10251,
+ ".input": 10252,
+ "Ġaux": 10253,
+ "Sup": 10254,
+ "Ġartist": 10255,
+ "Vec": 10256,
+ "_TIME": 10257,
+ ".stringify": 10258,
+ "etween": 10259,
+ "ĠCategory": 10260,
+ "Ġ[-": 10261,
+ "ĠDevExpress": 10262,
+ "ĠJul": 10263,
+ "Ġring": 10264,
+ ".ed": 10265,
+ "YY": 10266,
+ "Let": 10267,
+ "TextField": 10268,
+ "Ġflat": 10269,
+ "_print": 10270,
+ "ĠOTHER": 10271,
+ "adian": 10272,
+ "Ġchecked": 10273,
+ "ele": 10274,
+ "Align": 10275,
+ "standing": 10276,
+ "Ġ[],": 10277,
+ "Ġlab": 10278,
+ "ucky": 10279,
+ "ĠChristmas": 10280,
+ "(image": 10281,
+ ".module": 10282,
+ "Ġlots": 10283,
+ "Ġslightly": 10284,
+ "(final": 10285,
+ "erge": 10286,
+ "è¿": 10287,
+ "147": 10288,
+ "ĠPolice": 10289,
+ "143": 10290,
+ "ĠRight": 10291,
+ "Ġaward": 10292,
+ "ĠOS": 10293,
+ "Ġ{}ĊĊ": 10294,
+ "Ġptr": 10295,
+ "oves": 10296,
+ "icated": 10297,
+ "ем": 10298,
+ "Ġmanage": 10299,
+ "oliday": 10300,
+ "Amount": 10301,
+ "oolStrip": 10302,
+ "tbody": 10303,
+ "Nav": 10304,
+ "wrap": 10305,
+ "BB": 10306,
+ "Ġwatching": 10307,
+ "arios": 10308,
+ "Ġoptional": 10309,
+ "_K": 10310,
+ "ĠLicensed": 10311,
+ ".Map": 10312,
+ "Timer": 10313,
+ "ĠAP": 10314,
+ "ĠRev": 10315,
+ "(o": 10316,
+ ",c": 10317,
+ "umin": 10318,
+ "etailed": 10319,
+ "ĠHy": 10320,
+ "Ġblank": 10321,
+ "agger": 10322,
+ "ĠSelf": 10323,
+ "()[": 10324,
+ ".make": 10325,
+ "earn": 10326,
+ "channel": 10327,
+ ";Ċ": 10342,
+ "World": 10343,
+ "Ġpython": 10344,
+ "Ġlif": 10345,
+ "Ġtrav": 10346,
+ "Ġconven": 10347,
+ "company": 10348,
+ "ĠClub": 10349,
+ "138": 10350,
+ "Ver": 10351,
+ "Btn": 10352,
+ "Ġzone": 10353,
+ "products": 10354,
+ "ĠEduc": 10355,
+ "Ġverify": 10356,
+ "ĠMil": 10357,
+ "ono": 10358,
+ "]);ĊĊ": 10359,
+ "ENCE": 10360,
+ "Ġpacket": 10361,
+ "Ġcer": 10362,
+ "Ġenumer": 10363,
+ "Ġpars": 10364,
+ "formed": 10365,
+ "Ġoccup": 10366,
+ "tre": 10367,
+ "Ġexercise": 10368,
+ "Day": 10369,
+ "_sum": 10370,
+ "Ġasking": 10371,
+ "aption": 10372,
+ "Ġorders": 10373,
+ "Ġspending": 10374,
+ "ĠERR": 10375,
+ ".Dis": 10376,
+ "ĠUtil": 10377,
+ "âĢľI": 10378,
+ "\\'": 10379,
+ "?)": 10380,
+ "/>Ċ": 10381,
+ "Ġemot": 10382,
+ "Ġinfluence": 10383,
+ "ĠAfrica": 10384,
+ "atters": 10385,
+ "Ùħ": 10386,
+ ".session": 10387,
+ "Ġchief": 10388,
+ "ĉĉĉĉĉĉĉĉĉĉĉ": 10389,
+ "Ġtom": 10390,
+ "cluded": 10391,
+ "serial": 10392,
+ "_handler": 10393,
+ ".Type": 10394,
+ "aped": 10395,
+ "Ġpolicies": 10396,
+ "-ex": 10397,
+ "-tr": 10398,
+ "blank": 10399,
+ "merce": 10400,
+ "Ġcoverage": 10401,
+ "Ġrc": 10402,
+ "_matrix": 10403,
+ "_box": 10404,
+ "Ġcharges": 10405,
+ "ĠBoston": 10406,
+ "Pe": 10407,
+ "Ġcircum": 10408,
+ "Ġfilled": 10409,
+ "148": 10410,
+ "Ġnorth": 10411,
+ "ictureBox": 10412,
+ "ĉres": 10413,
+ "è®": 10414,
+ "Ġtermin": 10415,
+ "Ġ[â̦": 10416,
+ "IRECT": 10417,
+ "Ġber": 10418,
+ "Ġ\"../../": 10419,
+ "retch": 10420,
+ ".code": 10421,
+ "_col": 10422,
+ "ĠGovernment": 10423,
+ "Ġargv": 10424,
+ "ĠLord": 10425,
+ "asi": 10426,
+ "Exec": 10427,
+ "ĉlet": 10428,
+ "vertis": 10429,
+ "Ġdiscussion": 10430,
+ "enance": 10431,
+ "outube": 10432,
+ "typeof": 10433,
+ "Ġserved": 10434,
+ "ĠPut": 10435,
+ "ĉx": 10436,
+ "Ġsweet": 10437,
+ "Before": 10438,
+ "ategy": 10439,
+ ".of": 10440,
+ "ĠMaterial": 10441,
+ "Sort": 10442,
+ "ONT": 10443,
+ "igital": 10444,
+ "Why": 10445,
+ "Ġsust": 10446,
+ "Ġç": 10447,
+ "abet": 10448,
+ "Ġsegment": 10449,
+ "Ġ[],Ċ": 10450,
+ "ĠMuslim": 10451,
+ "ĠfindViewById": 10452,
+ "cut": 10453,
+ "_TEXT": 10454,
+ "ĠMary": 10455,
+ "Ġloved": 10456,
+ "Ġlie": 10457,
+ "ĠJO": 10458,
+ "Ġisset": 10459,
+ "month": 10460,
+ "Ġprime": 10461,
+ "ti": 10462,
+ "ĠCarol": 10463,
+ "Use": 10464,
+ "146": 10465,
+ "ĠPop": 10466,
+ "ĠSave": 10467,
+ "Interval": 10468,
+ "execute": 10469,
+ "dy": 10470,
+ "ĠIran": 10471,
+ "_cont": 10472,
+ "ĉT": 10473,
+ "Ġphase": 10474,
+ "checkbox": 10475,
+ "week": 10476,
+ "Ġhide": 10477,
+ "Ġtil": 10478,
+ "Ġju": 10479,
+ "Custom": 10480,
+ "burg": 10481,
+ "/M": 10482,
+ "TON": 10483,
+ "Ġquant": 10484,
+ "Ġrub": 10485,
+ "ixels": 10486,
+ "Ġinstalled": 10487,
+ "Ġdump": 10488,
+ "Ġproperly": 10489,
+ "(List": 10490,
+ "Ġdecide": 10491,
+ "apply": 10492,
+ "Has": 10493,
+ "Ġkeeping": 10494,
+ "Ġcitizens": 10495,
+ "Ġjoint": 10496,
+ "pool": 10497,
+ "Socket": 10498,
+ "_op": 10499,
+ "Ġweapon": 10500,
+ "gnore": 10501,
+ "ĠExec": 10502,
+ "otten": 10503,
+ "ĠMS": 10504,
+ "Ġ(-": 10505,
+ "ĠReview": 10506,
+ "Ġexamples": 10507,
+ "Ġtight": 10508,
+ "!(": 10509,
+ "DP": 10510,
+ "ĠMessageBox": 10511,
+ "Ġphotograph": 10512,
+ "164": 10513,
+ "URI": 10514,
+ "ét": 10515,
+ "low": 10516,
+ "ĠGrand": 10517,
+ ".persistence": 10518,
+ "Ġmaintain": 10519,
+ "Ġnums": 10520,
+ "Ġzip": 10521,
+ "ials": 10522,
+ "ĠGets": 10523,
+ "peg": 10524,
+ "ĠBuffer": 10525,
+ "~~~~": 10526,
+ "rastructure": 10527,
+ "ĠPL": 10528,
+ "uen": 10529,
+ "obby": 10530,
+ "sizeof": 10531,
+ "Ġpic": 10532,
+ "Ġseed": 10533,
+ "Ġexperienced": 10534,
+ "Ġodd": 10535,
+ "Ġkick": 10536,
+ "Ġprocedure": 10537,
+ "avigator": 10538,
+ "-on": 10539,
+ ",j": 10540,
+ "ĠAlthough": 10541,
+ "ĠuserId": 10542,
+ "accept": 10543,
+ "Blue": 10544,
+ "IColor": 10545,
+ "layer": 10546,
+ "available": 10547,
+ "Ġends": 10548,
+ ".table": 10549,
+ "Ġdataset": 10550,
+ "bus": 10551,
+ "Ġexplain": 10552,
+ "(pro": 10553,
+ "ĠCommittee": 10554,
+ "Ġnoted": 10555,
+ "]:Ċ": 10556,
+ "Dim": 10557,
+ "stdio": 10558,
+ "154": 10559,
+ ".\",Ċ": 10560,
+ "_source": 10561,
+ "181": 10562,
+ "ĠWeek": 10563,
+ "ĠEdge": 10564,
+ "Ġoperating": 10565,
+ "Ġeste": 10566,
+ "ipl": 10567,
+ "330": 10568,
+ "agination": 10569,
+ "Ġproceed": 10570,
+ "Ġanimation": 10571,
+ ".Models": 10572,
+ "ĠWatch": 10573,
+ "iat": 10574,
+ "Ġoppon": 10575,
+ "/A": 10576,
+ "Report": 10577,
+ "Ġsounds": 10578,
+ "_buf": 10579,
+ "IELD": 10580,
+ "Ġbund": 10581,
+ "ĉget": 10582,
+ ".pr": 10583,
+ "(tmp": 10584,
+ "Ġkid": 10585,
+ ">ĊĊĊ": 10586,
+ "Ġyang": 10587,
+ "NotFound": 10588,
+ "ÑĨ": 10589,
+ "math": 10590,
+ "@gmail": 10591,
+ "ĠLIMIT": 10592,
+ "redients": 10593,
+ "Ġvent": 10594,
+ "avigate": 10595,
+ "Look": 10596,
+ "Ġreligious": 10597,
+ "Ġrand": 10598,
+ "rio": 10599,
+ "(GL": 10600,
+ "_ip": 10601,
+ "uan": 10602,
+ "iciency": 10603,
+ "ĠChange": 10604,
+ ">čĊčĊ": 10605,
+ "ĠEntity": 10606,
+ "Ġrencontre": 10607,
+ "ĠRet": 10608,
+ "plan": 10609,
+ "én": 10610,
+ "BOOL": 10611,
+ "uries": 10612,
+ "train": 10613,
+ "Definition": 10614,
+ "============": 10615,
+ "zz": 10616,
+ "450": 10617,
+ "Animation": 10618,
+ "ĠOK": 10619,
+ "_menu": 10620,
+ ".bl": 10621,
+ "_score": 10622,
+ "Ġacad": 10623,
+ "(System": 10624,
+ "Ġrefresh": 10625,
+ "'=>$": 10626,
+ ".Graphics": 10627,
+ "amento": 10628,
+ "pid": 10629,
+ "tc": 10630,
+ "Ġtips": 10631,
+ "Ġhomes": 10632,
+ "Ġfuel": 10633,
+ "âĸ": 10634,
+ "_helper": 10635,
+ "ĠĠčĊ": 10636,
+ "ĠRoom": 10637,
+ ".Close": 10638,
+ "_attr": 10639,
+ "ĠMount": 10640,
+ "ĠEv": 10641,
+ "arser": 10642,
+ "_top": 10643,
+ "eah": 10644,
+ "ĠDelete": 10645,
+ "ãĢį": 10646,
+ "uke": 10647,
+ "Ġusage": 10648,
+ "aria": 10649,
+ "_dev": 10650,
+ "Ġtexture": 10651,
+ "Ġconversation": 10652,
+ "eper": 10653,
+ "Bean": 10654,
+ "done": 10655,
+ "nonatomic": 10656,
+ "ĠSecond": 10657,
+ "Ġshooting": 10658,
+ "_pre": 10659,
+ "Components": 10660,
+ "Ġ]ĊĊ": 10661,
+ "__,": 10662,
+ "stitution": 10663,
+ ".Char": 10664,
+ ">();ĊĊ": 10665,
+ "Ġpresented": 10666,
+ "Ġwa": 10667,
+ "oker": 10668,
+ "-ĊĊ": 10669,
+ "iner": 10670,
+ "Ġbecoming": 10671,
+ "Ġincident": 10672,
+ "Att": 10673,
+ "162": 10674,
+ "Ġrevealed": 10675,
+ "forc": 10676,
+ "Ġboot": 10677,
+ ".page": 10678,
+ "Enumerator": 10679,
+ "165": 10680,
+ "_->": 10681,
+ "Photo": 10682,
+ "Ġspring": 10683,
+ ".\",": 10684,
+ "ĠDictionary": 10685,
+ "BJECT": 10686,
+ "Ġlocations": 10687,
+ "Ġsamples": 10688,
+ "InputStream": 10689,
+ "ĠBrown": 10690,
+ "Ġstats": 10691,
+ "quality": 10692,
+ "Ñħ": 10693,
+ "-dis": 10694,
+ "Ġhelping": 10695,
+ "Ġped": 10696,
+ "224": 10697,
+ "(se": 10698,
+ "ĠWho": 10699,
+ "alian": 10700,
+ "internal": 10701,
+ "Ġft": 10702,
+ ">().": 10703,
+ "->{": 10704,
+ "Ġmine": 10705,
+ "Ġsector": 10706,
+ "Ġgro": 10707,
+ "Ġopportunities": 10708,
+ "Ġü": 10709,
+ "Ġmp": 10710,
+ "Ġalleged": 10711,
+ "Ġdoubt": 10712,
+ "Mouse": 10713,
+ "About": 10714,
+ "_part": 10715,
+ "Ġchair": 10716,
+ "Ġstopped": 10717,
+ "161": 10718,
+ "loop": 10719,
+ "entities": 10720,
+ "Ġapps": 10721,
+ "ansion": 10722,
+ "Ġmental": 10723,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 10724,
+ "FR": 10725,
+ "Ġdefend": 10726,
+ "care": 10727,
+ "Ġideal": 10728,
+ "/api": 10729,
+ "urface": 10730,
+ "011": 10731,
+ "Ġele": 10732,
+ "ulator": 10733,
+ "ĠRights": 10734,
+ "anguages": 10735,
+ "Ġfunds": 10736,
+ "Ġadapt": 10737,
+ "Attributes": 10738,
+ "Ġdeploy": 10739,
+ "opts": 10740,
+ "Ġvalidation": 10741,
+ "Ġconcerns": 10742,
+ "uce": 10743,
+ ".num": 10744,
+ "ulture": 10745,
+ "ila": 10746,
+ "Ġcup": 10747,
+ "Ġpure": 10748,
+ ".Fore": 10749,
+ "183": 10750,
+ "ĠHashMap": 10751,
+ ".valueOf": 10752,
+ "asm": 10753,
+ "MO": 10754,
+ "Ġcs": 10755,
+ "Ġstores": 10756,
+ "Ġ************************************************************************": 10757,
+ "Ġcommunication": 10758,
+ "mem": 10759,
+ ".EventHandler": 10760,
+ ".Status": 10761,
+ "_right": 10762,
+ ".setOn": 10763,
+ "Sheet": 10764,
+ "Ġidentify": 10765,
+ "enerated": 10766,
+ "ordered": 10767,
+ "Ġ\"[": 10768,
+ "Ġswe": 10769,
+ "Condition": 10770,
+ "ĠAccording": 10771,
+ "Ġprepare": 10772,
+ "Ġrob": 10773,
+ "Pool": 10774,
+ "Ġsport": 10775,
+ "rv": 10776,
+ "ĠRouter": 10777,
+ "Ġalternative": 10778,
+ "([]": 10779,
+ "ĠChicago": 10780,
+ "ipher": 10781,
+ "ische": 10782,
+ "ĠDirector": 10783,
+ "kl": 10784,
+ "ĠWil": 10785,
+ "keys": 10786,
+ "Ġmysql": 10787,
+ "Ġwelcome": 10788,
+ "king": 10789,
+ "ĠManager": 10790,
+ "Ġcaught": 10791,
+ ")}Ċ": 10792,
+ "Score": 10793,
+ "_PR": 10794,
+ "Ġsurvey": 10795,
+ "hab": 10796,
+ "Headers": 10797,
+ "ADER": 10798,
+ "Ġdecor": 10799,
+ "Ġturns": 10800,
+ "Ġradius": 10801,
+ "errupt": 10802,
+ "Cor": 10803,
+ "Ġmel": 10804,
+ "Ġintr": 10805,
+ "(q": 10806,
+ "ĠAC": 10807,
+ "amos": 10808,
+ "MAX": 10809,
+ "ĠGrid": 10810,
+ "ĠJesus": 10811,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 10812,
+ ".DE": 10813,
+ "Ġts": 10814,
+ "Ġlinked": 10815,
+ "free": 10816,
+ "ĠQt": 10817,
+ "Ġ/**čĊ": 10818,
+ "Ġfaster": 10819,
+ "ctr": 10820,
+ "_J": 10821,
+ "DT": 10822,
+ ".Check": 10823,
+ "Ġcombination": 10824,
+ "Ġintended": 10825,
+ "-the": 10826,
+ "-type": 10827,
+ "182": 10828,
+ "ectors": 10829,
+ "ami": 10830,
+ "uting": 10831,
+ "Ġuma": 10832,
+ "XML": 10833,
+ "UCT": 10834,
+ "Ap": 10835,
+ "ĠRandom": 10836,
+ "Ġran": 10837,
+ ".sort": 10838,
+ "Ġsorted": 10839,
+ ".Un": 10840,
+ "401": 10841,
+ "_PER": 10842,
+ "itory": 10843,
+ "Ġpriority": 10844,
+ "ĠGal": 10845,
+ "ĠOld": 10846,
+ "hot": 10847,
+ "ĠDisplay": 10848,
+ "(sub": 10849,
+ "_TH": 10850,
+ "_Y": 10851,
+ "ĠCare": 10852,
+ "loading": 10853,
+ "Kind": 10854,
+ "_handle": 10855,
+ ",,": 10856,
+ "rase": 10857,
+ "_replace": 10858,
+ ".addEventListener": 10859,
+ "ĠRT": 10860,
+ "172": 10861,
+ "Ġentered": 10862,
+ "gers": 10863,
+ "Ġich": 10864,
+ "(start": 10865,
+ "205": 10866,
+ "/app": 10867,
+ "Ġbrother": 10868,
+ "Memory": 10869,
+ "Outlet": 10870,
+ "Ġutf": 10871,
+ "prec": 10872,
+ "Ġnavigation": 10873,
+ "ORK": 10874,
+ "Ġdst": 10875,
+ "Detail": 10876,
+ "Ġaudience": 10877,
+ "Ġdur": 10878,
+ "Ġcluster": 10879,
+ "unched": 10880,
+ "Ġ],": 10881,
+ "Ġcomfortable": 10882,
+ ".values": 10883,
+ "ĠTotal": 10884,
+ "Ġsnap": 10885,
+ "Ġstandards": 10886,
+ "Ġperformed": 10887,
+ "hand": 10888,
+ "(\"@": 10889,
+ "åŃ": 10890,
+ "Ġphil": 10891,
+ "ibr": 10892,
+ "trim": 10893,
+ "Ġforget": 10894,
+ "157": 10895,
+ "Ġdoctor": 10896,
+ ".TextBox": 10897,
+ "377": 10898,
+ "icons": 10899,
+ ",s": 10900,
+ "ĠOp": 10901,
+ "Sm": 10902,
+ "Stop": 10903,
+ "ĉList": 10904,
+ "ĉu": 10905,
+ "Comment": 10906,
+ "_VERSION": 10907,
+ ".Xtra": 10908,
+ "Person": 10909,
+ "rb": 10910,
+ "LOB": 10911,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ": 10912,
+ "ĠCentral": 10913,
+ "270": 10914,
+ "ICK": 10915,
+ "raq": 10916,
+ "Ġputting": 10917,
+ "Ġmd": 10918,
+ "ĠLove": 10919,
+ "Program": 10920,
+ "Border": 10921,
+ "oor": 10922,
+ "Ġallowing": 10923,
+ "after": 10924,
+ "Ġentries": 10925,
+ "ĠMaybe": 10926,
+ "]).": 10927,
+ "ĠShort": 10928,
+ ")\\": 10929,
+ ".now": 10930,
+ "friend": 10931,
+ "Ġprefer": 10932,
+ "ĠGPIO": 10933,
+ "osis": 10934,
+ "ĠGameObject": 10935,
+ "Ġskip": 10936,
+ "Ġcompetition": 10937,
+ "_match": 10938,
+ "lications": 10939,
+ "_CONT": 10940,
+ ".groupBox": 10941,
+ "Ġals": 10942,
+ "666": 10943,
+ "\"We": 10944,
+ "_eq": 10945,
+ "lan": 10946,
+ "_search": 10947,
+ "ĠMusic": 10948,
+ "asis": 10949,
+ "Ġbind": 10950,
+ "ĠIsland": 10951,
+ "rum": 10952,
+ "(E": 10953,
+ "Ġseat": 10954,
+ "Video": 10955,
+ "Ġack": 10956,
+ "reek": 10957,
+ "={()": 10958,
+ "Ġrating": 10959,
+ "Ġrestaurant": 10960,
+ "456": 10961,
+ "DEX": 10962,
+ "(buf": 10963,
+ "pping": 10964,
+ "uality": 10965,
+ "Ġleague": 10966,
+ "176": 10967,
+ "Ġfocused": 10968,
+ "apon": 10969,
+ "$data": 10970,
+ "CLUD": 10971,
+ "CLUDING": 10972,
+ "Ġabsolute": 10973,
+ "(query": 10974,
+ "Ġtells": 10975,
+ "Ang": 10976,
+ "Ġcommunities": 10977,
+ "Ġhonest": 10978,
+ "oking": 10979,
+ "Ġapart": 10980,
+ "arity": 10981,
+ "/$": 10982,
+ "_module": 10983,
+ "ĠEnc": 10984,
+ ".an": 10985,
+ ".Config": 10986,
+ "Cre": 10987,
+ "Ġshock": 10988,
+ "ĠArab": 10989,
+ "IENT": 10990,
+ "/re": 10991,
+ "Ġretrie": 10992,
+ "ycler": 10993,
+ "isa": 10994,
+ "ĠOrgan": 10995,
+ ".graph": 10996,
+ "Ġí": 10997,
+ "ĠBAS": 10998,
+ "Enum": 10999,
+ "Ġpossibly": 11000,
+ "ÑĢаÐ": 11001,
+ "ĠJapanese": 11002,
+ "Ġcraft": 11003,
+ "ĠPlace": 11004,
+ "Ġtalent": 11005,
+ "Ġfunding": 11006,
+ "Ġconfirmed": 11007,
+ "Ġcycle": 11008,
+ "/x": 11009,
+ "GE": 11010,
+ "Ġhearing": 11011,
+ "Ġplants": 11012,
+ "Ġmouth": 11013,
+ "pages": 11014,
+ "oria": 11015,
+ "ĠRemove": 11016,
+ "_total": 11017,
+ "Ġod": 11018,
+ "ollapse": 11019,
+ "door": 11020,
+ "Ġbought": 11021,
+ "Ġaddr": 11022,
+ "ARCH": 11023,
+ "_dim": 11024,
+ "dden": 11025,
+ "Ġdecades": 11026,
+ "REQUEST": 11027,
+ "Ġversions": 11028,
+ "fire": 11029,
+ "006": 11030,
+ "Ġmoves": 11031,
+ "fb": 11032,
+ "Ġcoffee": 11033,
+ ".connect": 11034,
+ "ĠRow": 11035,
+ "Ġschema": 11036,
+ "Scope": 11037,
+ "-Type": 11038,
+ "Ġfighting": 11039,
+ "Ġretail": 11040,
+ "Ġmodified": 11041,
+ "TF": 11042,
+ "Files": 11043,
+ "nie": 11044,
+ "_command": 11045,
+ "stone": 11046,
+ "ĠÑĤ": 11047,
+ "_thread": 11048,
+ "Ġbond": 11049,
+ "ĠDevelopment": 11050,
+ "Ġpt": 11051,
+ "FORM": 11052,
+ "plet": 11053,
+ "Ġidentified": 11054,
+ "cpp": 11055,
+ "206": 11056,
+ "225": 11057,
+ "Ġcoding": 11058,
+ "oked": 11059,
+ "ĠMaster": 11060,
+ "IDTH": 11061,
+ "Ġresidents": 11062,
+ "redit": 11063,
+ "ĠPhoto": 11064,
+ "=-": 11065,
+ "unte": 11066,
+ "ateur": 11067,
+ "159": 11068,
+ "_STATE": 11069,
+ "ĠSing": 11070,
+ "Ġsheet": 11071,
+ ".val": 11072,
+ "orse": 11073,
+ "Ġhers": 11074,
+ "Ġdetermined": 11075,
+ "Common": 11076,
+ "Ġwed": 11077,
+ "_queue": 11078,
+ "PH": 11079,
+ "ĠAtl": 11080,
+ "cred": 11081,
+ "/LICENSE": 11082,
+ "Ġmes": 11083,
+ "Ġadvanced": 11084,
+ ".java": 11085,
+ ".Sh": 11086,
+ "Go": 11087,
+ "kill": 11088,
+ "fp": 11089,
+ "_settings": 11090,
+ "Ġpal": 11091,
+ "Ġtruck": 11092,
+ "Ġcombined": 11093,
+ "Ġ\"${": 11094,
+ "ĠCorpor": 11095,
+ "Ġjoined": 11096,
+ "ĠJose": 11097,
+ "ĠCup": 11098,
+ "uns": 11099,
+ "estival": 11100,
+ "levision": 11101,
+ "Ġbroken": 11102,
+ "Ġmarriage": 11103,
+ "ĠWestern": 11104,
+ "Ġrepresents": 11105,
+ "ĠTitle": 11106,
+ "Ġss": 11107,
+ ".Ass": 11108,
+ "ongoose": 11109,
+ "iento": 11110,
+ "<>();Ċ": 11111,
+ "Ġabsolutely": 11112,
+ "Ġsmooth": 11113,
+ "TERN": 11114,
+ "ĠUnless": 11115,
+ "Word": 11116,
+ "Ġmerge": 11117,
+ "igan": 11118,
+ "ĠVol": 11119,
+ "Ġnn": 11120,
+ ".getId": 11121,
+ "Ġз": 11122,
+ "171": 11123,
+ "Ġsexy": 11124,
+ "Ġseeking": 11125,
+ "Single": 11126,
+ ".this": 11127,
+ "179": 11128,
+ "Ġkom": 11129,
+ "bound": 11130,
+ ";\"": 11131,
+ "ĠfontSize": 11132,
+ "_df": 11133,
+ "Ġinjury": 11134,
+ "(H": 11135,
+ "Ġissued": 11136,
+ "_END": 11137,
+ ":self": 11138,
+ "020": 11139,
+ "Ġpatch": 11140,
+ "Ġleaves": 11141,
+ "Ġadopt": 11142,
+ "FileName": 11143,
+ "ãĢIJ": 11144,
+ "Ġexecutive": 11145,
+ "ĠByte": 11146,
+ "]))Ċ": 11147,
+ "Ġnu": 11148,
+ "outing": 11149,
+ "cluding": 11150,
+ "-R": 11151,
+ ".options": 11152,
+ "Ġsubstant": 11153,
+ "avax": 11154,
+ "ĠBUT": 11155,
+ "Ġtechnical": 11156,
+ "Ġtwice": 11157,
+ "Ġmás": 11158,
+ "Ġunivers": 11159,
+ "yr": 11160,
+ "Ġdrag": 11161,
+ "ĠDC": 11162,
+ "Ġsed": 11163,
+ "Ġbot": 11164,
+ "ĠPal": 11165,
+ "ĠHall": 11166,
+ "forcement": 11167,
+ "Ġauch": 11168,
+ ".mod": 11169,
+ "notation": 11170,
+ "_files": 11171,
+ ".line": 11172,
+ "_flag": 11173,
+ "[name": 11174,
+ "Ġresolution": 11175,
+ "Ġbott": 11176,
+ "(\"[": 11177,
+ "ende": 11178,
+ "(arr": 11179,
+ "Free": 11180,
+ "(@\"": 11181,
+ "ĠDistrict": 11182,
+ "PEC": 11183,
+ ":-": 11184,
+ "Picker": 11185,
+ "ĠJo": 11186,
+ "ĠĠĠĠĠĊ": 11187,
+ "ĠRiver": 11188,
+ "_rows": 11189,
+ "Ġhelpful": 11190,
+ "Ġmassive": 11191,
+ "---Ċ": 11192,
+ "Ġmeasures": 11193,
+ "007": 11194,
+ "ĠRuntime": 11195,
+ "Ġworry": 11196,
+ "ĠSpec": 11197,
+ "ĉD": 11198,
+ "ãĢij": 11199,
+ "Ġ){Ċ": 11200,
+ "Ġworse": 11201,
+ "(filename": 11202,
+ "Ġlay": 11203,
+ "Ġmagic": 11204,
+ "ĠTheir": 11205,
+ "oul": 11206,
+ "stroy": 11207,
+ "ĠWhere": 11208,
+ "280": 11209,
+ "Ġsudden": 11210,
+ "Ġdefe": 11211,
+ "Ġbinding": 11212,
+ "Ġflight": 11213,
+ "ĠOnInit": 11214,
+ "ĠWomen": 11215,
+ "ĠPolicy": 11216,
+ "Ġdrugs": 11217,
+ "ishing": 11218,
+ "('../": 11219,
+ "ĠMel": 11220,
+ "peat": 11221,
+ "tor": 11222,
+ "Ġproposed": 11223,
+ "Ġstated": 11224,
+ "_RES": 11225,
+ "Ġeast": 11226,
+ "212": 11227,
+ "ĠCONDITION": 11228,
+ "_desc": 11229,
+ "Ġwinning": 11230,
+ "folio": 11231,
+ "Mapper": 11232,
+ "ĠPan": 11233,
+ "ĠAnge": 11234,
+ ".servlet": 11235,
+ "Ġcopies": 11236,
+ "LM": 11237,
+ "Ġvm": 11238,
+ "åį": 11239,
+ "Ġdictionary": 11240,
+ "Seg": 11241,
+ "177": 11242,
+ "elines": 11243,
+ "ĠSend": 11244,
+ "Ġiron": 11245,
+ "ĠFort": 11246,
+ "166": 11247,
+ ".domain": 11248,
+ "Ġdebate": 11249,
+ "NotNull": 11250,
+ "eq": 11251,
+ "acher": 11252,
+ "lf": 11253,
+ "ĉfmt": 11254,
+ "Ġlawy": 11255,
+ "178": 11256,
+ "ÄŁ": 11257,
+ "ĠMen": 11258,
+ "Ġtrim": 11259,
+ "(NULL": 11260,
+ "Ġ!!": 11261,
+ "Ġpad": 11262,
+ "Ġfollows": 11263,
+ "\"][\"": 11264,
+ "requ": 11265,
+ "ĠEp": 11266,
+ ".github": 11267,
+ "(img": 11268,
+ "eto": 11269,
+ "('\\": 11270,
+ "Services": 11271,
+ "umbnail": 11272,
+ "_main": 11273,
+ "pleted": 11274,
+ "fortunately": 11275,
+ "Ġwindows": 11276,
+ "Ġplane": 11277,
+ "ĠConnection": 11278,
+ ".local": 11279,
+ "uard": 11280,
+ "}\\": 11281,
+ "==\"": 11282,
+ "andon": 11283,
+ "ĠRoy": 11284,
+ "west": 11285,
+ "158": 11286,
+ "iginal": 11287,
+ "emies": 11288,
+ "itz": 11289,
+ "'):Ċ": 11290,
+ "ĠPeter": 11291,
+ "Ġtough": 11292,
+ "Ġreduced": 11293,
+ "Ġcalculate": 11294,
+ "Ġrapid": 11295,
+ "customer": 11296,
+ "Ġefficient": 11297,
+ "Ġmedium": 11298,
+ "Ġfell": 11299,
+ ".ref": 11300,
+ "ĠCas": 11301,
+ "Ġfeedback": 11302,
+ "Speed": 11303,
+ "(output": 11304,
+ "aje": 11305,
+ "Ġcategories": 11306,
+ "Ġfee": 11307,
+ "};": 11308,
+ "Ġdeleted": 11309,
+ "reh": 11310,
+ "Ġproof": 11311,
+ "Desc": 11312,
+ "Build": 11313,
+ "Ġsides": 11314,
+ ".ArrayList": 11315,
+ "-%": 11316,
+ "ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ": 11317,
+ "ر": 11318,
+ ".match": 11319,
+ "ли": 11320,
+ "Ġfeels": 11321,
+ "Ġachieve": 11322,
+ "Ġclim": 11323,
+ "_ON": 11324,
+ "ĠCD": 11325,
+ "Ġteacher": 11326,
+ "_current": 11327,
+ "bn": 11328,
+ "_PL": 11329,
+ "isting": 11330,
+ "Enable": 11331,
+ "GEN": 11332,
+ "Ġtv": 11333,
+ "Ġsock": 11334,
+ "Ġplays": 11335,
+ "Ġdiscount": 11336,
+ "ĠKE": 11337,
+ "ĠDebug": 11338,
+ "Fore": 11339,
+ "ĠIraq": 11340,
+ "Ġappearance": 11341,
+ "Mon": 11342,
+ "Ġstyled": 11343,
+ "ĠHuman": 11344,
+ "iot": 11345,
+ "ĠHistory": 11346,
+ "Ġsac": 11347,
+ "ĠCollection": 11348,
+ "Ġrecommended": 11349,
+ ".Selected": 11350,
+ "Ġorganizations": 11351,
+ "Ġdiscovered": 11352,
+ "cohol": 11353,
+ "adas": 11354,
+ "ĠThomas": 11355,
+ "May": 11356,
+ "Ġconserv": 11357,
+ "Ġdomin": 11358,
+ "ĠFollow": 11359,
+ "ĠSection": 11360,
+ "ĠThanks": 11361,
+ "Username": 11362,
+ "Ġrecipe": 11363,
+ "Ġwonderful": 11364,
+ ".sleep": 11365,
+ "_if": 11366,
+ "ĉĊĉĊ": 11367,
+ "orno": 11368,
+ "Ġru": 11369,
+ "_target": 11370,
+ ".\"\"": 11371,
+ "à¦": 11372,
+ "EventArgs": 11373,
+ "Ġinputs": 11374,
+ "Ġfif": 11375,
+ "Ġvision": 11376,
+ "cy": 11377,
+ "ĠSeries": 11378,
+ ")(((": 11379,
+ "Ġtrading": 11380,
+ "Ġmarker": 11381,
+ "Begin": 11382,
+ "Ġtypically": 11383,
+ "Ġcauses": 11384,
+ "dropdown": 11385,
+ "_DEBUG": 11386,
+ "260": 11387,
+ "Ġdetect": 11388,
+ "country": 11389,
+ "!\");Ċ": 11390,
+ "ĉR": 11391,
+ "appy": 11392,
+ "Ġcref": 11393,
+ "('<": 11394,
+ "\"=>": 11395,
+ "ĠLE": 11396,
+ "reader": 11397,
+ "Ġadministr": 11398,
+ "õ": 11399,
+ "ucket": 11400,
+ "Ġfashion": 11401,
+ ".char": 11402,
+ "izar": 11403,
+ "Ġdisable": 11404,
+ "Ġsuc": 11405,
+ "ĠLive": 11406,
+ "issue": 11407,
+ "Ġmetadata": 11408,
+ "flags": 11409,
+ "ĠðŁ": 11410,
+ "Ġcommitted": 11411,
+ "Ġva": 11412,
+ "Ġrough": 11413,
+ "Ġ'''Ċ": 11414,
+ "Ġhighlight": 11415,
+ "_vars": 11416,
+ "VO": 11417,
+ "Ġencoding": 11418,
+ "-Z": 11419,
+ "_sign": 11420,
+ "$(\"#": 11421,
+ "Ġrain": 11422,
+ "reatest": 11423,
+ "ĠEND": 11424,
+ "Selection": 11425,
+ "Ġcandidates": 11426,
+ "Ġsav": 11427,
+ ".Empty": 11428,
+ "Ġdecisions": 11429,
+ "Ġcollabor": 11430,
+ "ridge": 11431,
+ "feed": 11432,
+ "ression": 11433,
+ "Ġpersons": 11434,
+ "VM": 11435,
+ "008": 11436,
+ "ega": 11437,
+ "_BIT": 11438,
+ "According": 11439,
+ "acked": 11440,
+ "Ġdollars": 11441,
+ "_loss": 11442,
+ "ĠCost": 11443,
+ "}\"Ċ": 11444,
+ "Notification": 11445,
+ "Ġprostit": 11446,
+ "Ġauthority": 11447,
+ ".rec": 11448,
+ "Ġspokes": 11449,
+ "ĠToday": 11450,
+ "istant": 11451,
+ "ĠHead": 11452,
+ "âĢĿ.": 11453,
+ "ertainment": 11454,
+ "cean": 11455,
+ "culate": 11456,
+ "Ġven": 11457,
+ "However": 11458,
+ "_arr": 11459,
+ "Ġtokens": 11460,
+ "Graph": 11461,
+ "ĠJud": 11462,
+ "ĠVirgin": 11463,
+ "ĠSerial": 11464,
+ "unning": 11465,
+ "Mutable": 11466,
+ "agers": 11467,
+ ".csv": 11468,
+ "Ġdeveloping": 11469,
+ "Ġinstructions": 11470,
+ "Ġpromise": 11471,
+ "Ġrequested": 11472,
+ "_encode": 11473,
+ "/\"": 11474,
+ "ĠIcon": 11475,
+ "uilt": 11476,
+ "-day": 11477,
+ "Ġintelligence": 11478,
+ ".IS": 11479,
+ "ĠObservable": 11480,
+ "ĠHard": 11481,
+ "Bool": 11482,
+ "211": 11483,
+ "idential": 11484,
+ ".Anchor": 11485,
+ "Ġselling": 11486,
+ "CI": 11487,
+ "AGES": 11488,
+ "tle": 11489,
+ "bur": 11490,
+ "UFFER": 11491,
+ "RY": 11492,
+ "Ġbigger": 11493,
+ "Ġrat": 11494,
+ "Ġfamous": 11495,
+ "Ġtypename": 11496,
+ "Ġexplained": 11497,
+ "}}Ċ": 11498,
+ "Ġnuclear": 11499,
+ "-N": 11500,
+ "Ġcrisis": 11501,
+ "ĠEnter": 11502,
+ "Ġanswers": 11503,
+ "/${": 11504,
+ "/pl": 11505,
+ "Ġsequ": 11506,
+ "_next": 11507,
+ "mask": 11508,
+ "Ġstanding": 11509,
+ "Ġplenty": 11510,
+ "ĠCross": 11511,
+ "ĉret": 11512,
+ "dro": 11513,
+ "ĠCast": 11514,
+ "167": 11515,
+ "=true": 11516,
+ "ĠChris": 11517,
+ "icio": 11518,
+ "ĠMike": 11519,
+ "Decimal": 11520,
+ "addComponent": 11521,
+ "Len": 11522,
+ "Ġcock": 11523,
+ "Ġ#{": 11524,
+ "URN": 11525,
+ "": 11657,
+ "Ġ*=": 11658,
+ "ĠPS": 11659,
+ "Ġdangerous": 11660,
+ "[p": 11661,
+ "OME": 11662,
+ "Other": 11663,
+ "ĠStringBuilder": 11664,
+ "Points": 11665,
+ "heading": 11666,
+ "Ġcurrency": 11667,
+ "Ġpercentage": 11668,
+ "_API": 11669,
+ "Ġclassic": 11670,
+ "thead": 11671,
+ "ĠMO": 11672,
+ "FE": 11673,
+ "Idx": 11674,
+ "await": 11675,
+ "Ġè": 11676,
+ "Ġaccident": 11677,
+ "Ġvariant": 11678,
+ "Ġmyst": 11679,
+ "ĠLand": 11680,
+ "ĠBre": 11681,
+ "Ġharm": 11682,
+ "ĠAcc": 11683,
+ "Ġcharged": 11684,
+ "iones": 11685,
+ "Visibility": 11686,
+ "arry": 11687,
+ "ĠLanguage": 11688,
+ "Ġwalking": 11689,
+ "\".ĊĊ": 11690,
+ "ifer": 11691,
+ "Ġleadership": 11692,
+ ".From": 11693,
+ "ynam": 11694,
+ "Ġtimestamp": 11695,
+ "ipt": 11696,
+ "ĠHas": 11697,
+ "REFER": 11698,
+ "ĠIts": 11699,
+ "Ġlistener": 11700,
+ "UTE": 11701,
+ "213": 11702,
+ "_description": 11703,
+ "Ġexperiences": 11704,
+ "Ġcreates": 11705,
+ "RS": 11706,
+ "cart": 11707,
+ "black": 11708,
+ "Ġchoices": 11709,
+ "war": 11710,
+ "750": 11711,
+ "Ġ'''": 11712,
+ "Ġordered": 11713,
+ "Ġevening": 11714,
+ "Ġpil": 11715,
+ "Ġtun": 11716,
+ "ĠBad": 11717,
+ "(app": 11718,
+ "random": 11719,
+ "Ġexplicit": 11720,
+ "Ġarrived": 11721,
+ "Ġfly": 11722,
+ "Ġeconom": 11723,
+ "-mail": 11724,
+ "Ġlists": 11725,
+ "Ġarchitect": 11726,
+ "234": 11727,
+ "ĠPay": 11728,
+ "Ġds": 11729,
+ "ĠSol": 11730,
+ "Ġvehicles": 11731,
+ "Hz": 11732,
+ "-com": 11733,
+ "Ġking": 11734,
+ "_equal": 11735,
+ "ĠHelp": 11736,
+ "Ġabuse": 11737,
+ "480": 11738,
+ "169": 11739,
+ "--;Ċ": 11740,
+ "Ġextr": 11741,
+ "Ġchemical": 11742,
+ "ä¿": 11743,
+ "Ġorient": 11744,
+ "Ġbreath": 11745,
+ "ĠSpace": 11746,
+ "(element": 11747,
+ "wait": 11748,
+ "DED": 11749,
+ "igma": 11750,
+ "Ġentr": 11751,
+ "Ġsob": 11752,
+ "-name": 11753,
+ "Ġaffected": 11754,
+ "ika": 11755,
+ "Ġcoal": 11756,
+ "_work": 11757,
+ "Ġhundreds": 11758,
+ "Ġpolitics": 11759,
+ "subject": 11760,
+ "Ġconsumer": 11761,
+ "ANGE": 11762,
+ "Ġrepeated": 11763,
+ "Send": 11764,
+ "Ġ#[": 11765,
+ "Ġprotocol": 11766,
+ "Ġleads": 11767,
+ "useum": 11768,
+ "Every": 11769,
+ "808": 11770,
+ "174": 11771,
+ "Import": 11772,
+ "(count": 11773,
+ "Ġchallenges": 11774,
+ "Ġnovel": 11775,
+ "Ġdepart": 11776,
+ "bits": 11777,
+ ".Current": 11778,
+ "Ġ`${": 11779,
+ "oting": 11780,
+ "(\\": 11781,
+ "Ġcreative": 11782,
+ "Ġbuff": 11783,
+ "Ġintroduced": 11784,
+ "usic": 11785,
+ "modules": 11786,
+ "Are": 11787,
+ "-doc": 11788,
+ "language": 11789,
+ "_cache": 11790,
+ "Ġtod": 11791,
+ "?>": 11792,
+ "omething": 11793,
+ "Ġhun": 11794,
+ "åº": 11795,
+ "aters": 11796,
+ "Intent": 11797,
+ "Ġimplemented": 11798,
+ "ĠCase": 11799,
+ "Children": 11800,
+ "Ġnotification": 11801,
+ "Renderer": 11802,
+ "Wrapper": 11803,
+ "Objects": 11804,
+ "tl": 11805,
+ ".Contains": 11806,
+ "Plugin": 11807,
+ ".row": 11808,
+ "Ġforg": 11809,
+ "Ġpermit": 11810,
+ "Ġtargets": 11811,
+ "ĠIF": 11812,
+ "Ġtip": 11813,
+ "sex": 11814,
+ "Ġsupports": 11815,
+ "Ġfold": 11816,
+ "photo": 11817,
+ "},čĊ": 11818,
+ "Ġgoogle": 11819,
+ "$('#": 11820,
+ "Ġsharing": 11821,
+ "Ġgoods": 11822,
+ "vs": 11823,
+ "ĠDan": 11824,
+ "Rate": 11825,
+ "ĠMartin": 11826,
+ "Ġmanner": 11827,
+ "lie": 11828,
+ ".The": 11829,
+ "Internal": 11830,
+ "ĠCONTR": 11831,
+ "Mock": 11832,
+ "RIGHT": 11833,
+ "Ġ'{": 11834,
+ "Ġcontrols": 11835,
+ "Mat": 11836,
+ "Ġmand": 11837,
+ "Ġextended": 11838,
+ "Ok": 11839,
+ "Ġembed": 11840,
+ "Ġplanet": 11841,
+ "ĠNon": 11842,
+ "-ch": 11843,
+ ")\",": 11844,
+ "epar": 11845,
+ "Ġbelieved": 11846,
+ "ĠEnvironment": 11847,
+ "ĠFriend": 11848,
+ "-res": 11849,
+ "Ġhandling": 11850,
+ "nic": 11851,
+ "-level": 11852,
+ "scri": 11853,
+ "Xml": 11854,
+ "BE": 11855,
+ "ungen": 11856,
+ "Ġalter": 11857,
+ "[idx": 11858,
+ "Pop": 11859,
+ "cam": 11860,
+ "Ġ(((": 11861,
+ "Ġshipping": 11862,
+ "Ġbattery": 11863,
+ "iddleware": 11864,
+ "MC": 11865,
+ "Ġimpl": 11866,
+ "otation": 11867,
+ "ĠLab": 11868,
+ "